2023-06-10T20:38:47.647 INFO:root:teuthology version: 0.0.1.dev120+g2ef0dcd 2023-06-10T20:38:47.648 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118 --owner scheduled_teuthology@teuthology --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --verbose --name teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/orig.config.yaml 2023-06-10T20:38:47.715 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-06-10T20:38:47.810 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7301118' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 1647 sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 targets: smithi064.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= smithi152.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61 timestamp: 2023-06-10_14:23:08 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.23459 2023-06-10T20:38:47.810 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa; will attempt to use it 2023-06-10T20:38:47.811 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa/tasks 2023-06-10T20:38:47.811 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-06-10T20:38:47.813 INFO:teuthology.task.internal:Checking packages... 2023-06-10T20:38:47.841 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '5937c68aed42cde57f938ad8f8bb264f13176009' 2023-06-10T20:38:47.841 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:38:47.841 INFO:teuthology.packaging:ref: None 2023-06-10T20:38:47.842 INFO:teuthology.packaging:tag: None 2023-06-10T20:38:47.842 INFO:teuthology.packaging:branch: reef 2023-06-10T20:38:47.842 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:38:47.842 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-06-10T20:38:48.253 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-3603.g5937c68a 2023-06-10T20:38:48.253 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-06-10T20:38:48.262 INFO:teuthology.task.internal:no buildpackages task found 2023-06-10T20:38:48.263 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-06-10T20:38:48.272 INFO:teuthology.task.internal:Saving configuration 2023-06-10T20:38:48.294 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-06-10T20:38:48.300 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-10T20:38:48.330 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-10 20:33:39.468106', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-06-10T20:38:48.353 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-10 20:33:39.466350', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-06-10T20:38:48.353 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-06-10T20:38:48.360 INFO:teuthology.task.internal:roles: ubuntu@smithi064.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-06-10T20:38:48.360 INFO:teuthology.task.internal:roles: ubuntu@smithi152.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-06-10T20:38:48.360 INFO:teuthology.run_tasks:Running task console_log... 2023-06-10T20:38:48.442 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8f0ad2e1f0>, signals=[15]) 2023-06-10T20:38:48.445 INFO:teuthology.run_tasks:Running task internal.connect... 2023-06-10T20:38:48.451 INFO:teuthology.task.internal:Opening connections... 2023-06-10T20:38:48.452 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi064.front.sepia.ceph.com 2023-06-10T20:38:48.453 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-10T20:38:48.525 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi152.front.sepia.ceph.com 2023-06-10T20:38:48.526 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-10T20:38:48.596 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-06-10T20:38:48.603 DEBUG:teuthology.orchestra.run.smithi064:> uname -m 2023-06-10T20:38:48.624 INFO:teuthology.orchestra.run.smithi064.stdout:x86_64 2023-06-10T20:38:48.624 DEBUG:teuthology.orchestra.run.smithi064:> cat /etc/os-release 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:NAME="Red Hat Enterprise Linux" 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION="8.6 (Ootpa)" 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:ID="rhel" 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:ID_LIKE="fedora" 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:VERSION_ID="8.6" 2023-06-10T20:38:48.684 INFO:teuthology.orchestra.run.smithi064.stdout:PLATFORM_ID="platform:el8" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:ANSI_COLOR="0;31" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:HOME_URL="https://www.redhat.com/" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-06-10T20:38:48.685 INFO:teuthology.orchestra.run.smithi064.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-06-10T20:38:48.686 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:38:48.686 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-06-10T20:38:48.686 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-06-10T20:38:48.686 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-06-10T20:38:48.686 INFO:teuthology.orchestra.run.smithi064.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-06-10T20:38:48.687 INFO:teuthology.lock.ops:Updating smithi064.front.sepia.ceph.com on lock server 2023-06-10T20:38:48.711 DEBUG:teuthology.orchestra.run.smithi152:> uname -m 2023-06-10T20:38:48.731 INFO:teuthology.orchestra.run.smithi152.stdout:x86_64 2023-06-10T20:38:48.731 DEBUG:teuthology.orchestra.run.smithi152:> cat /etc/os-release 2023-06-10T20:38:48.794 INFO:teuthology.orchestra.run.smithi152.stdout:NAME="Red Hat Enterprise Linux" 2023-06-10T20:38:48.794 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION="8.6 (Ootpa)" 2023-06-10T20:38:48.794 INFO:teuthology.orchestra.run.smithi152.stdout:ID="rhel" 2023-06-10T20:38:48.794 INFO:teuthology.orchestra.run.smithi152.stdout:ID_LIKE="fedora" 2023-06-10T20:38:48.794 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION_ID="8.6" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:PLATFORM_ID="platform:el8" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:ANSI_COLOR="0;31" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:HOME_URL="https://www.redhat.com/" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-06-10T20:38:48.795 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:38:48.796 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-06-10T20:38:48.796 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-06-10T20:38:48.796 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-06-10T20:38:48.796 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-06-10T20:38:48.796 INFO:teuthology.lock.ops:Updating smithi152.front.sepia.ceph.com on lock server 2023-06-10T20:38:48.819 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-06-10T20:38:48.828 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-06-10T20:38:48.834 INFO:teuthology.task.internal:Checking for old test directory... 2023-06-10T20:38:48.835 DEBUG:teuthology.orchestra.run.smithi064:> test '!' -e /home/ubuntu/cephtest 2023-06-10T20:38:48.837 DEBUG:teuthology.orchestra.run.smithi152:> test '!' -e /home/ubuntu/cephtest 2023-06-10T20:38:48.857 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-06-10T20:38:48.863 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-06-10T20:38:48.863 DEBUG:teuthology.orchestra.run.smithi064:> test -z $(ls -A /var/lib/ceph) 2023-06-10T20:38:48.897 DEBUG:teuthology.orchestra.run.smithi152:> test -z $(ls -A /var/lib/ceph) 2023-06-10T20:38:48.923 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-06-10T20:38:48.982 INFO:teuthology.run_tasks:Running task kernel... 2023-06-10T20:38:48.999 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-06-10T20:38:49.000 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-06-10T20:38:49.000 DEBUG:teuthology.orchestra.run.smithi064:> test -f /run/.containerenv -o -f /.dockerenv 2023-06-10T20:38:49.001 DEBUG:teuthology.orchestra.run.smithi152:> test -f /run/.containerenv -o -f /.dockerenv 2023-06-10T20:38:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T20:38:49.019 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2023-06-10T20:38:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T20:38:49.021 DEBUG:teuthology.orchestra.run.smithi064:> uname -r 2023-06-10T20:38:49.081 INFO:teuthology.orchestra.run.smithi152.stdout:4.18.0-372.9.1.el8.x86_64 2023-06-10T20:38:49.082 INFO:teuthology.task.kernel:Running kernel on smithi152: 4.18.0-372.9.1.el8.x86_64 2023-06-10T20:38:49.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum install -y kernel 2023-06-10T20:38:49.084 INFO:teuthology.orchestra.run.smithi064.stdout:4.18.0-372.9.1.el8.x86_64 2023-06-10T20:38:49.084 INFO:teuthology.task.kernel:Running kernel on smithi064: 4.18.0-372.9.1.el8.x86_64 2023-06-10T20:38:49.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum install -y kernel 2023-06-10T20:38:50.293 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:38:50.294 INFO:teuthology.orchestra.run.smithi152.stdout:Unable to read consumer identity 2023-06-10T20:38:50.355 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:38:50.355 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-06-10T20:38:54.066 INFO:teuthology.orchestra.run.smithi064.stdout:Copr repo for python3-asyncssh owned by ceph 1.4 kB/s | 3.5 kB 00:02 2023-06-10T20:38:54.184 INFO:teuthology.orchestra.run.smithi152.stdout:Copr repo for python3-asyncssh owned by ceph 1.4 kB/s | 3.5 kB 00:02 2023-06-10T20:38:54.849 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 21 MB/s | 16 MB 00:00 2023-06-10T20:38:55.460 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2023-06-10T20:38:58.507 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 407 kB/s | 24 kB 00:00 2023-06-10T20:38:58.885 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 369 kB/s | 24 kB 00:00 2023-06-10T20:38:59.227 INFO:teuthology.orchestra.run.smithi064.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-06-10T20:38:59.279 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:38:59.280 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:38:59.280 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:38:59.412 DEBUG:teuthology.orchestra.run.smithi064:> echo no | sudo yum reinstall kernel || true 2023-06-10T20:38:59.571 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-06-10T20:38:59.618 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:38:59.619 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:38:59.619 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:38:59.746 DEBUG:teuthology.orchestra.run.smithi152:> echo no | sudo yum reinstall kernel || true 2023-06-10T20:38:59.805 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:38:59.805 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-06-10T20:39:00.077 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:39:00.078 INFO:teuthology.orchestra.run.smithi152.stdout:Unable to read consumer identity 2023-06-10T20:39:00.078 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:02 ago on Sat 10 Jun 2023 08:38:58 PM UTC. 2023-06-10T20:39:00.123 INFO:teuthology.orchestra.run.smithi064.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-06-10T20:39:00.189 INFO:teuthology.orchestra.run.smithi064.stderr:Error: No packages marked for reinstall. 2023-06-10T20:39:00.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum reinstall -y kernel || true 2023-06-10T20:39:00.334 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:02 ago on Sat 10 Jun 2023 08:38:58 PM UTC. 2023-06-10T20:39:00.374 INFO:teuthology.orchestra.run.smithi152.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-06-10T20:39:00.409 INFO:teuthology.orchestra.run.smithi152.stderr:Error: No packages marked for reinstall. 2023-06-10T20:39:00.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum reinstall -y kernel || true 2023-06-10T20:39:00.582 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:39:00.582 INFO:teuthology.orchestra.run.smithi064.stdout:Unable to read consumer identity 2023-06-10T20:39:00.784 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:39:00.784 INFO:teuthology.orchestra.run.smithi152.stdout:Unable to read consumer identity 2023-06-10T20:39:00.847 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:02 ago on Sat 10 Jun 2023 08:38:58 PM UTC. 2023-06-10T20:39:00.891 INFO:teuthology.orchestra.run.smithi064.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-06-10T20:39:00.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: No packages marked for reinstall. 2023-06-10T20:39:00.981 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q kernel | sort -rV | head -n 1 2023-06-10T20:39:01.040 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:03 ago on Sat 10 Jun 2023 08:38:58 PM UTC. 2023-06-10T20:39:01.063 INFO:teuthology.orchestra.run.smithi064.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-06-10T20:39:01.064 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-06-10T20:39:01.064 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-06-10T20:39:01.064 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-06-10T20:39:01.065 DEBUG:teuthology.orchestra.run.smithi064:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-06-10T20:39:01.079 INFO:teuthology.orchestra.run.smithi152.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-06-10T20:39:01.099 INFO:teuthology.orchestra.run.smithi152.stderr:Error: No packages marked for reinstall. 2023-06-10T20:39:01.099 INFO:teuthology.orchestra.run.smithi064.stdout:ttyS1 2023-06-10T20:39:01.110 DEBUG:teuthology.parallel:result is None 2023-06-10T20:39:01.146 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2023-06-10T20:39:01.227 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-06-10T20:39:01.227 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-06-10T20:39:01.227 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-06-10T20:39:01.227 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-06-10T20:39:01.228 DEBUG:teuthology.orchestra.run.smithi152:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-06-10T20:39:01.259 INFO:teuthology.orchestra.run.smithi152.stdout:ttyS1 2023-06-10T20:39:01.272 DEBUG:teuthology.parallel:result is None 2023-06-10T20:39:01.273 INFO:teuthology.run_tasks:Running task internal.base... 2023-06-10T20:39:01.282 INFO:teuthology.task.internal:Creating test directory... 2023-06-10T20:39:01.283 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-06-10T20:39:01.285 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-06-10T20:39:01.329 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-06-10T20:39:01.339 INFO:teuthology.run_tasks:Running task internal.archive... 2023-06-10T20:39:01.352 INFO:teuthology.task.internal:Creating archive directory... 2023-06-10T20:39:01.352 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-06-10T20:39:01.355 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-06-10T20:39:01.398 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-06-10T20:39:01.408 INFO:teuthology.task.internal:Enabling coredump saving... 2023-06-10T20:39:01.408 DEBUG:teuthology.orchestra.run.smithi064:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-06-10T20:39:01.421 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-06-10T20:39:01.447 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-10T20:39:01.458 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-10T20:39:01.476 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-10T20:39:01.486 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-10T20:39:01.487 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-06-10T20:39:01.495 INFO:teuthology.task.internal:Configuring sudo... 2023-06-10T20:39:01.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-06-10T20:39:01.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-06-10T20:39:01.527 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-06-10T20:39:01.536 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-06-10T20:39:01.537 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-06-10T20:39:01.568 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-06-10T20:39:01.585 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-10T20:39:01.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-10T20:39:01.720 DEBUG:teuthology.orchestra.run.smithi064:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-10T20:39:01.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-10T20:39:01.845 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:39:01.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-06-10T20:39:01.912 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-10T20:39:01.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-10T20:39:02.008 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-10T20:39:02.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-10T20:39:02.132 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:39:02.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-06-10T20:39:02.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo service rsyslog restart 2023-06-10T20:39:02.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo service rsyslog restart 2023-06-10T20:39:02.251 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-06-10T20:39:02.295 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-06-10T20:39:02.704 INFO:teuthology.run_tasks:Running task internal.timer... 2023-06-10T20:39:02.713 INFO:teuthology.task.internal:Starting timer... 2023-06-10T20:39:02.713 INFO:teuthology.run_tasks:Running task pcp... 2023-06-10T20:39:02.729 INFO:teuthology.run_tasks:Running task selinux... 2023-06-10T20:39:02.743 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-06-10T20:39:02.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo service auditd rotate 2023-06-10T20:39:02.806 INFO:teuthology.orchestra.run.smithi064.stdout:Rotating logs: 2023-06-10T20:39:02.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo service auditd rotate 2023-06-10T20:39:02.871 INFO:teuthology.orchestra.run.smithi152.stdout:Rotating logs: 2023-06-10T20:39:02.873 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-06-10T20:39:02.873 DEBUG:teuthology.orchestra.run.smithi064:> /usr/sbin/getenforce 2023-06-10T20:39:02.898 INFO:teuthology.orchestra.run.smithi064.stdout:Permissive 2023-06-10T20:39:02.898 DEBUG:teuthology.orchestra.run.smithi152:> /usr/sbin/getenforce 2023-06-10T20:39:02.920 INFO:teuthology.orchestra.run.smithi152.stdout:Permissive 2023-06-10T20:39:02.921 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi064.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi152.front.sepia.ceph.com': 'permissive'} 2023-06-10T20:39:02.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-06-10T20:39:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T20:39:02.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-06-10T20:39:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T20:39:02.994 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-06-10T20:39:02.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo /usr/sbin/setenforce permissive 2023-06-10T20:39:03.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo /usr/sbin/setenforce permissive 2023-06-10T20:39:03.083 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-06-10T20:39:03.104 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-06-10T20:39:03.111 INFO:teuthology.repo_utils:Fetching main from origin 2023-06-10T20:39:03.906 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-06-10T20:39:03.948 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-06-10T20:39:03.950 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi064.front.sepia.ceph.com,smithi152.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-06-10T20:47:24.861 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi064.front.sepia.ceph.com'), Remote(name='ubuntu@smithi152.front.sepia.ceph.com')] 2023-06-10T20:47:24.863 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi064.front.sepia.ceph.com' 2023-06-10T20:47:24.864 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi064.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-10T20:47:24.933 DEBUG:teuthology.orchestra.run.smithi064:> true 2023-06-10T20:47:25.002 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2023-06-10T20:47:25.003 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-10T20:47:25.070 DEBUG:teuthology.orchestra.run.smithi152:> true 2023-06-10T20:47:25.138 INFO:teuthology.run_tasks:Running task clock... 2023-06-10T20:47:25.149 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-06-10T20:47:25.149 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-06-10T20:47:25.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-10T20:47:25.154 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-06-10T20:47:25.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-10T20:47:25.184 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-06-10T20:47:25.199 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-06-10T20:47:25.212 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-06-10T20:47:25.227 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-06-10T20:47:25.276 INFO:teuthology.orchestra.run.smithi064.stderr:sudo: ntpd: command not found 2023-06-10T20:47:25.288 INFO:teuthology.orchestra.run.smithi064.stdout:506 Cannot talk to daemon 2023-06-10T20:47:25.293 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: ntpd: command not found 2023-06-10T20:47:25.303 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-06-10T20:47:25.305 INFO:teuthology.orchestra.run.smithi152.stdout:506 Cannot talk to daemon 2023-06-10T20:47:25.318 INFO:teuthology.orchestra.run.smithi064.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-06-10T20:47:25.319 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-06-10T20:47:25.333 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-06-10T20:47:25.393 INFO:teuthology.orchestra.run.smithi064.stderr:bash: ntpq: command not found 2023-06-10T20:47:25.396 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-06-10T20:47:25.396 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2023-06-10T20:47:25.396 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.396 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.397 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.397 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.402 INFO:teuthology.orchestra.run.smithi152.stderr:bash: ntpq: command not found 2023-06-10T20:47:25.406 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-06-10T20:47:25.406 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2023-06-10T20:47:25.406 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.407 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.407 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.407 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T20:47:25.407 INFO:teuthology.run_tasks:Running task pexec... 2023-06-10T20:47:25.418 INFO:teuthology.task.pexec:Executing custom commands... 2023-06-10T20:47:25.419 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi064.front.sepia.ceph.com 2023-06-10T20:47:25.419 DEBUG:teuthology.orchestra.run.smithi064:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-06-10T20:47:25.420 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi152.front.sepia.ceph.com 2023-06-10T20:47:25.420 DEBUG:teuthology.orchestra.run.smithi152:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-06-10T20:47:25.819 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:47:25.827 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:47:26.666 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:36 ago on Sat 10 Jun 2023 08:45:50 PM UTC. 2023-06-10T20:47:26.682 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:25 ago on Sat 10 Jun 2023 08:46:01 PM UTC. 2023-06-10T20:47:28.535 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:47:28.536 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:47:28.536 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2023-06-10T20:47:28.536 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:47:28.536 INFO:teuthology.orchestra.run.smithi152.stdout:Resetting modules: 2023-06-10T20:47:28.537 INFO:teuthology.orchestra.run.smithi152.stdout: container-tools 2023-06-10T20:47:28.537 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:47:28.537 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:47:28.537 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:47:28.537 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:47:28.685 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:47:29.051 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:47:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-06-10T20:47:29.051 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:47:29.051 INFO:teuthology.orchestra.run.smithi064.stdout:Resetting modules: 2023-06-10T20:47:29.051 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools 2023-06-10T20:47:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:47:29.052 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:47:29.052 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:47:29.052 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:47:29.501 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:39 ago on Sat 10 Jun 2023 08:45:50 PM UTC. 2023-06-10T20:47:29.694 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:28 ago on Sat 10 Jun 2023 08:46:01 PM UTC. 2023-06-10T20:47:32.229 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:47:32.539 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:47:32.680 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:47:33.014 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:47:33.479 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:43 ago on Sat 10 Jun 2023 08:45:50 PM UTC. 2023-06-10T20:47:33.874 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:32 ago on Sat 10 Jun 2023 08:46:01 PM UTC. 2023-06-10T20:47:35.157 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:47:35.250 INFO:teuthology.orchestra.run.smithi152.stdout:========================================================================================================================== 2023-06-10T20:47:35.250 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:47:35.250 INFO:teuthology.orchestra.run.smithi152.stdout:========================================================================================================================== 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout:Installing group/module packages: 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-06-10T20:47:35.251 INFO:teuthology.orchestra.run.smithi152.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.252 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:47:35.252 INFO:teuthology.orchestra.run.smithi152.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout:Downgrading: 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-06-10T20:47:35.253 INFO:teuthology.orchestra.run.smithi152.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-06-10T20:47:35.254 INFO:teuthology.orchestra.run.smithi152.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout:Installing module profiles: 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout: container-tools/common 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling module streams: 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout: container-tools 3.0 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:47:35.255 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:47:35.256 INFO:teuthology.orchestra.run.smithi152.stdout:========================================================================================================================== 2023-06-10T20:47:35.256 INFO:teuthology.orchestra.run.smithi152.stdout:Install 7 Packages 2023-06-10T20:47:35.256 INFO:teuthology.orchestra.run.smithi152.stdout:Downgrade 12 Packages 2023-06-10T20:47:35.257 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:47:35.257 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 51 M 2023-06-10T20:47:35.257 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:47:35.344 INFO:teuthology.orchestra.run.smithi152.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.0 MB/s | 91 kB 00:00 2023-06-10T20:47:35.369 INFO:teuthology.orchestra.run.smithi152.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 461 kB/s | 53 kB 00:00 2023-06-10T20:47:35.394 INFO:teuthology.orchestra.run.smithi152.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 366 kB/s | 51 kB 00:00 2023-06-10T20:47:35.436 INFO:teuthology.orchestra.run.smithi152.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 5.4 MB/s | 511 kB 00:00 2023-06-10T20:47:35.462 INFO:teuthology.orchestra.run.smithi152.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.0 MB/s | 69 kB 00:00 2023-06-10T20:47:35.487 INFO:teuthology.orchestra.run.smithi152.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.4 MB/s | 72 kB 00:00 2023-06-10T20:47:35.562 INFO:teuthology.orchestra.run.smithi152.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 17 MB/s | 2.9 MB 00:00 2023-06-10T20:47:35.587 INFO:teuthology.orchestra.run.smithi152.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 557 kB/s | 56 kB 00:00 2023-06-10T20:47:35.612 INFO:teuthology.orchestra.run.smithi152.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.0 MB/s | 51 kB 00:00 2023-06-10T20:47:35.679 INFO:teuthology.orchestra.run.smithi152.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 4.7 MB/s | 322 kB 00:00 2023-06-10T20:47:35.729 INFO:teuthology.orchestra.run.smithi152.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 818 kB/s | 41 kB 00:00 2023-06-10T20:47:35.833 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:47:35.880 INFO:teuthology.orchestra.run.smithi152.stdout:(12/19): podman-3.0.1-9.module+el8.6.0+14874+64 26 MB/s | 11 MB 00:00 2023-06-10T20:47:35.934 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout:Installing group/module packages: 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-06-10T20:47:35.935 INFO:teuthology.orchestra.run.smithi064.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout:Downgrading: 2023-06-10T20:47:35.936 INFO:teuthology.orchestra.run.smithi064.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-06-10T20:47:35.937 INFO:teuthology.orchestra.run.smithi064.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout:Installing module profiles: 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools/common 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout:Enabling module streams: 2023-06-10T20:47:35.938 INFO:teuthology.orchestra.run.smithi064.stdout: container-tools 3.0 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout:========================================================================================================================== 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout:Install 7 Packages 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout:Downgrade 12 Packages 2023-06-10T20:47:35.939 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:47:35.940 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 51 M 2023-06-10T20:47:35.940 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:47:35.955 INFO:teuthology.orchestra.run.smithi152.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-06-10T20:47:36.005 INFO:teuthology.orchestra.run.smithi064.stdout:(1/19): container-selinux-2.178.0-2.module+el8. 800 kB/s | 53 kB 00:00 2023-06-10T20:47:36.030 INFO:teuthology.orchestra.run.smithi064.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 563 kB/s | 51 kB 00:00 2023-06-10T20:47:36.055 INFO:teuthology.orchestra.run.smithi064.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 780 kB/s | 91 kB 00:00 2023-06-10T20:47:36.090 INFO:teuthology.orchestra.run.smithi064.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.2 MB/s | 69 kB 00:00 2023-06-10T20:47:36.123 INFO:teuthology.orchestra.run.smithi064.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 4.2 MB/s | 511 kB 00:00 2023-06-10T20:47:36.148 INFO:teuthology.orchestra.run.smithi064.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-06-10T20:47:36.161 INFO:teuthology.orchestra.run.smithi152.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 23 MB/s | 6.4 MB 00:00 2023-06-10T20:47:36.190 INFO:teuthology.orchestra.run.smithi064.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-06-10T20:47:36.224 INFO:teuthology.orchestra.run.smithi064.stdout:(8/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.5 MB/s | 51 kB 00:00 2023-06-10T20:47:36.274 INFO:teuthology.orchestra.run.smithi064.stdout:(9/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 13 MB/s | 2.9 MB 00:00 2023-06-10T20:47:36.316 INFO:teuthology.orchestra.run.smithi064.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 7.5 MB/s | 322 kB 00:00 2023-06-10T20:47:36.350 INFO:teuthology.orchestra.run.smithi064.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 1.2 MB/s | 41 kB 00:00 2023-06-10T20:47:36.381 INFO:teuthology.orchestra.run.smithi152.stdout:(15/19): containernetworking-plugins-0.9.1-1.mo 24 MB/s | 19 MB 00:00 2023-06-10T20:47:36.423 INFO:teuthology.orchestra.run.smithi152.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 4.1 MB/s | 1.1 MB 00:00 2023-06-10T20:47:36.442 INFO:teuthology.orchestra.run.smithi064.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 24 MB/s | 2.2 MB 00:00 2023-06-10T20:47:36.515 INFO:teuthology.orchestra.run.smithi152.stdout:(17/19): buildah-1.19.9-3.module+el8.6.0+14874+ 12 MB/s | 6.8 MB 00:00 2023-06-10T20:47:36.540 INFO:teuthology.orchestra.run.smithi152.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 1.1 MB/s | 185 kB 00:00 2023-06-10T20:47:36.565 INFO:teuthology.orchestra.run.smithi152.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 356 kB/s | 51 kB 00:00 2023-06-10T20:47:36.566 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:47:36.566 INFO:teuthology.orchestra.run.smithi152.stdout:Total 39 MB/s | 51 MB 00:01 2023-06-10T20:47:36.717 INFO:teuthology.orchestra.run.smithi064.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 18 MB/s | 11 MB 00:00 2023-06-10T20:47:36.859 INFO:teuthology.orchestra.run.smithi064.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 15 MB/s | 6.4 MB 00:00 2023-06-10T20:47:36.926 INFO:teuthology.orchestra.run.smithi064.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 16 MB/s | 1.1 MB 00:00 2023-06-10T20:47:36.968 INFO:teuthology.orchestra.run.smithi064.stdout:(16/19): crun-0.18-3.module+el8.6.0+14874+64436 4.4 MB/s | 185 kB 00:00 2023-06-10T20:47:37.041 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:47:37.051 INFO:teuthology.orchestra.run.smithi064.stdout:(17/19): buildah-1.19.9-3.module+el8.6.0+14874+ 20 MB/s | 6.8 MB 00:00 2023-06-10T20:47:37.077 INFO:teuthology.orchestra.run.smithi064.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 466 kB/s | 51 kB 00:00 2023-06-10T20:47:37.084 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:47:37.084 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:47:37.277 INFO:teuthology.orchestra.run.smithi064.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 18 MB/s | 19 MB 00:01 2023-06-10T20:47:37.277 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:47:37.278 INFO:teuthology.orchestra.run.smithi064.stdout:Total 38 MB/s | 51 MB 00:01 2023-06-10T20:47:37.508 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:47:37.512 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:47:37.766 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:47:37.812 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:47:37.812 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:47:38.256 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:47:38.260 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:47:38.692 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:47:38.749 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-06-10T20:47:38.917 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-06-10T20:47:38.926 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-06-10T20:47:38.947 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:39.175 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:39.454 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:47:39.502 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-06-10T20:47:39.678 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-06-10T20:47:39.687 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-06-10T20:47:39.706 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:39.936 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:56.091 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:57.839 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-06-10T20:47:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-06-10T20:47:58.045 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-06-10T20:47:58.256 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-06-10T20:47:58.428 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-06-10T20:47:58.663 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-06-10T20:47:58.664 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-06-10T20:47:58.664 INFO:teuthology.orchestra.run.smithi152.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-06-10T20:47:58.664 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:47:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-06-10T20:47:59.654 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-06-10T20:47:59.878 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-06-10T20:48:00.128 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-06-10T20:48:00.322 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-06-10T20:48:00.515 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-06-10T20:48:00.702 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-06-10T20:48:00.702 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-06-10T20:48:00.702 INFO:teuthology.orchestra.run.smithi064.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-06-10T20:48:00.702 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:01.148 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-06-10T20:48:01.468 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-06-10T20:48:01.499 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-06-10T20:48:01.710 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-06-10T20:48:01.847 INFO:teuthology.orchestra.run.smithi152.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-06-10T20:48:02.131 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-06-10T20:48:02.855 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-06-10T20:48:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-06-10T20:48:03.567 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-06-10T20:48:03.573 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-06-10T20:48:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-06-10T20:48:03.963 INFO:teuthology.orchestra.run.smithi064.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-06-10T20:48:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-06-10T20:48:04.179 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-06-10T20:48:04.224 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-06-10T20:48:04.327 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-06-10T20:48:04.328 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-06-10T20:48:04.622 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-06-10T20:48:04.787 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-06-10T20:48:04.788 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-06-10T20:48:04.947 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-06-10T20:48:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-06-10T20:48:05.088 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-06-10T20:48:05.264 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-06-10T20:48:05.405 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-06-10T20:48:05.562 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-06-10T20:48:05.705 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-06-10T20:48:05.736 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-06-10T20:48:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-06-10T20:48:05.992 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-06-10T20:48:05.998 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-06-10T20:48:05.999 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:06.181 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-06-10T20:48:06.221 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:06.399 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-06-10T20:48:06.516 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-06-10T20:48:06.516 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-06-10T20:48:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-06-10T20:48:06.811 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-06-10T20:48:06.973 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:06.974 INFO:teuthology.orchestra.run.smithi152.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-06-10T20:48:06.974 INFO:teuthology.orchestra.run.smithi152.stdout:All rules containing unresolvable specifiers will be skipped. 2023-06-10T20:48:06.974 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:48:07.019 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-06-10T20:48:07.019 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-06-10T20:48:07.143 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-06-10T20:48:07.143 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-06-10T20:48:07.144 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-06-10T20:48:07.145 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-06-10T20:48:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-06-10T20:48:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-06-10T20:48:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-06-10T20:48:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-06-10T20:48:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-06-10T20:48:07.148 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-06-10T20:48:07.149 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-06-10T20:48:07.149 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-06-10T20:48:07.149 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-06-10T20:48:07.149 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-06-10T20:48:07.150 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-06-10T20:48:07.201 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-06-10T20:48:07.342 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-06-10T20:48:07.509 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-06-10T20:48:07.667 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-06-10T20:48:07.827 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-06-10T20:48:08.000 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-06-10T20:48:08.133 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-06-10T20:48:08.287 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-06-10T20:48:08.288 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:08.458 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:08.589 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-06-10T20:48:08.589 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:18 ago on Sat 10 Jun 2023 08:45:50 PM UTC. 2023-06-10T20:48:09.006 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-06-10T20:48:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-06-10T20:48:09.253 INFO:teuthology.orchestra.run.smithi064.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-06-10T20:48:09.253 INFO:teuthology.orchestra.run.smithi064.stdout:All rules containing unresolvable specifiers will be skipped. 2023-06-10T20:48:09.253 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-06-10T20:48:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-06-10T20:48:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-06-10T20:48:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-06-10T20:48:09.432 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-06-10T20:48:09.433 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-06-10T20:48:09.434 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-06-10T20:48:09.435 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-06-10T20:48:09.435 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-06-10T20:48:09.435 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-06-10T20:48:09.435 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-06-10T20:48:09.435 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-06-10T20:48:09.436 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-06-10T20:48:10.876 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-06-10T20:48:10.876 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:09 ago on Sat 10 Jun 2023 08:46:01 PM UTC. 2023-06-10T20:48:10.899 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:48:11.092 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:48:11.092 INFO:teuthology.orchestra.run.smithi152.stdout:Downgraded: 2023-06-10T20:48:11.092 INFO:teuthology.orchestra.run.smithi152.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.093 INFO:teuthology.orchestra.run.smithi152.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.094 INFO:teuthology.orchestra.run.smithi152.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: yajl-2.1.0-10.el8.x86_64 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:48:11.095 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:48:11.309 DEBUG:teuthology.parallel:result is None 2023-06-10T20:48:13.414 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:48:13.628 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout:Downgraded: 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.629 INFO:teuthology.orchestra.run.smithi064.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.630 INFO:teuthology.orchestra.run.smithi064.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-06-10T20:48:13.631 INFO:teuthology.orchestra.run.smithi064.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-06-10T20:48:13.632 INFO:teuthology.orchestra.run.smithi064.stdout: yajl-2.1.0-10.el8.x86_64 2023-06-10T20:48:13.632 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:13.632 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:48:13.825 DEBUG:teuthology.parallel:result is None 2023-06-10T20:48:13.825 INFO:teuthology.run_tasks:Running task install... 2023-06-10T20:48:13.835 DEBUG:teuthology.task.install:project ceph 2023-06-10T20:48:13.836 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'}} 2023-06-10T20:48:13.836 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'} 2023-06-10T20:48:13.836 INFO:teuthology.task.install:Using flavor: default 2023-06-10T20:48:13.842 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-06-10T20:48:13.843 INFO:teuthology.task.install:extra packages: [] 2023-06-10T20:48:13.843 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009', 'tag': None, 'wait_for_package': False} 2023-06-10T20:48:13.844 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:48:13.844 INFO:teuthology.packaging:ref: None 2023-06-10T20:48:13.844 INFO:teuthology.packaging:tag: None 2023-06-10T20:48:13.845 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:48:13.845 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:48:13.845 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-06-10T20:48:13.847 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'pacific', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009', 'tag': None, 'wait_for_package': False} 2023-06-10T20:48:13.847 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:48:13.847 INFO:teuthology.packaging:ref: None 2023-06-10T20:48:13.848 INFO:teuthology.packaging:tag: None 2023-06-10T20:48:13.848 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:48:13.848 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:48:13.848 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-06-10T20:48:14.192 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/ 2023-06-10T20:48:14.195 INFO:teuthology.task.install.rpm:Package version is 16.2.13-257.gd8c5d349 2023-06-10T20:48:14.209 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/ 2023-06-10T20:48:14.210 INFO:teuthology.task.install.rpm:Package version is 16.2.13-257.gd8c5d349 2023-06-10T20:48:14.319 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-06-10T20:48:14.320 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:48:14.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-06-10T20:48:14.352 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-06-10T20:48:14.353 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:48:14.353 INFO:teuthology.packaging:ref: None 2023-06-10T20:48:14.353 INFO:teuthology.packaging:tag: None 2023-06-10T20:48:14.353 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:48:14.353 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:48:14.354 DEBUG:teuthology.orchestra.run.smithi152:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-06-10T20:48:14.385 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-06-10T20:48:14.385 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:48:14.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-06-10T20:48:14.416 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-06-10T20:48:14.417 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:48:14.417 INFO:teuthology.packaging:ref: None 2023-06-10T20:48:14.417 INFO:teuthology.packaging:tag: None 2023-06-10T20:48:14.418 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:48:14.418 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:48:14.418 DEBUG:teuthology.orchestra.run.smithi064:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/pacific/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-06-10T20:48:14.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-06-10T20:48:14.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-06-10T20:48:14.507 DEBUG:teuthology.orchestra.run.smithi152:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-06-10T20:48:14.566 DEBUG:teuthology.orchestra.run.smithi064:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-06-10T20:48:14.576 INFO:teuthology.orchestra.run.smithi152.stdout:check_obsoletes = 1 2023-06-10T20:48:14.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum clean all 2023-06-10T20:48:14.633 INFO:teuthology.orchestra.run.smithi064.stdout:check_obsoletes = 1 2023-06-10T20:48:14.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum clean all 2023-06-10T20:48:14.951 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:48:15.025 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:48:15.225 INFO:teuthology.orchestra.run.smithi152.stdout:44 files removed 2023-06-10T20:48:15.268 INFO:teuthology.orchestra.run.smithi064.stdout:44 files removed 2023-06-10T20:48:15.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-radosgw 2023-06-10T20:48:15.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-radosgw 2023-06-10T20:48:15.605 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:48:15.665 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:48:16.294 INFO:teuthology.orchestra.run.smithi064.stdout:ceph packages for x86_64 286 kB/s | 76 kB 00:00 2023-06-10T20:48:16.344 INFO:teuthology.orchestra.run.smithi152.stdout:ceph packages for x86_64 212 kB/s | 76 kB 00:00 2023-06-10T20:48:16.653 INFO:teuthology.orchestra.run.smithi152.stdout:ceph noarch packages 55 kB/s | 15 kB 00:00 2023-06-10T20:48:16.687 INFO:teuthology.orchestra.run.smithi064.stdout:ceph noarch packages 42 kB/s | 15 kB 00:00 2023-06-10T20:48:17.021 INFO:teuthology.orchestra.run.smithi152.stdout:ceph source packages 4.1 kB/s | 1.4 kB 00:00 2023-06-10T20:48:17.021 INFO:teuthology.orchestra.run.smithi064.stdout:ceph source packages 4.6 kB/s | 1.4 kB 00:00 2023-06-10T20:48:19.938 INFO:teuthology.orchestra.run.smithi064.stdout:Copr repo for python3-asyncssh owned by ceph 1.2 kB/s | 3.5 kB 00:02 2023-06-10T20:48:20.505 INFO:teuthology.orchestra.run.smithi152.stdout:Copr repo for python3-asyncssh owned by ceph 1.0 kB/s | 3.5 kB 00:03 2023-06-10T20:48:20.922 INFO:teuthology.orchestra.run.smithi064.stdout:Extra Packages for Enterprise Linux 16 MB/s | 16 MB 00:00 2023-06-10T20:48:21.799 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2023-06-10T20:48:24.564 INFO:teuthology.orchestra.run.smithi064.stdout:lab-extras 336 kB/s | 24 kB 00:00 2023-06-10T20:48:25.264 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 419 kB/s | 24 kB 00:00 2023-06-10T20:48:25.491 INFO:teuthology.orchestra.run.smithi064.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 62 MB/s | 53 MB 00:00 2023-06-10T20:48:26.141 INFO:teuthology.orchestra.run.smithi152.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 59 MB/s | 47 MB 00:00 2023-06-10T20:48:34.043 INFO:teuthology.orchestra.run.smithi152.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 63 MB/s | 53 MB 00:00 2023-06-10T20:48:34.320 INFO:teuthology.orchestra.run.smithi064.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-06-10T20:48:45.356 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:48:45.359 INFO:teuthology.orchestra.run.smithi152.stdout:==================================================================================================== 2023-06-10T20:48:45.359 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:48:45.359 INFO:teuthology.orchestra.run.smithi152.stdout:==================================================================================================== 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 12 M 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout: librados2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.7 M 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 4.2 M 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:48:45.360 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 6.1 M 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 24 M 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 26 k 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 823 k 2023-06-10T20:48:45.361 INFO:teuthology.orchestra.run.smithi152.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 493 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.8 M 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-06-10T20:48:45.362 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 45 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 99 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 215 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 389 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 368 k 2023-06-10T20:48:45.363 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 114 k 2023-06-10T20:48:45.364 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:48:45.364 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:48:45.365 INFO:teuthology.orchestra.run.smithi152.stdout:==================================================================================================== 2023-06-10T20:48:45.365 INFO:teuthology.orchestra.run.smithi152.stdout:Install 21 Packages 2023-06-10T20:48:45.365 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2023-06-10T20:48:45.365 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:48:45.365 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 57 M 2023-06-10T20:48:45.366 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:48:46.077 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:48:46.079 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-06-10T20:48:46.079 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 12 M 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrading: 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout: librados2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.7 M 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 4.2 M 2023-06-10T20:48:46.080 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 6.1 M 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 24 M 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 26 k 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 823 k 2023-06-10T20:48:46.081 INFO:teuthology.orchestra.run.smithi064.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 493 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2 x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.8 M 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 45 k 2023-06-10T20:48:46.082 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 99 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 215 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 389 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 368 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 114 k 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:46.083 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:48:46.084 INFO:teuthology.orchestra.run.smithi064.stdout:==================================================================================================== 2023-06-10T20:48:46.084 INFO:teuthology.orchestra.run.smithi064.stdout:Install 21 Packages 2023-06-10T20:48:46.084 INFO:teuthology.orchestra.run.smithi064.stdout:Upgrade 2 Packages 2023-06-10T20:48:46.084 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:48:46.085 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 57 M 2023-06-10T20:48:46.085 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:48:46.985 INFO:teuthology.orchestra.run.smithi152.stdout:(1/23): ceph-base-16.2.13-257.gd8c5d349.el8.x86 4.1 MB/s | 6.1 MB 00:01 2023-06-10T20:48:47.077 INFO:teuthology.orchestra.run.smithi152.stdout:(2/23): ceph-selinux-16.2.13-257.gd8c5d349.el8. 280 kB/s | 26 kB 00:00 2023-06-10T20:48:47.285 INFO:teuthology.orchestra.run.smithi152.stdout:(3/23): libcephfs2-16.2.13-257.gd8c5d349.el8.x8 3.9 MB/s | 823 kB 00:00 2023-06-10T20:48:47.536 INFO:teuthology.orchestra.run.smithi152.stdout:(4/23): ceph-common-16.2.13-257.gd8c5d349.el8.x 12 MB/s | 24 MB 00:02 2023-06-10T20:48:47.586 INFO:teuthology.orchestra.run.smithi152.stdout:(5/23): libradosstriper1-16.2.13-257.gd8c5d349. 1.6 MB/s | 493 kB 00:00 2023-06-10T20:48:47.678 INFO:teuthology.orchestra.run.smithi152.stdout:(6/23): python3-ceph-argparse-16.2.13-257.gd8c5 491 kB/s | 45 kB 00:00 2023-06-10T20:48:47.778 INFO:teuthology.orchestra.run.smithi152.stdout:(7/23): python3-ceph-common-16.2.13-257.gd8c5d3 986 kB/s | 99 kB 00:00 2023-06-10T20:48:47.887 INFO:teuthology.orchestra.run.smithi152.stdout:(8/23): python3-cephfs-16.2.13-257.gd8c5d349.el 1.9 MB/s | 215 kB 00:00 2023-06-10T20:48:48.043 INFO:teuthology.orchestra.run.smithi064.stdout:(1/23): ceph-base-16.2.13-257.gd8c5d349.el8.x86 3.3 MB/s | 6.1 MB 00:01 2023-06-10T20:48:48.079 INFO:teuthology.orchestra.run.smithi152.stdout:(9/23): ceph-radosgw-16.2.13-257.gd8c5d349.el8. 4.5 MB/s | 12 MB 00:02 2023-06-10T20:48:48.093 INFO:teuthology.orchestra.run.smithi064.stdout:(2/23): ceph-selinux-16.2.13-257.gd8c5d349.el8. 515 kB/s | 26 kB 00:00 2023-06-10T20:48:48.146 INFO:teuthology.orchestra.run.smithi152.stdout:(10/23): librgw2-16.2.13-257.gd8c5d349.el8.x86_ 6.2 MB/s | 3.8 MB 00:00 2023-06-10T20:48:48.172 INFO:teuthology.orchestra.run.smithi152.stdout:(11/23): python3-rados-16.2.13-257.gd8c5d349.el 1.3 MB/s | 389 kB 00:00 2023-06-10T20:48:48.197 INFO:teuthology.orchestra.run.smithi152.stdout:(12/23): python3-rbd-16.2.13-257.gd8c5d349.el8. 3.1 MB/s | 368 kB 00:00 2023-06-10T20:48:48.222 INFO:teuthology.orchestra.run.smithi152.stdout:(13/23): python3-rgw-16.2.13-257.gd8c5d349.el8. 1.5 MB/s | 114 kB 00:00 2023-06-10T20:48:48.226 INFO:teuthology.orchestra.run.smithi064.stdout:(3/23): libcephfs2-16.2.13-257.gd8c5d349.el8.x8 6.0 MB/s | 823 kB 00:00 2023-06-10T20:48:48.326 INFO:teuthology.orchestra.run.smithi064.stdout:(4/23): libradosstriper1-16.2.13-257.gd8c5d349. 4.8 MB/s | 493 kB 00:00 2023-06-10T20:48:48.398 INFO:teuthology.orchestra.run.smithi152.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 336 kB/s | 59 kB 00:00 2023-06-10T20:48:48.423 INFO:teuthology.orchestra.run.smithi152.stdout:(15/23): leveldb-1.22-1.el8.x86_64.rpm 801 kB/s | 181 kB 00:00 2023-06-10T20:48:48.465 INFO:teuthology.orchestra.run.smithi152.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.0 MB/s | 306 kB 00:00 2023-06-10T20:48:48.499 INFO:teuthology.orchestra.run.smithi152.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.4 MB/s | 259 kB 00:00 2023-06-10T20:48:48.549 INFO:teuthology.orchestra.run.smithi152.stdout:(18/23): libunwind-1.3.1-3.el8.x86_64.rpm 497 kB/s | 75 kB 00:00 2023-06-10T20:48:48.574 INFO:teuthology.orchestra.run.smithi152.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 404 kB/s | 44 kB 00:00 2023-06-10T20:48:48.608 INFO:teuthology.orchestra.run.smithi152.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 3.3 MB/s | 354 kB 00:00 2023-06-10T20:48:48.633 INFO:teuthology.orchestra.run.smithi152.stdout:(21/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 558 kB/s | 47 kB 00:00 2023-06-10T20:48:48.835 INFO:teuthology.orchestra.run.smithi064.stdout:(5/23): librgw2-16.2.13-257.gd8c5d349.el8.x86_6 7.4 MB/s | 3.8 MB 00:00 2023-06-10T20:48:48.927 INFO:teuthology.orchestra.run.smithi064.stdout:(6/23): python3-ceph-argparse-16.2.13-257.gd8c5 491 kB/s | 45 kB 00:00 2023-06-10T20:48:49.027 INFO:teuthology.orchestra.run.smithi064.stdout:(7/23): python3-ceph-common-16.2.13-257.gd8c5d3 986 kB/s | 99 kB 00:00 2023-06-10T20:48:49.119 INFO:teuthology.orchestra.run.smithi064.stdout:(8/23): python3-cephfs-16.2.13-257.gd8c5d349.el 2.3 MB/s | 215 kB 00:00 2023-06-10T20:48:49.219 INFO:teuthology.orchestra.run.smithi064.stdout:(9/23): python3-rados-16.2.13-257.gd8c5d349.el8 3.8 MB/s | 389 kB 00:00 2023-06-10T20:48:49.267 INFO:teuthology.orchestra.run.smithi152.stdout:(22/23): librbd1-16.2.13-257.gd8c5d349.el8.x86_ 6.3 MB/s | 4.2 MB 00:00 2023-06-10T20:48:49.303 INFO:teuthology.orchestra.run.smithi064.stdout:(10/23): python3-rbd-16.2.13-257.gd8c5d349.el8. 4.3 MB/s | 368 kB 00:00 2023-06-10T20:48:49.581 INFO:teuthology.orchestra.run.smithi064.stdout:(11/23): ceph-common-16.2.13-257.gd8c5d349.el8. 7.2 MB/s | 24 MB 00:03 2023-06-10T20:48:49.612 INFO:teuthology.orchestra.run.smithi064.stdout:(12/23): python3-rgw-16.2.13-257.gd8c5d349.el8. 371 kB/s | 114 kB 00:00 2023-06-10T20:48:49.675 INFO:teuthology.orchestra.run.smithi152.stdout:(23/23): librados2-16.2.13-257.gd8c5d349.el8.x8 3.4 MB/s | 3.7 MB 00:01 2023-06-10T20:48:49.677 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:48:49.678 INFO:teuthology.orchestra.run.smithi152.stdout:Total 13 MB/s | 57 MB 00:04 2023-06-10T20:48:49.726 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:48:49.772 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:48:49.773 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:48:49.829 INFO:teuthology.orchestra.run.smithi064.stdout:(13/23): ceph-radosgw-16.2.13-257.gd8c5d349.el8 3.2 MB/s | 12 MB 00:03 2023-06-10T20:48:49.854 INFO:teuthology.orchestra.run.smithi064.stdout:(14/23): leveldb-1.22-1.el8.x86_64.rpm 748 kB/s | 181 kB 00:00 2023-06-10T20:48:49.888 INFO:teuthology.orchestra.run.smithi064.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 999 kB/s | 306 kB 00:00 2023-06-10T20:48:49.913 INFO:teuthology.orchestra.run.smithi064.stdout:(16/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.4 MB/s | 75 kB 00:00 2023-06-10T20:48:49.947 INFO:teuthology.orchestra.run.smithi064.stdout:(17/23): liboath-2.6.2-3.el8.x86_64.rpm 502 kB/s | 59 kB 00:00 2023-06-10T20:48:50.005 INFO:teuthology.orchestra.run.smithi064.stdout:(18/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 402 kB/s | 47 kB 00:00 2023-06-10T20:48:50.031 INFO:teuthology.orchestra.run.smithi064.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 657 kB/s | 44 kB 00:00 2023-06-10T20:48:50.056 INFO:teuthology.orchestra.run.smithi064.stdout:(20/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.8 MB/s | 259 kB 00:00 2023-06-10T20:48:50.081 INFO:teuthology.orchestra.run.smithi064.stdout:(21/23): librdkafka-0.11.4-3.el8.x86_64.rpm 4.6 MB/s | 354 kB 00:00 2023-06-10T20:48:50.098 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:48:50.102 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:48:50.406 INFO:teuthology.orchestra.run.smithi064.stdout:(22/23): librbd1-16.2.13-257.gd8c5d349.el8.x86_ 12 MB/s | 4.2 MB 00:00 2023-06-10T20:48:50.474 INFO:teuthology.orchestra.run.smithi064.stdout:(23/23): librados2-16.2.13-257.gd8c5d349.el8.x8 8.5 MB/s | 3.7 MB 00:00 2023-06-10T20:48:50.478 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:48:50.478 INFO:teuthology.orchestra.run.smithi064.stdout:Total 13 MB/s | 57 MB 00:04 2023-06-10T20:48:50.525 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:48:50.577 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:48:50.577 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:48:50.929 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:48:50.933 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:48:51.139 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:48:51.321 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-06-10T20:48:51.741 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-06-10T20:48:51.896 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:48:51.990 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:48:51.992 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:48:52.160 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 3/25 2023-06-10T20:48:52.218 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-06-10T20:48:52.387 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-06-10T20:48:52.538 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:48:52.630 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-06-10T20:48:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:48:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:48:52.940 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:48:53.104 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 6/25 2023-06-10T20:48:53.107 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 3/25 2023-06-10T20:48:53.207 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 6/25 2023-06-10T20:48:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-06-10T20:48:53.432 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-06-10T20:48:53.509 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:48:53.588 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-06-10T20:48:53.947 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:48:53.967 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-06-10T20:48:54.109 INFO:teuthology.orchestra.run.smithi064.stdout: Upgrading : librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 6/25 2023-06-10T20:48:54.129 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 9/25 2023-06-10T20:48:54.206 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 9/25 2023-06-10T20:48:54.234 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 6/25 2023-06-10T20:48:54.384 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-06-10T20:48:54.427 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-06-10T20:48:54.455 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-06-10T20:48:54.568 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8. 11/25 2023-06-10T20:48:54.741 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 12/25 2023-06-10T20:48:54.947 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 13/25 2023-06-10T20:48:54.958 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-06-10T20:48:55.117 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 9/25 2023-06-10T20:48:55.129 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 14/25 2023-06-10T20:48:55.228 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 9/25 2023-06-10T20:48:55.287 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 15/25 2023-06-10T20:48:55.358 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 15/25 2023-06-10T20:48:55.382 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/25 2023-06-10T20:48:55.497 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-06-10T20:48:55.524 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8. 11/25 2023-06-10T20:48:55.701 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-06-10T20:48:55.729 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 12/25 2023-06-10T20:48:55.932 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-06-10T20:48:55.946 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 13/25 2023-06-10T20:48:56.071 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x8 19/25 2023-06-10T20:48:56.145 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 14/25 2023-06-10T20:48:56.284 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 15/25 2023-06-10T20:48:56.356 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 15/25 2023-06-10T20:48:56.503 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-06-10T20:48:56.755 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-06-10T20:48:56.985 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-06-10T20:48:57.151 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x8 19/25 2023-06-10T20:48:59.616 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:48:59.837 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:49:00.535 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:49:00.794 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 21/25 2023-06-10T20:49:00.858 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 21/25 2023-06-10T20:49:00.948 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:49:00.978 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:01.210 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:49:01.940 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 20/25 2023-06-10T20:49:02.252 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 21/25 2023-06-10T20:49:02.338 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 21/25 2023-06-10T20:49:02.475 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:19.326 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /sys 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /proc 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /dev 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /run 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /mnt 2023-06-10T20:49:19.327 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /var/tmp 2023-06-10T20:49:19.328 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /home 2023-06-10T20:49:19.328 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /tmp 2023-06-10T20:49:19.328 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /dev 2023-06-10T20:49:19.328 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:20.508 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 23/25 2023-06-10T20:49:20.571 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 23/25 2023-06-10T20:49:20.571 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-06-10T20:49:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-06-10T20:49:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:21.219 INFO:teuthology.orchestra.run.smithi152.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-06-10T20:49:21.219 INFO:teuthology.orchestra.run.smithi152.stdout:All rules containing unresolvable specifiers will be skipped. 2023-06-10T20:49:21.219 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:21.378 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 1/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 3/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 4/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 6/25 2023-06-10T20:49:21.379 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 7/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8. 8/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x8 9/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 10/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 11/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 12/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 13/25 2023-06-10T20:49:21.380 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-06-10T20:49:21.381 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-06-10T20:49:21.381 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:21.382 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-06-10T20:49:21.383 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 24/25 2023-06-10T20:49:21.453 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:21.453 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /sys 2023-06-10T20:49:21.453 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /proc 2023-06-10T20:49:21.453 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /dev 2023-06-10T20:49:21.453 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /run 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /mnt 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /var/tmp 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /home 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /tmp 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout:skipping the directory /dev 2023-06-10T20:49:21.454 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 23/25 2023-06-10T20:49:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 23/25 2023-06-10T20:49:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-06-10T20:49:22.770 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:22.770 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:49 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:22.966 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-06-10T20:49:22.966 INFO:teuthology.orchestra.run.smithi064.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:23.429 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:23.430 INFO:teuthology.orchestra.run.smithi064.stdout:[/usr/lib/tmpfiles.d/journal-nocow.conf:26] Failed to resolve specifier: uninitialized /etc detected, skipping 2023-06-10T20:49:23.430 INFO:teuthology.orchestra.run.smithi064.stdout:All rules containing unresolvable specifiers will be skipped. 2023-06-10T20:49:23.430 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:23.620 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 1/25 2023-06-10T20:49:23.620 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2/25 2023-06-10T20:49:23.620 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 3/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 4/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 5/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_6 6/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 7/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8. 8/25 2023-06-10T20:49:23.621 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x8 9/25 2023-06-10T20:49:23.622 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 10/25 2023-06-10T20:49:23.622 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 11/25 2023-06-10T20:49:23.622 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 12/25 2023-06-10T20:49:23.622 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 13/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : leveldb-1.22-1.el8.x86_64 15/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 16/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 17/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-06-10T20:49:23.623 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-06-10T20:49:23.624 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-06-10T20:49:23.624 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-06-10T20:49:23.624 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 22/25 2023-06-10T20:49:23.624 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-06-10T20:49:23.625 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 24/25 2023-06-10T20:49:25.012 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-06-10T20:49:25.013 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:52 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:25.563 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:49:25.753 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:25.754 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2023-06-10T20:49:25.754 INFO:teuthology.orchestra.run.smithi152.stdout: librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.755 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.755 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:49:25.755 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.755 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.756 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.756 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.756 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-06-10T20:49:25.757 INFO:teuthology.orchestra.run.smithi152.stdout: leveldb-1.22-1.el8.x86_64 2023-06-10T20:49:25.757 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.758 INFO:teuthology.orchestra.run.smithi152.stdout: liboath-2.6.2-3.el8.x86_64 2023-06-10T20:49:25.758 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-06-10T20:49:25.758 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.758 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-06-10T20:49:25.758 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind-1.3.1-3.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.759 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-06-10T20:49:25.760 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.760 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.760 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:25.760 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:25.760 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:49:25.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-test 2023-06-10T20:49:26.272 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:49:27.133 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:54 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:49:27.922 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:Upgraded: 2023-06-10T20:49:27.923 INFO:teuthology.orchestra.run.smithi064.stdout: librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.923 INFO:teuthology.orchestra.run.smithi064.stdout: librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.923 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:49:27.923 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-radosgw-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-selinux-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: leveldb-1.22-1.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: liboath-2.6.2-3.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: libradosstriper1-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: librgw2-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: libunwind-1.3.1-3.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-06-10T20:49:27.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-argparse-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-ceph-common-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:27.926 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:49:28.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-test 2023-06-10T20:49:28.456 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:49:28.518 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:49:28.519 INFO:teuthology.orchestra.run.smithi152.stdout:============================================================================================ 2023-06-10T20:49:28.519 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:49:28.519 INFO:teuthology.orchestra.run.smithi152.stdout:============================================================================================ 2023-06-10T20:49:28.519 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:49:28.519 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 51 M 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 174 k 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:28.520 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:49:28.521 INFO:teuthology.orchestra.run.smithi152.stdout:============================================================================================ 2023-06-10T20:49:28.521 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2023-06-10T20:49:28.521 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:28.521 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 52 M 2023-06-10T20:49:28.522 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 193 M 2023-06-10T20:49:28.522 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:49:28.557 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.9 MB/s | 70 kB 00:00 2023-06-10T20:49:28.666 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.2 MB/s | 323 kB 00:00 2023-06-10T20:49:28.716 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-06-10T20:49:28.774 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 3.4 MB/s | 202 kB 00:00 2023-06-10T20:49:28.849 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): libcephsqlite-16.2.13-257.gd8c5d349.el8. 529 kB/s | 174 kB 00:00 2023-06-10T20:49:29.332 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:00:56 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:30.861 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:49:30.862 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-06-10T20:49:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:49:30.862 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-06-10T20:49:30.862 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 51 M 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 174 k 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-06-10T20:49:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:30.864 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:49:30.864 INFO:teuthology.orchestra.run.smithi064.stdout:============================================================================================ 2023-06-10T20:49:30.864 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2023-06-10T20:49:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:30.865 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 52 M 2023-06-10T20:49:30.865 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 193 M 2023-06-10T20:49:30.865 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:49:30.894 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.3 MB/s | 70 kB 00:00 2023-06-10T20:49:30.969 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.2 MB/s | 323 kB 00:00 2023-06-10T20:49:31.008 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): ceph-test-16.2.13-257.gd8c5d349.el8.x86_ 21 MB/s | 51 MB 00:02 2023-06-10T20:49:31.010 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:49:31.010 INFO:teuthology.orchestra.run.smithi152.stdout:Total 21 MB/s | 52 MB 00:02 2023-06-10T20:49:31.011 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-06-10T20:49:31.050 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:49:31.053 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-06-10T20:49:31.071 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:49:31.071 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:49:31.186 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): libcephsqlite-16.2.13-257.gd8c5d349.el8. 540 kB/s | 174 kB 00:00 2023-06-10T20:49:31.344 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:49:31.347 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:49:31.799 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:49:31.957 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-06-10T20:49:32.036 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-06-10T20:49:32.251 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-06-10T20:49:32.399 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-06-10T20:49:32.538 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-06-10T20:49:32.780 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 5/6 2023-06-10T20:49:33.561 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): ceph-test-16.2.13-257.gd8c5d349.el8.x86_ 19 MB/s | 51 MB 00:02 2023-06-10T20:49:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:49:33.564 INFO:teuthology.orchestra.run.smithi064.stdout:Total 19 MB/s | 52 MB 00:02 2023-06-10T20:49:33.603 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:49:33.625 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:49:33.626 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:49:33.914 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:49:33.918 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:49:34.387 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:49:34.512 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-06-10T20:49:34.593 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-06-10T20:49:34.809 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-06-10T20:49:35.005 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-06-10T20:49:35.178 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-06-10T20:49:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 5/6 2023-06-10T20:49:37.226 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 5/6 2023-06-10T20:49:37.454 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 6/6 2023-06-10T20:49:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 6/6 2023-06-10T20:49:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 1/6 2023-06-10T20:49:37.684 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 2/6 2023-06-10T20:49:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-06-10T20:49:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-06-10T20:49:37.685 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-06-10T20:49:38.781 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-06-10T20:49:38.781 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:05 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:40.193 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 5/6 2023-06-10T20:49:40.601 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 6/6 2023-06-10T20:49:40.837 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 6/6 2023-06-10T20:49:40.838 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 1/6 2023-06-10T20:49:40.838 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 2/6 2023-06-10T20:49:40.838 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-06-10T20:49:40.839 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-06-10T20:49:40.839 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-06-10T20:49:41.135 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:49:41.321 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:41.321 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:49:41.321 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: jq-1.6-3.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: socat-1.7.4.1-1.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:41.322 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:49:41.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph 2023-06-10T20:49:41.804 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:49:41.958 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-06-10T20:49:41.958 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:08 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:42.596 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:09 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:43.982 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================= 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================= 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout: ceph x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 6.4 k 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:49:43.986 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 2.4 M 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 1.6 M 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 244 k 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 5.0 M 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 18 M 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-06-10T20:49:43.987 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-06-10T20:49:43.988 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-06-10T20:49:43.989 INFO:teuthology.orchestra.run.smithi152.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-06-10T20:49:43.990 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-06-10T20:49:43.991 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:43.991 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:49:43.991 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================= 2023-06-10T20:49:43.991 INFO:teuthology.orchestra.run.smithi152.stdout:Install 32 Packages 2023-06-10T20:49:43.991 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:49:43.992 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 31 M 2023-06-10T20:49:43.992 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 101 M 2023-06-10T20:49:43.992 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:49:44.169 INFO:teuthology.orchestra.run.smithi152.stdout:(1/32): ceph-16.2.13-257.gd8c5d349.el8.x86_64.r 59 kB/s | 6.4 kB 00:00 2023-06-10T20:49:44.406 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:49:44.428 INFO:teuthology.orchestra.run.smithi152.stdout:(2/32): ceph-mds-16.2.13-257.gd8c5d349.el8.x86_ 6.7 MB/s | 2.4 MB 00:00 2023-06-10T20:49:44.607 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:44.607 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:49:44.607 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-test-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:44.607 INFO:teuthology.orchestra.run.smithi064.stdout: jq-1.6-3.el8.x86_64 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout: libcephsqlite-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout: socat-1.7.4.1-1.el8.x86_64 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:44.608 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:49:44.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph 2023-06-10T20:49:44.803 INFO:teuthology.orchestra.run.smithi152.stdout:(3/32): ceph-mon-16.2.13-257.gd8c5d349.el8.x86_ 7.9 MB/s | 5.0 MB 00:00 2023-06-10T20:49:44.887 INFO:teuthology.orchestra.run.smithi152.stdout:(4/32): ceph-mgr-modules-core-16.2.13-257.gd8c5 2.9 MB/s | 244 kB 00:00 2023-06-10T20:49:45.071 INFO:teuthology.orchestra.run.smithi152.stdout:(5/32): ceph-mgr-16.2.13-257.gd8c5d349.el8.x86_ 1.6 MB/s | 1.6 MB 00:01 2023-06-10T20:49:45.095 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:49:45.104 INFO:teuthology.orchestra.run.smithi152.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 201 kB/s | 44 kB 00:00 2023-06-10T20:49:45.421 INFO:teuthology.orchestra.run.smithi152.stdout:(7/32): ceph-osd-16.2.13-257.gd8c5d349.el8.x86_ 18 MB/s | 18 MB 00:00 2023-06-10T20:49:45.447 INFO:teuthology.orchestra.run.smithi152.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 504 kB/s | 173 kB 00:00 2023-06-10T20:49:45.472 INFO:teuthology.orchestra.run.smithi152.stdout:(9/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 462 kB/s | 185 kB 00:00 2023-06-10T20:49:45.522 INFO:teuthology.orchestra.run.smithi152.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 3.7 MB/s | 384 kB 00:00 2023-06-10T20:49:45.548 INFO:teuthology.orchestra.run.smithi152.stdout:(11/32): python3-jaraco-6.2-6.el8.noarch.rpm 110 kB/s | 11 kB 00:00 2023-06-10T20:49:45.573 INFO:teuthology.orchestra.run.smithi152.stdout:(12/32): python3-jaraco-functools-2.0-4.el8.noa 175 kB/s | 18 kB 00:00 2023-06-10T20:49:45.598 INFO:teuthology.orchestra.run.smithi152.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 644 kB/s | 49 kB 00:00 2023-06-10T20:49:45.624 INFO:teuthology.orchestra.run.smithi152.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 777 kB/s | 59 kB 00:00 2023-06-10T20:49:45.657 INFO:teuthology.orchestra.run.smithi152.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 265 kB/s | 16 kB 00:00 2023-06-10T20:49:45.674 INFO:teuthology.orchestra.run.smithi152.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 371 kB/s | 19 kB 00:00 2023-06-10T20:49:45.708 INFO:teuthology.orchestra.run.smithi152.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 475 kB/s | 24 kB 00:00 2023-06-10T20:49:45.725 INFO:teuthology.orchestra.run.smithi152.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 584 kB/s | 29 kB 00:00 2023-06-10T20:49:45.750 INFO:teuthology.orchestra.run.smithi152.stdout:(19/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.6 MB/s | 283 kB 00:00 2023-06-10T20:49:45.767 INFO:teuthology.orchestra.run.smithi152.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 467 kB/s | 27 kB 00:00 2023-06-10T20:49:45.817 INFO:teuthology.orchestra.run.smithi152.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 3.7 MB/s | 251 kB 00:00 2023-06-10T20:49:45.843 INFO:teuthology.orchestra.run.smithi152.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.1 MB/s | 255 kB 00:00 2023-06-10T20:49:45.868 INFO:teuthology.orchestra.run.smithi152.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 848 kB/s | 85 kB 00:00 2023-06-10T20:49:45.893 INFO:teuthology.orchestra.run.smithi152.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 303 kB/s | 23 kB 00:00 2023-06-10T20:49:45.952 INFO:teuthology.orchestra.run.smithi152.stdout:(25/32): python3-pytz-2017.2-9.el8.noarch.rpm 497 kB/s | 54 kB 00:00 2023-06-10T20:49:45.960 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:12 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:45.977 INFO:teuthology.orchestra.run.smithi152.stdout:(26/32): python3-markupsafe-0.23-19.el8.x86_64. 357 kB/s | 39 kB 00:00 2023-06-10T20:49:46.003 INFO:teuthology.orchestra.run.smithi152.stdout:(27/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.6 MB/s | 157 kB 00:00 2023-06-10T20:49:46.028 INFO:teuthology.orchestra.run.smithi152.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-06-10T20:49:46.061 INFO:teuthology.orchestra.run.smithi152.stdout:(29/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.1 MB/s | 457 kB 00:00 2023-06-10T20:49:46.087 INFO:teuthology.orchestra.run.smithi152.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 1.3 MB/s | 109 kB 00:00 2023-06-10T20:49:46.121 INFO:teuthology.orchestra.run.smithi152.stdout:(31/32): python3-cryptography-3.2.1-5.el8.x86_6 5.9 MB/s | 559 kB 00:00 2023-06-10T20:49:46.146 INFO:teuthology.orchestra.run.smithi152.stdout:(32/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.8 MB/s | 238 kB 00:00 2023-06-10T20:49:46.147 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:49:46.147 INFO:teuthology.orchestra.run.smithi152.stdout:Total 14 MB/s | 31 MB 00:02 2023-06-10T20:49:46.241 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:49:46.280 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:49:46.280 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:49:46.496 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:49:46.500 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:49:47.470 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:49:47.473 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout: ceph x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 6.4 k 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 2.4 M 2023-06-10T20:49:47.474 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 1.6 M 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 244 k 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 5.0 M 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 18 M 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-06-10T20:49:47.475 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-06-10T20:49:47.476 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-06-10T20:49:47.477 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:47.478 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:49:47.479 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================= 2023-06-10T20:49:47.479 INFO:teuthology.orchestra.run.smithi064.stdout:Install 32 Packages 2023-06-10T20:49:47.479 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:49:47.479 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 31 M 2023-06-10T20:49:47.480 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 101 M 2023-06-10T20:49:47.480 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:49:47.770 INFO:teuthology.orchestra.run.smithi064.stdout:(1/32): ceph-16.2.13-257.gd8c5d349.el8.x86_64.r 28 kB/s | 6.4 kB 00:00 2023-06-10T20:49:47.792 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-06-10T20:49:47.921 INFO:teuthology.orchestra.run.smithi064.stdout:(2/32): ceph-mgr-16.2.13-257.gd8c5d349.el8.x86_ 4.2 MB/s | 1.6 MB 00:00 2023-06-10T20:49:47.979 INFO:teuthology.orchestra.run.smithi064.stdout:(3/32): ceph-mds-16.2.13-257.gd8c5d349.el8.x86_ 5.6 MB/s | 2.4 MB 00:00 2023-06-10T20:49:48.016 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-06-10T20:49:48.055 INFO:teuthology.orchestra.run.smithi064.stdout:(4/32): ceph-mgr-modules-core-16.2.13-257.gd8c5 3.2 MB/s | 244 kB 00:00 2023-06-10T20:49:48.222 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-06-10T20:49:48.296 INFO:teuthology.orchestra.run.smithi064.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 181 kB/s | 44 kB 00:00 2023-06-10T20:49:48.446 INFO:teuthology.orchestra.run.smithi064.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.2 MB/s | 185 kB 00:00 2023-06-10T20:49:48.522 INFO:teuthology.orchestra.run.smithi064.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.3 MB/s | 173 kB 00:00 2023-06-10T20:49:48.618 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-06-10T20:49:48.639 INFO:teuthology.orchestra.run.smithi064.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 3.2 MB/s | 384 kB 00:00 2023-06-10T20:49:48.864 INFO:teuthology.orchestra.run.smithi064.stdout:(9/32): ceph-osd-16.2.13-257.gd8c5d349.el8.x86_ 19 MB/s | 18 MB 00:00 2023-06-10T20:49:48.889 INFO:teuthology.orchestra.run.smithi064.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 44 kB/s | 11 kB 00:00 2023-06-10T20:49:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-06-10T20:49:48.973 INFO:teuthology.orchestra.run.smithi064.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 584 kB/s | 49 kB 00:00 2023-06-10T20:49:49.048 INFO:teuthology.orchestra.run.smithi064.stdout:(12/32): ceph-mon-16.2.13-257.gd8c5d349.el8.x86 3.9 MB/s | 5.0 MB 00:01 2023-06-10T20:49:49.091 INFO:teuthology.orchestra.run.smithi064.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 500 kB/s | 59 kB 00:00 2023-06-10T20:49:49.124 INFO:teuthology.orchestra.run.smithi064.stdout:(14/32): python3-jaraco-functools-2.0-4.el8.noa 68 kB/s | 18 kB 00:00 2023-06-10T20:49:49.145 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-06-10T20:49:49.157 INFO:teuthology.orchestra.run.smithi064.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 254 kB/s | 16 kB 00:00 2023-06-10T20:49:49.191 INFO:teuthology.orchestra.run.smithi064.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 279 kB/s | 19 kB 00:00 2023-06-10T20:49:49.225 INFO:teuthology.orchestra.run.smithi064.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 360 kB/s | 24 kB 00:00 2023-06-10T20:49:49.258 INFO:teuthology.orchestra.run.smithi064.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 437 kB/s | 29 kB 00:00 2023-06-10T20:49:49.292 INFO:teuthology.orchestra.run.smithi064.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 410 kB/s | 27 kB 00:00 2023-06-10T20:49:49.373 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-06-10T20:49:49.392 INFO:teuthology.orchestra.run.smithi064.stdout:(20/32): python3-pecan-1.3.2-9.el8.noarch.rpm 823 kB/s | 283 kB 00:00 2023-06-10T20:49:49.417 INFO:teuthology.orchestra.run.smithi064.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-06-10T20:49:49.443 INFO:teuthology.orchestra.run.smithi064.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.4 MB/s | 255 kB 00:00 2023-06-10T20:49:49.468 INFO:teuthology.orchestra.run.smithi064.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.1 MB/s | 85 kB 00:00 2023-06-10T20:49:49.493 INFO:teuthology.orchestra.run.smithi064.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-06-10T20:49:49.553 INFO:teuthology.orchestra.run.smithi064.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.0 MB/s | 109 kB 00:00 2023-06-10T20:49:49.619 INFO:teuthology.orchestra.run.smithi064.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 2.1 MB/s | 238 kB 00:00 2023-06-10T20:49:49.646 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-06-10T20:49:49.652 INFO:teuthology.orchestra.run.smithi064.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 540 kB/s | 54 kB 00:00 2023-06-10T20:49:49.678 INFO:teuthology.orchestra.run.smithi064.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 670 kB/s | 39 kB 00:00 2023-06-10T20:49:49.703 INFO:teuthology.orchestra.run.smithi064.stdout:(29/32): python3-cryptography-3.2.1-5.el8.x86_6 2.3 MB/s | 559 kB 00:00 2023-06-10T20:49:49.728 INFO:teuthology.orchestra.run.smithi064.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-06-10T20:49:49.753 INFO:teuthology.orchestra.run.smithi064.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-06-10T20:49:49.796 INFO:teuthology.orchestra.run.smithi064.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.8 MB/s | 457 kB 00:00 2023-06-10T20:49:49.800 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:49:49.800 INFO:teuthology.orchestra.run.smithi064.stdout:Total 13 MB/s | 31 MB 00:02 2023-06-10T20:49:49.857 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-06-10T20:49:49.893 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:49:49.934 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:49:49.934 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:49:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-06-10T20:49:50.164 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:49:50.168 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:49:50.300 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-06-10T20:49:50.511 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-06-10T20:49:50.745 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-06-10T20:49:50.935 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-06-10T20:49:51.149 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-06-10T20:49:51.304 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-06-10T20:49:51.372 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:49:51.481 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-06-10T20:49:51.609 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-06-10T20:49:51.654 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-06-10T20:49:51.807 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-06-10T20:49:51.942 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-06-10T20:49:52.023 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-06-10T20:49:52.189 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-06-10T20:49:52.217 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-06-10T20:49:52.417 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-06-10T20:49:52.474 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-06-10T20:49:52.660 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-06-10T20:49:52.775 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-06-10T20:49:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-06-10T20:49:52.963 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-06-10T20:49:53.082 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-06-10T20:49:53.303 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-06-10T20:49:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-06-10T20:49:53.599 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-06-10T20:49:53.731 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-06-10T20:49:53.901 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-06-10T20:49:53.928 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8. 27/32 2023-06-10T20:49:54.110 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-06-10T20:49:54.318 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-06-10T20:49:54.542 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-06-10T20:49:54.586 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 28/32 2023-06-10T20:49:54.717 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-06-10T20:49:54.881 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-06-10T20:49:55.072 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-06-10T20:49:55.246 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-06-10T20:49:55.382 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-06-10T20:49:55.634 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-06-10T20:49:55.864 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-06-10T20:49:56.074 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-06-10T20:49:56.090 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 28/32 2023-06-10T20:49:56.287 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 29/32 2023-06-10T20:49:56.727 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 29/32 2023-06-10T20:49:56.729 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-06-10T20:49:56.894 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 30/32 2023-06-10T20:49:56.985 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-06-10T20:49:57.113 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 30/32 2023-06-10T20:49:57.205 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-06-10T20:49:57.261 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 31/32 2023-06-10T20:49:57.279 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 31/32 2023-06-10T20:49:57.417 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 32/32 2023-06-10T20:49:57.503 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-06-10T20:49:57.774 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 32/32 2023-06-10T20:49:57.774 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 1/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 2/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 3/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 4/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 5/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8. 6/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-06-10T20:49:57.775 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-06-10T20:49:57.776 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-06-10T20:49:57.777 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-06-10T20:49:57.778 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-06-10T20:49:57.779 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-06-10T20:49:57.779 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-06-10T20:49:57.779 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-06-10T20:49:57.828 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-06-10T20:49:58.138 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8. 27/32 2023-06-10T20:49:58.303 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 28/32 2023-06-10T20:49:59.327 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-06-10T20:49:59.327 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:26 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:49:59.890 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 28/32 2023-06-10T20:50:00.101 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 29/32 2023-06-10T20:50:00.559 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 29/32 2023-06-10T20:50:00.717 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 30/32 2023-06-10T20:50:00.947 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 30/32 2023-06-10T20:50:01.117 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 31/32 2023-06-10T20:50:01.136 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 31/32 2023-06-10T20:50:01.257 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 32/32 2023-06-10T20:50:01.600 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:01.608 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 32/32 2023-06-10T20:50:01.608 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 1/32 2023-06-10T20:50:01.608 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 2/32 2023-06-10T20:50:01.608 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 3/32 2023-06-10T20:50:01.608 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 4/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 5/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8. 6/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-06-10T20:50:01.609 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-06-10T20:50:01.610 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-06-10T20:50:01.611 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-06-10T20:50:01.612 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-06-10T20:50:01.612 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-06-10T20:50:01.612 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-06-10T20:50:01.791 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:01.791 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:01.791 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:01.791 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:01.791 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-06-10T20:50:01.792 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-06-10T20:50:01.793 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-06-10T20:50:01.793 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-06-10T20:50:01.793 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-06-10T20:50:01.793 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-6.2-6.el8.noarch 2023-06-10T20:50:01.793 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako-1.0.6-13.el8.noarch 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-06-10T20:50:01.794 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend-2.6-1.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser-2.14-14.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz-2017.2-9.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-06-10T20:50:01.795 INFO:teuthology.orchestra.run.smithi152.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:01.796 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:01.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-base 2023-06-10T20:50:02.305 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:03.108 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:30 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-06-10T20:50:03.190 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:30 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:04.352 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:50:04.463 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:04.464 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:50:04.465 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:04.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install cephadm 2023-06-10T20:50:04.914 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:05.714 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:50:05.775 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:32 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:05.921 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:05.922 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:50:05.922 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:05.922 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mds-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:05.922 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:05.923 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-modules-core-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:05.923 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mon-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:05.923 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-osd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:05.924 INFO:teuthology.orchestra.run.smithi064.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-06-10T20:50:05.924 INFO:teuthology.orchestra.run.smithi064.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-06-10T20:50:05.924 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-06-10T20:50:05.924 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-06-10T20:50:05.924 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-6.2-6.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-mako-1.0.6-13.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-06-10T20:50:05.925 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-portend-2.6-1.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pycparser-2.14-14.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pytz-2017.2-9.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-06-10T20:50:05.926 INFO:teuthology.orchestra.run.smithi064.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-06-10T20:50:05.927 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:05.928 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:06.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-base 2023-06-10T20:50:06.445 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:07.145 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:07.146 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:07.146 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:50:07.146 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:07.146 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:50:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 84 k 2023-06-10T20:50:07.147 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:07.147 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:50:07.147 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:07.147 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:50:07.148 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:07.148 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 84 k 2023-06-10T20:50:07.148 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 348 k 2023-06-10T20:50:07.149 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:50:07.307 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:34 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:07.307 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm-16.2.13-257.gd8c5d349.el8.noarch.rpm 526 kB/s | 84 kB 00:00 2023-06-10T20:50:07.308 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:07.308 INFO:teuthology.orchestra.run.smithi152.stdout:Total 522 kB/s | 84 kB 00:00 2023-06-10T20:50:07.309 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:50:07.315 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:50:07.316 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:50:07.317 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:50:07.321 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:50:07.365 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:50:08.718 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-base-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:50:08.839 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:08.841 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:50:08.841 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:08.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install cephadm 2023-06-10T20:50:08.999 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:09.137 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:09.306 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:09.324 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:10.145 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:37 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:10.368 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:10.369 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:37 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:11.663 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:11.664 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 84 k 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:50:11.665 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:11.666 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:50:11.666 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:11.666 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 84 k 2023-06-10T20:50:11.666 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 348 k 2023-06-10T20:50:11.666 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:50:11.867 INFO:teuthology.orchestra.run.smithi064.stdout:cephadm-16.2.13-257.gd8c5d349.el8.noarch.rpm 417 kB/s | 84 kB 00:00 2023-06-10T20:50:11.868 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:11.868 INFO:teuthology.orchestra.run.smithi064.stdout:Total 415 kB/s | 84 kB 00:00 2023-06-10T20:50:11.868 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:50:11.876 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:50:11.876 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:50:11.878 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:50:11.882 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:50:11.925 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:50:12.691 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:12.875 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:12.875 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:12.875 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:12.875 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:12.875 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:13.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-immutable-object-cache 2023-06-10T20:50:13.363 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:14.158 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:41 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:14.492 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:14.630 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:14.804 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:15.550 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:15.551 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:15.551 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 161 k 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:50:15.552 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:50:15.553 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:15.553 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 161 k 2023-06-10T20:50:15.553 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 471 k 2023-06-10T20:50:15.553 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:50:15.732 INFO:teuthology.orchestra.run.smithi152.stdout:ceph-immutable-object-cache-16.2.13-257.gd8c5d3 899 kB/s | 161 kB 00:00 2023-06-10T20:50:15.732 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:15.733 INFO:teuthology.orchestra.run.smithi152.stdout:Total 894 kB/s | 161 kB 00:00 2023-06-10T20:50:15.733 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:50:15.743 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:50:15.743 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:50:15.798 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:50:15.801 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:50:15.894 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/1 2023-06-10T20:50:15.894 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:42 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:15.932 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:50:16.100 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:17.546 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:17.547 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:44 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:18.821 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:50:19.019 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:19.019 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:50:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:19.020 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:19.020 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:19.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-immutable-object-cache 2023-06-10T20:50:19.588 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:19.848 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:20.028 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:20.028 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:20.029 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:20.029 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:20.029 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:20.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr 2023-06-10T20:50:20.408 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:47 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:20.510 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:21.336 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:48 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:21.940 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:21.941 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:21.941 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 161 k 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:50:21.942 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:21.943 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 161 k 2023-06-10T20:50:21.943 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 471 k 2023-06-10T20:50:21.943 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:50:22.131 INFO:teuthology.orchestra.run.smithi064.stdout:ceph-immutable-object-cache-16.2.13-257.gd8c5d3 854 kB/s | 161 kB 00:00 2023-06-10T20:50:22.131 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:22.132 INFO:teuthology.orchestra.run.smithi064.stdout:Total 848 kB/s | 161 kB 00:00 2023-06-10T20:50:22.132 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:50:22.143 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:50:22.144 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:50:22.202 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:50:22.205 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:50:22.336 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:50:22.531 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:22.611 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:50:22.720 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:22.721 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:50:22.722 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:22.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-dashboard 2023-06-10T20:50:22.896 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:23.178 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349. 1/1 2023-06-10T20:50:23.997 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:50 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:24.006 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:51 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:25.369 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:25.370 INFO:teuthology.orchestra.run.smithi152.stdout:=================================================================================================== 2023-06-10T20:50:25.370 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout:=================================================================================================== 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 3.1 M 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 24 k 2023-06-10T20:50:25.371 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 15 k 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout:=================================================================================================== 2023-06-10T20:50:25.372 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2023-06-10T20:50:25.373 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:25.373 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 3.4 M 2023-06-10T20:50:25.373 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 22 M 2023-06-10T20:50:25.373 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:50:25.771 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): ceph-grafana-dashboards-16.2.13-257.gd8c 101 kB/s | 24 kB 00:00 2023-06-10T20:50:25.788 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): ceph-prometheus-alerts-16.2.13-257.gd8c5 58 kB/s | 15 kB 00:00 2023-06-10T20:50:25.863 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 366 kB/s | 34 kB 00:00 2023-06-10T20:50:25.930 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.4 MB/s | 196 kB 00:00 2023-06-10T20:50:25.947 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 510 kB/s | 43 kB 00:00 2023-06-10T20:50:26.031 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): ceph-mgr-dashboard-16.2.13-257.gd8c5d349 6.3 MB/s | 3.1 MB 00:00 2023-06-10T20:50:26.032 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:26.033 INFO:teuthology.orchestra.run.smithi152.stdout:Total 5.2 MB/s | 3.4 MB 00:00 2023-06-10T20:50:26.044 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:50:26.054 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:50:26.054 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:50:26.121 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:50:26.125 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:50:26.426 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:50:26.473 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:50:26.574 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-06-10T20:50:26.679 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:26.679 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:50:26.679 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-immutable-object-cache-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:50:26.679 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:26.679 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:26.812 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-06-10T20:50:26.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr 2023-06-10T20:50:27.008 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-06-10T20:50:27.129 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.n 4/6 2023-06-10T20:50:27.190 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:28.011 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:55 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:28.177 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8. 5/6 2023-06-10T20:50:28.738 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 6/6 2023-06-10T20:50:28.972 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 6/6 2023-06-10T20:50:28.972 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8. 1/6 2023-06-10T20:50:28.972 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 2/6 2023-06-10T20:50:28.972 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.n 3/6 2023-06-10T20:50:28.972 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-06-10T20:50:28.973 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-06-10T20:50:29.353 INFO:teuthology.orchestra.run.smithi064.stdout:Package ceph-mgr-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:50:29.471 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:29.472 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:50:29.472 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:29.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-dashboard 2023-06-10T20:50:30.019 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-06-10T20:50:30.020 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:57 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:30.283 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:31.160 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:01:58 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:32.661 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:32.662 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-06-10T20:50:32.662 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 3.1 M 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 24 k 2023-06-10T20:50:32.663 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 15 k 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:50:32.664 INFO:teuthology.orchestra.run.smithi064.stdout:=================================================================================================== 2023-06-10T20:50:32.665 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2023-06-10T20:50:32.665 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:32.665 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 3.4 M 2023-06-10T20:50:32.665 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 22 M 2023-06-10T20:50:32.666 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:50:32.729 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:32.870 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): ceph-grafana-dashboards-16.2.13-257.gd8c 170 kB/s | 24 kB 00:00 2023-06-10T20:50:32.886 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): ceph-prometheus-alerts-16.2.13-257.gd8c5 93 kB/s | 15 kB 00:00 2023-06-10T20:50:32.915 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:32.915 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes-2.4.1-12.el8.noarch 2023-06-10T20:50:32.916 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:32.917 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:32.987 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 288 kB/s | 34 kB 00:00 2023-06-10T20:50:33.045 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 733 kB/s | 43 kB 00:00 2023-06-10T20:50:33.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-diskprediction-local 2023-06-10T20:50:33.079 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.0 MB/s | 196 kB 00:00 2023-06-10T20:50:33.262 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): ceph-mgr-dashboard-16.2.13-257.gd8c5d349 5.8 MB/s | 3.1 MB 00:00 2023-06-10T20:50:33.264 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:33.264 INFO:teuthology.orchestra.run.smithi064.stdout:Total 5.7 MB/s | 3.4 MB 00:00 2023-06-10T20:50:33.276 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:50:33.286 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:50:33.287 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:50:33.359 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:50:33.363 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:50:33.400 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:33.662 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:50:33.839 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-06-10T20:50:34.062 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-06-10T20:50:34.244 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:01 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:34.273 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-06-10T20:50:34.394 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.n 4/6 2023-06-10T20:50:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8. 5/6 2023-06-10T20:50:35.672 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:35.673 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================================= 2023-06-10T20:50:35.673 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:50:35.673 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================================= 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 8.6 M 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-06-10T20:50:35.674 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================================= 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout:Install 8 Packages 2023-06-10T20:50:35.675 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:35.676 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 37 M 2023-06-10T20:50:35.676 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 210 M 2023-06-10T20:50:35.676 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:50:35.748 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 6/6 2023-06-10T20:50:35.864 INFO:teuthology.orchestra.run.smithi152.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 MB/s | 3.7 MB 00:00 2023-06-10T20:50:35.914 INFO:teuthology.orchestra.run.smithi152.stdout:(2/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-06-10T20:50:35.991 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 6/6 2023-06-10T20:50:35.991 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8. 1/6 2023-06-10T20:50:35.991 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarc 2/6 2023-06-10T20:50:35.991 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.n 3/6 2023-06-10T20:50:35.991 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-06-10T20:50:35.992 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-06-10T20:50:36.115 INFO:teuthology.orchestra.run.smithi152.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 31 MB/s | 14 MB 00:00 2023-06-10T20:50:36.215 INFO:teuthology.orchestra.run.smithi152.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 16 MB/s | 4.9 MB 00:00 2023-06-10T20:50:36.324 INFO:teuthology.orchestra.run.smithi152.stdout:(5/8): ceph-mgr-diskprediction-local-16.2.13-25 13 MB/s | 8.6 MB 00:00 2023-06-10T20:50:36.382 INFO:teuthology.orchestra.run.smithi152.stdout:(6/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.8 MB/s | 644 kB 00:00 2023-06-10T20:50:36.407 INFO:teuthology.orchestra.run.smithi152.stdout:(7/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 2.0 MB/s | 171 kB 00:00 2023-06-10T20:50:36.499 INFO:teuthology.orchestra.run.smithi152.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-06-10T20:50:36.500 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:36.500 INFO:teuthology.orchestra.run.smithi152.stdout:Total 44 MB/s | 37 MB 00:00 2023-06-10T20:50:36.737 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:50:36.755 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:50:36.755 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:50:37.053 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:50:37.056 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:50:37.077 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-06-10T20:50:37.078 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:04 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:37.605 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:50:37.746 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-06-10T20:50:37.862 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-06-10T20:50:38.029 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-06-10T20:50:38.659 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-06-10T20:50:38.912 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-06-10T20:50:39.507 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-06-10T20:50:39.562 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:50:39.670 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-06-10T20:50:39.763 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:39.763 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:50:39.763 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-grafana-dashboards-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:39.764 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-dashboard-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:39.764 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-prometheus-alerts-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:39.764 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-06-10T20:50:39.764 INFO:teuthology.orchestra.run.smithi064.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-06-10T20:50:39.765 INFO:teuthology.orchestra.run.smithi064.stdout: python3-routes-2.4.1-12.el8.noarch 2023-06-10T20:50:39.765 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:39.765 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:39.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-diskprediction-local 2023-06-10T20:50:40.271 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:40.348 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-06-10T20:50:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-06-10T20:50:40.812 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-06-10T20:50:41.116 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:08 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-06-10T20:50:42.650 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:42.651 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-06-10T20:50:42.651 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:50:42.651 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-06-10T20:50:42.651 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 8.6 M 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-06-10T20:50:42.652 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================================= 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout:Install 8 Packages 2023-06-10T20:50:42.653 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:42.654 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 37 M 2023-06-10T20:50:42.654 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 210 M 2023-06-10T20:50:42.654 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:50:42.726 INFO:teuthology.orchestra.run.smithi064.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 8.7 MB/s | 644 kB 00:00 2023-06-10T20:50:42.751 INFO:teuthology.orchestra.run.smithi064.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.7 MB/s | 171 kB 00:00 2023-06-10T20:50:42.869 INFO:teuthology.orchestra.run.smithi064.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 32 MB/s | 3.7 MB 00:00 2023-06-10T20:50:42.911 INFO:teuthology.orchestra.run.smithi064.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-06-10T20:50:43.061 INFO:teuthology.orchestra.run.smithi064.stdout:(5/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 32 MB/s | 4.9 MB 00:00 2023-06-10T20:50:43.219 INFO:teuthology.orchestra.run.smithi064.stdout:(6/8): python3-scipy-1.0.0-21.module+el8.5.0+10 28 MB/s | 14 MB 00:00 2023-06-10T20:50:43.345 INFO:teuthology.orchestra.run.smithi064.stdout:(7/8): ceph-mgr-diskprediction-local-16.2.13-25 12 MB/s | 8.6 MB 00:00 2023-06-10T20:50:43.453 INFO:teuthology.orchestra.run.smithi064.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-06-10T20:50:43.454 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:43.454 INFO:teuthology.orchestra.run.smithi064.stdout:Total 46 MB/s | 37 MB 00:00 2023-06-10T20:50:43.698 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:50:43.718 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:50:43.718 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:50:43.837 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-06-10T20:50:44.034 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:50:44.038 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:50:44.096 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 8/8 2023-06-10T20:50:44.363 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 8/8 2023-06-10T20:50:44.363 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 1/8 2023-06-10T20:50:44.363 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-06-10T20:50:44.363 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-06-10T20:50:44.364 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-06-10T20:50:44.364 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-06-10T20:50:44.364 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-06-10T20:50:44.364 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-06-10T20:50:44.606 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:50:44.752 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-06-10T20:50:44.883 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-06-10T20:50:45.061 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-06-10T20:50:45.416 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-06-10T20:50:45.416 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:12 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-06-10T20:50:46.285 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-06-10T20:50:46.924 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-06-10T20:50:47.102 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-06-10T20:50:47.762 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:47.828 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-06-10T20:50:47.949 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:47.949 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:47.949 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:47.949 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-06-10T20:50:47.949 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-0.3.15-3.el8.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:47.950 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:48.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-rook 2023-06-10T20:50:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-06-10T20:50:48.385 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-06-10T20:50:48.435 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:49.237 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:16 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:50.258 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-06-10T20:50:50.612 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================== 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================== 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 27 k 2023-06-10T20:50:50.614 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-06-10T20:50:50.615 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:50:50.616 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================================== 2023-06-10T20:50:50.617 INFO:teuthology.orchestra.run.smithi152.stdout:Install 13 Packages 2023-06-10T20:50:50.617 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:50.617 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 1.9 M 2023-06-10T20:50:50.618 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 24 M 2023-06-10T20:50:50.618 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:50:50.823 INFO:teuthology.orchestra.run.smithi152.stdout:(1/13): ceph-mgr-rook-16.2.13-257.gd8c5d349.el8 187 kB/s | 27 kB 00:00 2023-06-10T20:50:50.848 INFO:teuthology.orchestra.run.smithi152.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 95 kB/s | 16 kB 00:00 2023-06-10T20:50:50.874 INFO:teuthology.orchestra.run.smithi152.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 168 kB/s | 33 kB 00:00 2023-06-10T20:50:50.932 INFO:teuthology.orchestra.run.smithi152.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 747 kB/s | 81 kB 00:00 2023-06-10T20:50:50.999 INFO:teuthology.orchestra.run.smithi152.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 919 kB/s | 61 kB 00:00 2023-06-10T20:50:51.074 INFO:teuthology.orchestra.run.smithi152.stdout:(6/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.4 MB/s | 110 kB 00:00 2023-06-10T20:50:51.100 INFO:teuthology.orchestra.run.smithi152.stdout:(7/13): python3-rsa-4.8-1.el8.noarch.rpm 272 kB/s | 61 kB 00:00 2023-06-10T20:50:51.133 INFO:teuthology.orchestra.run.smithi152.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 4.1 MB/s | 1.2 MB 00:00 2023-06-10T20:50:51.167 INFO:teuthology.orchestra.run.smithi152.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 287 kB/s | 27 kB 00:00 2023-06-10T20:50:51.201 INFO:teuthology.orchestra.run.smithi152.stdout:(10/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2023-06-10T20:50:51.226 INFO:teuthology.orchestra.run.smithi152.stdout:(11/13): python3-jsonpointer-1.10-11.el8.noarch 230 kB/s | 20 kB 00:00 2023-06-10T20:50:51.251 INFO:teuthology.orchestra.run.smithi152.stdout:(12/13): python3-requests-oauthlib-1.0.0-1.el8. 546 kB/s | 43 kB 00:00 2023-06-10T20:50:51.276 INFO:teuthology.orchestra.run.smithi152.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 2.0 MB/s | 155 kB 00:00 2023-06-10T20:50:51.277 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:51.277 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.9 MB/s | 1.9 MB 00:00 2023-06-10T20:50:51.337 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:50:51.352 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:50:51.352 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:50:51.423 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:50:51.427 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:50:51.611 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-06-10T20:50:51.827 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 8/8 2023-06-10T20:50:51.922 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:50:52.098 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 8/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d34 1/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-06-10T20:50:52.099 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-06-10T20:50:52.100 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-06-10T20:50:52.127 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-06-10T20:50:52.346 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rsa-4.8-1.el8.noarch 2/13 2023-06-10T20:50:52.562 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-06-10T20:50:52.780 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-06-10T20:50:52.949 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-06-10T20:50:53.117 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-06-10T20:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-06-10T20:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:20 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:53.287 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-06-10T20:50:53.475 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-06-10T20:50:53.657 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-06-10T20:50:53.840 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-06-10T20:50:54.875 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-06-10T20:50:55.125 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-06-10T20:50:55.276 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 13/13 2023-06-10T20:50:55.519 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 13/13 2023-06-10T20:50:55.519 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 1/13 2023-06-10T20:50:55.519 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 6/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-06-10T20:50:55.520 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-06-10T20:50:55.521 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-06-10T20:50:55.521 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-06-10T20:50:55.521 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-06-10T20:50:55.521 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-06-10T20:50:55.685 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:50:55.887 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-diskprediction-local-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-0.3.15-3.el8.x86_64 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-06-10T20:50:55.888 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-06-10T20:50:55.889 INFO:teuthology.orchestra.run.smithi064.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-06-10T20:50:55.889 INFO:teuthology.orchestra.run.smithi064.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-06-10T20:50:55.889 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:55.889 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:50:56.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-rook 2023-06-10T20:50:56.415 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:50:56.669 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-06-10T20:50:56.669 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:23 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:57.298 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:24 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:50:58.828 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 27 k 2023-06-10T20:50:58.830 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-06-10T20:50:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:58.832 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:50:58.833 INFO:teuthology.orchestra.run.smithi064.stdout:======================================================================================================== 2023-06-10T20:50:58.833 INFO:teuthology.orchestra.run.smithi064.stdout:Install 13 Packages 2023-06-10T20:50:58.833 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:50:58.833 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 1.9 M 2023-06-10T20:50:58.833 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 24 M 2023-06-10T20:50:58.834 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:50:59.016 INFO:teuthology.orchestra.run.smithi064.stdout:(1/13): ceph-mgr-rook-16.2.13-257.gd8c5d349.el8 174 kB/s | 27 kB 00:00 2023-06-10T20:50:59.042 INFO:teuthology.orchestra.run.smithi064.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 89 kB/s | 16 kB 00:00 2023-06-10T20:50:59.042 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:50:59.067 INFO:teuthology.orchestra.run.smithi064.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 159 kB/s | 33 kB 00:00 2023-06-10T20:50:59.159 INFO:teuthology.orchestra.run.smithi064.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 571 kB/s | 81 kB 00:00 2023-06-10T20:50:59.226 INFO:teuthology.orchestra.run.smithi064.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 918 kB/s | 61 kB 00:00 2023-06-10T20:50:59.229 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:59.229 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:50:59.229 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-06-10T20:50:59.230 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa-4.8-1.el8.noarch 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:50:59.231 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:50:59.251 INFO:teuthology.orchestra.run.smithi064.stdout:(6/13): python3-rsa-4.8-1.el8.noarch.rpm 334 kB/s | 61 kB 00:00 2023-06-10T20:50:59.318 INFO:teuthology.orchestra.run.smithi064.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 4.2 MB/s | 1.2 MB 00:00 2023-06-10T20:50:59.343 INFO:teuthology.orchestra.run.smithi064.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 364 kB/s | 43 kB 00:00 2023-06-10T20:50:59.366 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-cephadm 2023-06-10T20:50:59.377 INFO:teuthology.orchestra.run.smithi064.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.2 MB/s | 155 kB 00:00 2023-06-10T20:50:59.402 INFO:teuthology.orchestra.run.smithi064.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 528 kB/s | 27 kB 00:00 2023-06-10T20:50:59.427 INFO:teuthology.orchestra.run.smithi064.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 1.0 MB/s | 110 kB 00:00 2023-06-10T20:50:59.452 INFO:teuthology.orchestra.run.smithi064.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 399 kB/s | 20 kB 00:00 2023-06-10T20:50:59.478 INFO:teuthology.orchestra.run.smithi064.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2023-06-10T20:50:59.478 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:50:59.479 INFO:teuthology.orchestra.run.smithi064.stdout:Total 3.0 MB/s | 1.9 MB 00:00 2023-06-10T20:50:59.539 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:50:59.555 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:50:59.555 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:50:59.631 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:50:59.635 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:50:59.721 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:00.124 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:51:00.353 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-06-10T20:51:00.557 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:27 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:00.572 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-rsa-4.8-1.el8.noarch 2/13 2023-06-10T20:51:00.757 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-06-10T20:51:00.925 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-06-10T20:51:01.167 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-06-10T20:51:01.373 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-06-10T20:51:01.630 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-06-10T20:51:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-06-10T20:51:01.910 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:01.911 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================================== 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================================== 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 116 k 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:51:01.912 INFO:teuthology.orchestra.run.smithi152.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-06-10T20:51:01.913 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-06-10T20:51:01.913 INFO:teuthology.orchestra.run.smithi152.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-06-10T20:51:01.913 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-06-10T20:51:01.913 INFO:teuthology.orchestra.run.smithi152.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-06-10T20:51:01.914 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:01.914 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:51:01.914 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================================== 2023-06-10T20:51:01.914 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2023-06-10T20:51:01.914 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:01.915 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 5.8 M 2023-06-10T20:51:01.915 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 24 M 2023-06-10T20:51:01.915 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:51:02.041 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-06-10T20:51:02.087 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): python3-apipkg-1.5-6.el8.noarch.rpm 189 kB/s | 21 kB 00:00 2023-06-10T20:51:02.220 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): python3-remoto-1.2.1-1.el8.noarch.rpm 366 kB/s | 49 kB 00:00 2023-06-10T20:51:02.245 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): ceph-mgr-cephadm-16.2.13-257.gd8c5d349.e 429 kB/s | 116 kB 00:00 2023-06-10T20:51:02.277 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-06-10T20:51:02.293 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.1 MB/s | 355 kB 00:00 2023-06-10T20:51:02.396 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.0 MB/s | 538 kB 00:00 2023-06-10T20:51:02.488 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): python3-babel-2.5.1-7.el8.noarch.rpm 20 MB/s | 4.8 MB 00:00 2023-06-10T20:51:02.489 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:02.489 INFO:teuthology.orchestra.run.smithi152.stdout:Total 10 MB/s | 5.8 MB 00:00 2023-06-10T20:51:02.543 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:51:02.553 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:51:02.553 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:51:02.631 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:51:02.635 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:51:03.404 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-06-10T20:51:03.602 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:51:03.658 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-06-10T20:51:03.799 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 13/13 2023-06-10T20:51:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 13/13 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 1/13 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-06-10T20:51:04.062 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 6/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-06-10T20:51:04.063 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-06-10T20:51:04.248 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-06-10T20:51:04.491 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-06-10T20:51:04.711 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-06-10T20:51:04.933 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-06-10T20:51:05.091 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 6/6 2023-06-10T20:51:05.215 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-06-10T20:51:05.215 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:32 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:05.352 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 6/6 2023-06-10T20:51:05.352 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/6 2023-06-10T20:51:05.353 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-06-10T20:51:05.353 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-06-10T20:51:05.353 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-06-10T20:51:05.353 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-06-10T20:51:06.422 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-06-10T20:51:06.422 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:33 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:07.682 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:51:07.883 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-rook-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-06-10T20:51:07.884 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-06-10T20:51:07.885 INFO:teuthology.orchestra.run.smithi064.stdout: python3-rsa-4.8-1.el8.noarch 2023-06-10T20:51:07.886 INFO:teuthology.orchestra.run.smithi064.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-06-10T20:51:07.886 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:07.886 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:08.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-mgr-cephadm 2023-06-10T20:51:08.432 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:08.953 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:51:09.139 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: python3-apipkg-1.5-6.el8.noarch 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel-2.5.1-7.el8.noarch 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-06-10T20:51:09.140 INFO:teuthology.orchestra.run.smithi152.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-06-10T20:51:09.141 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:09.141 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:09.284 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-fuse 2023-06-10T20:51:09.291 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:36 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:09.642 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:10.433 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:37 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:10.821 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:10.822 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-06-10T20:51:10.822 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:51:10.822 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-06-10T20:51:10.822 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm noarch 2:16.2.13-257.gd8c5d349.el8 ceph-noarch 116 k 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-06-10T20:51:10.823 INFO:teuthology.orchestra.run.smithi064.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-06-10T20:51:10.824 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:10.824 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:51:10.824 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================================== 2023-06-10T20:51:10.824 INFO:teuthology.orchestra.run.smithi064.stdout:Install 6 Packages 2023-06-10T20:51:10.824 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:10.825 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 5.8 M 2023-06-10T20:51:10.825 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 24 M 2023-06-10T20:51:10.825 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:51:11.037 INFO:teuthology.orchestra.run.smithi064.stdout:(1/6): ceph-mgr-cephadm-16.2.13-257.gd8c5d349.e 629 kB/s | 116 kB 00:00 2023-06-10T20:51:11.061 INFO:teuthology.orchestra.run.smithi064.stdout:(2/6): python3-apipkg-1.5-6.el8.noarch.rpm 101 kB/s | 21 kB 00:00 2023-06-10T20:51:11.129 INFO:teuthology.orchestra.run.smithi064.stdout:(3/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 7.9 MB/s | 538 kB 00:00 2023-06-10T20:51:11.162 INFO:teuthology.orchestra.run.smithi064.stdout:(4/6): python3-execnet-1.7.1-1.el8.noarch.rpm 1.1 MB/s | 355 kB 00:00 2023-06-10T20:51:11.271 INFO:teuthology.orchestra.run.smithi064.stdout:(5/6): python3-babel-2.5.1-7.el8.noarch.rpm 34 MB/s | 4.8 MB 00:00 2023-06-10T20:51:11.296 INFO:teuthology.orchestra.run.smithi064.stdout:(6/6): python3-remoto-1.2.1-1.el8.noarch.rpm 189 kB/s | 49 kB 00:00 2023-06-10T20:51:11.298 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:11.298 INFO:teuthology.orchestra.run.smithi064.stdout:Total 12 MB/s | 5.8 MB 00:00 2023-06-10T20:51:11.353 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:51:11.364 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:51:11.364 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:51:11.448 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:51:11.452 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:51:11.802 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:11.803 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:11.803 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2023-06-10T20:51:11.803 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:11.803 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:51:11.803 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout: x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 909 k 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:51:11.804 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:11.805 INFO:teuthology.orchestra.run.smithi152.stdout:Install 2 Packages 2023-06-10T20:51:11.805 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:11.806 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 992 k 2023-06-10T20:51:11.806 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 2.8 M 2023-06-10T20:51:11.806 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:51:11.899 INFO:teuthology.orchestra.run.smithi152.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 879 kB/s | 83 kB 00:00 2023-06-10T20:51:12.082 INFO:teuthology.orchestra.run.smithi152.stdout:(2/2): ceph-fuse-16.2.13-257.gd8c5d349.el8.x86_ 3.2 MB/s | 909 kB 00:00 2023-06-10T20:51:12.083 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:12.083 INFO:teuthology.orchestra.run.smithi152.stdout:Total 3.5 MB/s | 992 kB 00:00 2023-06-10T20:51:12.094 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:51:12.114 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:51:12.115 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:51:12.188 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:51:12.192 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:51:12.493 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:51:12.614 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-06-10T20:51:12.747 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2/2 2023-06-10T20:51:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/6 2023-06-10T20:51:13.079 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2/2 2023-06-10T20:51:13.080 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/2 2023-06-10T20:51:13.122 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/6 2023-06-10T20:51:13.446 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 3/6 2023-06-10T20:51:13.704 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 4/6 2023-06-10T20:51:13.983 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 5/6 2023-06-10T20:51:14.148 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 6/6 2023-06-10T20:51:14.163 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-06-10T20:51:14.164 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:41 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:14.377 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 6/6 2023-06-10T20:51:14.378 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 1/6 2023-06-10T20:51:14.378 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 2/6 2023-06-10T20:51:14.378 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 3/6 2023-06-10T20:51:14.378 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 4/6 2023-06-10T20:51:14.378 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-06-10T20:51:15.479 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 6/6 2023-06-10T20:51:15.479 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:42 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:16.453 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:51:16.638 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:16.638 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:51:16.639 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-06-10T20:51:16.639 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:16.639 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:17.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados-devel 2023-06-10T20:51:18.004 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:51:18.006 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:18.209 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-mgr-cephadm-2:16.2.13-257.gd8c5d349.el8.noarch 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout: python3-apipkg-1.5-6.el8.noarch 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout: python3-babel-2.5.1-7.el8.noarch 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout: python3-execnet-1.7.1-1.el8.noarch 2023-06-10T20:51:18.210 INFO:teuthology.orchestra.run.smithi064.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-06-10T20:51:18.211 INFO:teuthology.orchestra.run.smithi064.stdout: python3-remoto-1.2.1-1.el8.noarch 2023-06-10T20:51:18.211 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:18.211 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:18.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install ceph-fuse 2023-06-10T20:51:18.771 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:18.827 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:45 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:19.631 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:46 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:20.274 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:20.274 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 131 k 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:51:20.275 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:20.276 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:51:20.276 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:20.276 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 131 k 2023-06-10T20:51:20.277 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 432 k 2023-06-10T20:51:20.277 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:51:20.581 INFO:teuthology.orchestra.run.smithi152.stdout:librados-devel-16.2.13-257.gd8c5d349.el8.x86_64 430 kB/s | 131 kB 00:00 2023-06-10T20:51:20.581 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:20.581 INFO:teuthology.orchestra.run.smithi152.stdout:Total 428 kB/s | 131 kB 00:00 2023-06-10T20:51:20.582 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:51:20.587 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:51:20.587 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:51:20.642 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:51:20.645 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:51:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:51:20.920 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:21.137 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:21.159 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repository Size 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse 2023-06-10T20:51:21.160 INFO:teuthology.orchestra.run.smithi064.stdout: x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 909 k 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout:Installing dependencies: 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout:Install 2 Packages 2023-06-10T20:51:21.161 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:21.162 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 992 k 2023-06-10T20:51:21.162 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 2.8 M 2023-06-10T20:51:21.162 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:51:21.247 INFO:teuthology.orchestra.run.smithi064.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 971 kB/s | 83 kB 00:00 2023-06-10T20:51:21.572 INFO:teuthology.orchestra.run.smithi064.stdout:(2/2): ceph-fuse-16.2.13-257.gd8c5d349.el8.x86_ 2.2 MB/s | 909 kB 00:00 2023-06-10T20:51:21.573 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:21.573 INFO:teuthology.orchestra.run.smithi064.stdout:Total 2.4 MB/s | 992 kB 00:00 2023-06-10T20:51:21.584 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:51:21.606 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:51:21.606 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:51:21.685 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:51:21.689 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:51:21.892 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:51:22.185 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-06-10T20:51:22.223 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:22.224 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:49 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:22.388 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2/2 2023-06-10T20:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2/2 2023-06-10T20:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/2 2023-06-10T20:51:24.613 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:51:24.661 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-06-10T20:51:24.661 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:51 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:24.797 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:51:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:51:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:24.797 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:24.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs2 2023-06-10T20:51:25.289 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:26.107 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:53 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:27.108 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:51:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:51:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: ceph-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-06-10T20:51:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:27.310 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:27.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados-devel 2023-06-10T20:51:27.476 INFO:teuthology.orchestra.run.smithi152.stdout:Package libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:27.585 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:27.586 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:27.587 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:27.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs-devel 2023-06-10T20:51:27.832 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:28.069 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:28.637 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:55 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:28.911 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:55 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:30.180 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:30.180 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 131 k 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:30.181 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:51:30.182 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:30.182 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:51:30.182 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:30.182 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 131 k 2023-06-10T20:51:30.182 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 432 k 2023-06-10T20:51:30.183 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:51:30.350 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:30.350 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:30.350 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2023-06-10T20:51:30.350 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 25 k 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:30.351 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:51:30.352 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:30.352 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 25 k 2023-06-10T20:51:30.352 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 107 k 2023-06-10T20:51:30.352 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:51:30.370 INFO:teuthology.orchestra.run.smithi064.stdout:librados-devel-16.2.13-257.gd8c5d349.el8.x86_64 697 kB/s | 131 kB 00:00 2023-06-10T20:51:30.370 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:30.371 INFO:teuthology.orchestra.run.smithi064.stdout:Total 693 kB/s | 131 kB 00:00 2023-06-10T20:51:30.371 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:51:30.377 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:51:30.377 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:51:30.436 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:51:30.440 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:51:30.560 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:51:30.612 INFO:teuthology.orchestra.run.smithi152.stdout:libcephfs-devel-16.2.13-257.gd8c5d349.el8.x86_6 98 kB/s | 25 kB 00:00 2023-06-10T20:51:30.613 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:30.613 INFO:teuthology.orchestra.run.smithi152.stdout:Total 97 kB/s | 25 kB 00:00 2023-06-10T20:51:30.613 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:51:30.617 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:51:30.617 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:51:30.619 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:51:30.623 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:51:30.670 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:30.826 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:30.973 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:31.027 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:32.051 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:32.051 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:02:59 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:32.092 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:32.093 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:02:59 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:34.584 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:51:34.764 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:34.765 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:51:34.765 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:51:34.765 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:34.765 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:34.882 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:51:34.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados2 2023-06-10T20:51:35.082 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:35.082 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:51:35.082 INFO:teuthology.orchestra.run.smithi064.stdout: librados-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:51:35.082 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:35.083 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:35.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs2 2023-06-10T20:51:35.254 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:35.616 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:36.042 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:03 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:36.449 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:03 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:37.355 INFO:teuthology.orchestra.run.smithi152.stdout:Package librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:37.465 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:37.467 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:37.467 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:37.590 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librbd1 2023-06-10T20:51:37.853 INFO:teuthology.orchestra.run.smithi064.stdout:Package libcephfs2-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:37.944 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:51:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:38.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install libcephfs-devel 2023-06-10T20:51:38.466 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:38.800 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:05 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:39.319 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:06 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:40.064 INFO:teuthology.orchestra.run.smithi152.stdout:Package librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:40.173 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:40.174 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:40.174 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:40.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rados 2023-06-10T20:51:40.653 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:40.789 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:40.790 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:40.790 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-06-10T20:51:40.790 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 25 k 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:51:40.791 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:40.792 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 25 k 2023-06-10T20:51:40.792 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 107 k 2023-06-10T20:51:40.792 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:51:40.967 INFO:teuthology.orchestra.run.smithi064.stdout:libcephfs-devel-16.2.13-257.gd8c5d349.el8.x86_6 145 kB/s | 25 kB 00:00 2023-06-10T20:51:40.968 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:40.968 INFO:teuthology.orchestra.run.smithi064.stdout:Total 144 kB/s | 25 kB 00:00 2023-06-10T20:51:40.968 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:51:40.972 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:51:40.972 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:51:40.975 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:51:40.979 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:51:41.050 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:51:41.218 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:41.438 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:41.460 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:08 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:42.544 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:42.544 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:09 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:42.705 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:42.811 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:42.812 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:42.813 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:42.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rgw 2023-06-10T20:51:43.274 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:44.113 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:11 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:45.169 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:51:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:45.371 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:51:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: libcephfs-devel-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:51:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:51:45.371 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:45.434 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:45.543 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:45.544 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:45.544 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:45.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librados2 2023-06-10T20:51:45.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-cephfs 2023-06-10T20:51:45.977 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:46.017 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:46.858 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:13 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:46.878 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:13 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:48.141 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:48.250 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:48.251 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:48.252 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:48.259 INFO:teuthology.orchestra.run.smithi064.stdout:Package librados2-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:48.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rbd 2023-06-10T20:51:48.378 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:48.380 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:51:48.380 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:48.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install librbd1 2023-06-10T20:51:48.732 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:48.884 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:49.635 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:16 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:49.843 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:16 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:50.880 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:50.992 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:50.992 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2023-06-10T20:51:50.993 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:51.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-fuse 2023-06-10T20:51:51.245 INFO:teuthology.orchestra.run.smithi064.stdout:Package librbd1-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:51.361 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:51.362 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:51:51.362 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:51.451 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:51.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rados 2023-06-10T20:51:51.862 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:52.299 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:19 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:52.717 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:19 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:53.675 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:51:53.676 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:53.676 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2023-06-10T20:51:53.676 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:53.676 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:51:53.676 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 88 k 2023-06-10T20:51:53.677 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:53.677 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:51:53.677 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:51:53.677 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:51:53.677 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:53.678 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 88 k 2023-06-10T20:51:53.678 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 240 k 2023-06-10T20:51:53.678 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:51:53.841 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-fuse-16.2.13-257.gd8c5d349.el8.x86_64.rpm 538 kB/s | 88 kB 00:00 2023-06-10T20:51:53.842 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:51:53.843 INFO:teuthology.orchestra.run.smithi152.stdout:Total 534 kB/s | 88 kB 00:00 2023-06-10T20:51:53.843 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:51:53.852 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:51:53.852 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:51:53.906 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:51:53.909 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:51:54.014 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:51:54.114 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rados-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:54.164 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:54.235 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:54.236 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:51:54.236 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:54.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rgw 2023-06-10T20:51:54.394 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:54.757 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:55.456 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:51:55.457 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:22 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:55.601 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:22 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:57.017 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rgw-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:51:57.135 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:51:57.136 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:51:57.136 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:51:57.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-cephfs 2023-06-10T20:51:57.630 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:57.733 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:51:57.919 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:57.919 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:51:57.919 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:51:57.919 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:51:57.919 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:51:58.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-mirror 2023-06-10T20:51:58.411 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:51:58.449 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:25 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:59.240 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:26 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:51:59.881 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-cephfs-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:52:00.005 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:52:00.007 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:52:00.007 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:52:00.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install python3-rbd 2023-06-10T20:52:00.513 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:52:00.602 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:52:00.603 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:00.603 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2023-06-10T20:52:00.603 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:00.603 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.7 M 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:00.604 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 3.7 M 2023-06-10T20:52:00.605 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 15 M 2023-06-10T20:52:00.605 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:52:00.986 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-mirror-16.2.13-257.gd8c5d349.el8.x86_64.rpm 9.6 MB/s | 3.7 MB 00:00 2023-06-10T20:52:00.987 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:52:00.987 INFO:teuthology.orchestra.run.smithi152.stdout:Total 9.6 MB/s | 3.7 MB 00:00 2023-06-10T20:52:00.988 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:52:00.999 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:52:00.999 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:52:01.065 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:52:01.068 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:52:01.355 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:28 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:01.511 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:52:01.658 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:01.998 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:02.765 INFO:teuthology.orchestra.run.smithi064.stdout:Package python3-rbd-2:16.2.13-257.gd8c5d349.el8.x86_64 is already installed. 2023-06-10T20:52:02.885 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:52:02.886 INFO:teuthology.orchestra.run.smithi064.stdout:Nothing to do. 2023-06-10T20:52:02.886 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:52:03.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-fuse 2023-06-10T20:52:03.087 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:03.088 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:30 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:03.380 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:52:04.185 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:31 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:05.432 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:52:05.619 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:05.619 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:52:05.620 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:52:05.620 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:05.620 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:52:05.726 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:52:05.726 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:05.726 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-06-10T20:52:05.726 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 88 k 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:52:05.727 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:05.728 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 88 k 2023-06-10T20:52:05.728 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 240 k 2023-06-10T20:52:05.728 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:52:05.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-nbd 2023-06-10T20:52:06.026 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-fuse-16.2.13-257.gd8c5d349.el8.x86_64.rpm 295 kB/s | 88 kB 00:00 2023-06-10T20:52:06.027 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:52:06.028 INFO:teuthology.orchestra.run.smithi064.stdout:Total 294 kB/s | 88 kB 00:00 2023-06-10T20:52:06.028 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:52:06.039 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:52:06.039 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:52:06.098 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:52:06.102 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:52:06.133 INFO:teuthology.orchestra.run.smithi152.stdout:Updating Subscription Management repositories. 2023-06-10T20:52:06.216 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:52:06.365 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:06.592 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:06.943 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:33 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:07.706 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:07.706 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:34 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:08.379 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2023-06-10T20:52:08.380 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2023-06-10T20:52:08.380 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:08.380 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2023-06-10T20:52:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 173 k 2023-06-10T20:52:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:08.381 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2023-06-10T20:52:08.381 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2023-06-10T20:52:08.381 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2023-06-10T20:52:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:08.382 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 173 k 2023-06-10T20:52:08.382 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 515 k 2023-06-10T20:52:08.382 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2023-06-10T20:52:08.573 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-nbd-16.2.13-257.gd8c5d349.el8.x86_64.rpm 901 kB/s | 173 kB 00:00 2023-06-10T20:52:08.574 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:52:08.574 INFO:teuthology.orchestra.run.smithi152.stdout:Total 895 kB/s | 173 kB 00:00 2023-06-10T20:52:08.574 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2023-06-10T20:52:08.585 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2023-06-10T20:52:08.585 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2023-06-10T20:52:08.639 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2023-06-10T20:52:08.643 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2023-06-10T20:52:08.754 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2023-06-10T20:52:08.904 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:09.125 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:10.199 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:10.200 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:03:37 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:10.207 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:52:10.407 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:10.407 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:52:10.407 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-fuse-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:52:10.407 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:10.407 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:52:10.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-mirror 2023-06-10T20:52:10.947 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:52:11.895 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:38 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:12.612 INFO:teuthology.orchestra.run.smithi152.stdout:Installed products updated. 2023-06-10T20:52:12.796 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:12.797 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2023-06-10T20:52:12.797 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:52:12.797 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:52:12.797 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2023-06-10T20:52:12.923 DEBUG:teuthology.parallel:result is None 2023-06-10T20:52:13.408 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:52:13.409 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:13.409 INFO:teuthology.orchestra.run.smithi064.stdout: Package Arch Version Repo Size 2023-06-10T20:52:13.409 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:13.409 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 3.7 M 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:52:13.410 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:13.411 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 3.7 M 2023-06-10T20:52:13.411 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 15 M 2023-06-10T20:52:13.411 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:52:13.790 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-mirror-16.2.13-257.gd8c5d349.el8.x86_64.rpm 9.7 MB/s | 3.7 MB 00:00 2023-06-10T20:52:13.790 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:52:13.791 INFO:teuthology.orchestra.run.smithi064.stdout:Total 9.6 MB/s | 3.7 MB 00:00 2023-06-10T20:52:13.791 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:52:13.804 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:52:13.804 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:52:13.876 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:52:13.880 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:52:14.349 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:52:14.499 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:14.901 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:16.003 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:16.003 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:42 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:18.598 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:52:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:18.801 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:52:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-mirror-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:52:18.801 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:18.802 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:52:18.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo yum -y install rbd-nbd 2023-06-10T20:52:19.321 INFO:teuthology.orchestra.run.smithi064.stdout:Updating Subscription Management repositories. 2023-06-10T20:52:20.197 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:47 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:21.726 INFO:teuthology.orchestra.run.smithi064.stdout:Dependencies resolved. 2023-06-10T20:52:21.726 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:21.727 INFO:teuthology.orchestra.run.smithi064.stdout: Package Architecture Version Repository Size 2023-06-10T20:52:21.727 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:21.727 INFO:teuthology.orchestra.run.smithi064.stdout:Installing: 2023-06-10T20:52:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd x86_64 2:16.2.13-257.gd8c5d349.el8 ceph 173 k 2023-06-10T20:52:21.728 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:21.728 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction Summary 2023-06-10T20:52:21.728 INFO:teuthology.orchestra.run.smithi064.stdout:================================================================================ 2023-06-10T20:52:21.729 INFO:teuthology.orchestra.run.smithi064.stdout:Install 1 Package 2023-06-10T20:52:21.729 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:21.730 INFO:teuthology.orchestra.run.smithi064.stdout:Total download size: 173 k 2023-06-10T20:52:21.730 INFO:teuthology.orchestra.run.smithi064.stdout:Installed size: 515 k 2023-06-10T20:52:21.731 INFO:teuthology.orchestra.run.smithi064.stdout:Downloading Packages: 2023-06-10T20:52:21.918 INFO:teuthology.orchestra.run.smithi064.stdout:rbd-nbd-16.2.13-257.gd8c5d349.el8.x86_64.rpm 911 kB/s | 173 kB 00:00 2023-06-10T20:52:21.919 INFO:teuthology.orchestra.run.smithi064.stdout:-------------------------------------------------------------------------------- 2023-06-10T20:52:21.919 INFO:teuthology.orchestra.run.smithi064.stdout:Total 905 kB/s | 173 kB 00:00 2023-06-10T20:52:21.919 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction check 2023-06-10T20:52:21.930 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction check succeeded. 2023-06-10T20:52:21.931 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction test 2023-06-10T20:52:21.988 INFO:teuthology.orchestra.run.smithi064.stdout:Transaction test succeeded. 2023-06-10T20:52:21.992 INFO:teuthology.orchestra.run.smithi064.stdout:Running transaction 2023-06-10T20:52:22.113 INFO:teuthology.orchestra.run.smithi064.stdout: Preparing : 1/1 2023-06-10T20:52:22.287 INFO:teuthology.orchestra.run.smithi064.stdout: Installing : rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:22.517 INFO:teuthology.orchestra.run.smithi064.stdout: Running scriptlet: rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:23.634 INFO:teuthology.orchestra.run.smithi064.stdout: Verifying : rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 1/1 2023-06-10T20:52:23.634 INFO:teuthology.orchestra.run.smithi064.stdout:Last metadata expiration check: 0:03:50 ago on Sat 10 Jun 2023 08:48:33 PM UTC. 2023-06-10T20:52:26.121 INFO:teuthology.orchestra.run.smithi064.stdout:Installed products updated. 2023-06-10T20:52:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:26.322 INFO:teuthology.orchestra.run.smithi064.stdout:Installed: 2023-06-10T20:52:26.322 INFO:teuthology.orchestra.run.smithi064.stdout: rbd-nbd-2:16.2.13-257.gd8c5d349.el8.x86_64 2023-06-10T20:52:26.323 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:52:26.323 INFO:teuthology.orchestra.run.smithi064.stdout:Complete! 2023-06-10T20:52:26.485 DEBUG:teuthology.parallel:result is None 2023-06-10T20:52:26.485 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:52:26.486 INFO:teuthology.packaging:ref: None 2023-06-10T20:52:26.486 INFO:teuthology.packaging:tag: None 2023-06-10T20:52:26.486 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:52:26.486 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:52:26.487 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-06-10T20:52:26.905 DEBUG:teuthology.orchestra.run.smithi064:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-06-10T20:52:26.928 INFO:teuthology.orchestra.run.smithi064.stdout:16.2.13-257.gd8c5d349.el8 2023-06-10T20:52:26.929 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-257.gd8c5d349.el8 2023-06-10T20:52:26.929 INFO:teuthology.task.install:The correct ceph version 16.2.13-257.gd8c5d349 is installed. 2023-06-10T20:52:26.930 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-10T20:52:26.930 INFO:teuthology.packaging:ref: None 2023-06-10T20:52:26.930 INFO:teuthology.packaging:tag: None 2023-06-10T20:52:26.931 INFO:teuthology.packaging:branch: pacific 2023-06-10T20:52:26.931 INFO:teuthology.packaging:sha1: 5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T20:52:26.931 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=pacific 2023-06-10T20:52:27.307 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-06-10T20:52:27.336 INFO:teuthology.orchestra.run.smithi152.stdout:16.2.13-257.gd8c5d349.el8 2023-06-10T20:52:27.336 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-257.gd8c5d349.el8 2023-06-10T20:52:27.336 INFO:teuthology.task.install:The correct ceph version 16.2.13-257.gd8c5d349 is installed. 2023-06-10T20:52:27.337 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-06-10T20:52:27.337 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:52:27.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-06-10T20:52:27.369 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:52:27.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-06-10T20:52:27.409 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-06-10T20:52:27.474 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:52:27.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/daemon-helper 2023-06-10T20:52:27.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-06-10T20:52:27.572 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:52:27.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/daemon-helper 2023-06-10T20:52:27.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-06-10T20:52:27.675 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-06-10T20:52:27.676 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:52:27.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/adjust-ulimits 2023-06-10T20:52:27.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-06-10T20:52:27.780 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:52:27.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/adjust-ulimits 2023-06-10T20:52:27.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-06-10T20:52:27.880 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-06-10T20:52:27.880 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:52:27.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/usr/bin/stdin-killer 2023-06-10T20:52:27.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-06-10T20:52:27.981 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:52:27.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/stdin-killer 2023-06-10T20:52:28.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-06-10T20:52:28.078 INFO:teuthology.run_tasks:Running task cephadm... 2023-06-10T20:52:28.404 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'} 2023-06-10T20:52:28.405 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-06-10T20:52:28.471 INFO:tasks.cephadm:Cluster fsid is b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:52:28.472 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-06-10T20:52:28.472 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.64', 'mon.c': '[v2:172.21.15.64:3301,v1:172.21.15.64:6790]', 'mon.b': '172.21.15.152'} 2023-06-10T20:52:28.473 INFO:tasks.cephadm:First mon is mon.a on smithi064 2023-06-10T20:52:28.473 INFO:tasks.cephadm:First mgr is y 2023-06-10T20:52:28.473 INFO:tasks.cephadm:Normalizing hostnames... 2023-06-10T20:52:28.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo hostname $(hostname -s) 2023-06-10T20:52:28.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo hostname $(hostname -s) 2023-06-10T20:52:28.534 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-06-10T20:52:28.534 DEBUG:teuthology.orchestra.run.smithi064:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:28.702 INFO:teuthology.orchestra.run.smithi064.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Jun 10 20:52 /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:28.703 DEBUG:teuthology.orchestra.run.smithi152:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:28.954 INFO:teuthology.orchestra.run.smithi152.stdout:-rw-rw-r--. 1 ubuntu ubuntu 350873 Jun 10 20:52 /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:28.954 DEBUG:teuthology.orchestra.run.smithi064:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:28.976 DEBUG:teuthology.orchestra.run.smithi152:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-06-10T20:52:29.005 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-06-10T20:52:29.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-06-10T20:52:29.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-06-10T20:52:30.157 INFO:teuthology.orchestra.run.smithi152.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-10T20:52:30.197 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-10T20:53:07.036 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T20:53:07.037 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph_version": "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)", 2023-06-10T20:53:07.037 INFO:teuthology.orchestra.run.smithi064.stdout: "image_id": "b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c", 2023-06-10T20:53:07.037 INFO:teuthology.orchestra.run.smithi064.stdout: "repo_digests": [ 2023-06-10T20:53:07.037 INFO:teuthology.orchestra.run.smithi064.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:b41db2dcff2a7a7fc551b7b1b28c0ae195aaddc42f719f7bcd571090f6080399" 2023-06-10T20:53:07.037 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T20:53:07.038 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T20:53:07.681 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T20:53:07.682 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph_version": "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)", 2023-06-10T20:53:07.682 INFO:teuthology.orchestra.run.smithi152.stdout: "image_id": "b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c", 2023-06-10T20:53:07.682 INFO:teuthology.orchestra.run.smithi152.stdout: "repo_digests": [ 2023-06-10T20:53:07.682 INFO:teuthology.orchestra.run.smithi152.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:b41db2dcff2a7a7fc551b7b1b28c0ae195aaddc42f719f7bcd571090f6080399" 2023-06-10T20:53:07.682 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T20:53:07.683 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T20:53:07.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo mkdir -p /etc/ceph 2023-06-10T20:53:07.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo mkdir -p /etc/ceph 2023-06-10T20:53:07.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 777 /etc/ceph 2023-06-10T20:53:07.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 777 /etc/ceph 2023-06-10T20:53:07.849 INFO:tasks.cephadm:Writing seed config... 2023-06-10T20:53:07.853 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-06-10T20:53:07.854 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-06-10T20:53:07.854 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-06-10T20:53:07.855 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-06-10T20:53:07.855 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-06-10T20:53:07.855 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-06-10T20:53:07.856 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-06-10T20:53:07.856 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-06-10T20:53:07.857 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-06-10T20:53:07.857 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-06-10T20:53:07.858 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-06-10T20:53:07.858 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:53:07.859 DEBUG:teuthology.orchestra.run.smithi064:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-06-10T20:53:07.878 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b618768e-07d0-11ee-9b1e-001a4aab830c mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-06-10T20:53:07.878 DEBUG:teuthology.orchestra.run.smithi064:mon.a> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service 2023-06-10T20:53:07.920 DEBUG:teuthology.orchestra.run.smithi064:mgr.y> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service 2023-06-10T20:53:07.950 INFO:journalctl@ceph.mon.a.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:53:07.951 INFO:tasks.cephadm:Bootstrapping... 2023-06-10T20:53:07.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid b618768e-07d0-11ee-9b1e-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.64 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-06-10T20:53:07.979 INFO:journalctl@ceph.mgr.y.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:53:08.165 INFO:teuthology.orchestra.run.smithi064.stderr:-------------------------------------------------------------------------------- 2023-06-10T20:53:08.165 INFO:teuthology.orchestra.run.smithi064.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', 'b618768e-07d0-11ee-9b1e-001a4aab830c', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.64', '--skip-admin-label'] 2023-06-10T20:53:08.212 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: 3.0.1 2023-06-10T20:53:08.233 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying podman|docker is present... 2023-06-10T20:53:08.278 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: 3.0.1 2023-06-10T20:53:08.295 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying lvm2 is present... 2023-06-10T20:53:08.296 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying time synchronization is in place... 2023-06-10T20:53:08.302 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-06-10T20:53:08.308 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:53:08.314 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: enabled 2023-06-10T20:53:08.319 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: active 2023-06-10T20:53:08.320 INFO:teuthology.orchestra.run.smithi064.stderr:Unit chronyd.service is enabled and running 2023-06-10T20:53:08.320 INFO:teuthology.orchestra.run.smithi064.stderr:Repeating the final host check... 2023-06-10T20:53:08.365 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: 3.0.1 2023-06-10T20:53:08.382 INFO:teuthology.orchestra.run.smithi064.stderr:podman (/bin/podman) version 3.0.1 is present 2023-06-10T20:53:08.383 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl is present 2023-06-10T20:53:08.383 INFO:teuthology.orchestra.run.smithi064.stderr:lvcreate is present 2023-06-10T20:53:08.388 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-06-10T20:53:08.394 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:53:08.400 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: enabled 2023-06-10T20:53:08.406 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: active 2023-06-10T20:53:08.406 INFO:teuthology.orchestra.run.smithi064.stderr:Unit chronyd.service is enabled and running 2023-06-10T20:53:08.406 INFO:teuthology.orchestra.run.smithi064.stderr:Host looks OK 2023-06-10T20:53:08.407 INFO:teuthology.orchestra.run.smithi064.stderr:Cluster fsid: b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:08.407 INFO:teuthology.orchestra.run.smithi064.stderr:Acquiring lock 140085174802920 on /run/cephadm/b618768e-07d0-11ee-9b1e-001a4aab830c.lock 2023-06-10T20:53:08.407 INFO:teuthology.orchestra.run.smithi064.stderr:Lock 140085174802920 acquired on /run/cephadm/b618768e-07d0-11ee-9b1e-001a4aab830c.lock 2023-06-10T20:53:08.407 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying IP 172.21.15.64 port 3300 ... 2023-06-10T20:53:08.408 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying IP 172.21.15.64 port 6789 ... 2023-06-10T20:53:08.408 INFO:teuthology.orchestra.run.smithi064.stderr:Base mon IP(s) is [172.21.15.64:3300, 172.21.15.64:6789], mon addrv is [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2023-06-10T20:53:08.411 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2023-06-10T20:53:08.411 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.64 metric 100 2023-06-10T20:53:08.413 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-06-10T20:53:08.414 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2023-06-10T20:53:08.414 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2023-06-10T20:53:08.416 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-06-10T20:53:08.416 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-06-10T20:53:08.416 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-06-10T20:53:08.416 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-06-10T20:53:08.417 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: inet6 fe80::bbc4:c6c:5d6f:21f2/64 scope link noprefixroute 2023-06-10T20:53:08.417 INFO:teuthology.orchestra.run.smithi064.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-06-10T20:53:08.417 INFO:teuthology.orchestra.run.smithi064.stderr:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2023-06-10T20:53:08.417 INFO:teuthology.orchestra.run.smithi064.stderr:Mon IP `172.21.15.64` is in CIDR network `172.21.0.0/20` 2023-06-10T20:53:08.417 INFO:teuthology.orchestra.run.smithi064.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-06-10T20:53:08.418 INFO:teuthology.orchestra.run.smithi064.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-06-10T20:53:08.419 INFO:teuthology.orchestra.run.smithi064.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-10T20:53:08.491 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Trying to pull quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-10T20:53:08.578 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Getting image source signatures 2023-06-10T20:53:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Copying blob sha256:20efa0c35fe33bdc6d53f63557b6ea7c60ae97ba8b29b9716b0a3b4544cd92d9 2023-06-10T20:53:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Copying blob sha256:b0bc32d942d2d315b1d05cce39b77b5565809cf3c4a89a52484008ff9c3c55b5 2023-06-10T20:53:08.579 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Copying config sha256:b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c 2023-06-10T20:53:08.634 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Writing manifest to image destination 2023-06-10T20:53:08.634 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: Storing signatures 2023-06-10T20:53:08.811 INFO:teuthology.orchestra.run.smithi064.stderr:/bin/podman: b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c 2023-06-10T20:53:09.305 INFO:teuthology.orchestra.run.smithi064.stderr:ceph: ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable) 2023-06-10T20:53:09.785 INFO:teuthology.orchestra.run.smithi064.stderr:Ceph version: ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable) 2023-06-10T20:53:09.785 INFO:teuthology.orchestra.run.smithi064.stderr:Extracting ceph user uid/gid from container image... 2023-06-10T20:53:10.195 INFO:teuthology.orchestra.run.smithi064.stderr:stat: 167 167 2023-06-10T20:53:10.638 INFO:teuthology.orchestra.run.smithi064.stderr:Creating initial keys... 2023-06-10T20:53:11.011 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-authtool: AQC34oRkiHpQABAAT6nFba4nd7/zwPAwAVgU9A== 2023-06-10T20:53:11.836 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-authtool: AQC34oRkauNTMRAAbWFI5EXhxujbLRjR9N10/Q== 2023-06-10T20:53:12.694 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-authtool: AQC44oRk2HHFKBAA9Gnj3TqT0hoR85/mSWi8rQ== 2023-06-10T20:53:13.147 INFO:teuthology.orchestra.run.smithi064.stderr:Creating initial monmap... 2023-06-10T20:53:13.530 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-06-10T20:53:13.530 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:13.530 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-06-10T20:53:14.044 INFO:teuthology.orchestra.run.smithi064.stderr:monmaptool for a [v2:172.21.15.64:3300,v1:172.21.15.64:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-06-10T20:53:14.044 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/monmaptool: set fsid to b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:14.045 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-06-10T20:53:14.045 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:53:14.045 INFO:teuthology.orchestra.run.smithi064.stderr:Creating mon... 2023-06-10T20:53:14.409 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.387+0000 7f937b324880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-10T20:53:14.409 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.387+0000 7f937b324880 1 imported monmap: 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: epoch 0 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: last_changed 2023-06-10T20:53:13.526882+0000 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: created 2023-06-10T20:53:13.526882+0000 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-06-10T20:53:14.410 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon.a 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.387+0000 7f937b324880 0 /usr/bin/ceph-mon: set fsid to b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: RocksDB version: 6.8.1 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Compile date May 28 2023 2023-06-10T20:53:14.411 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: DB SUMMARY 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.error_if_exists: 0 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.create_if_missing: 1 2023-06-10T20:53:14.412 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.paranoid_checks: 1 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.env: 0x55f5b4fd4080 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.fs: Posix File System 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.info_log: 0x55f5b6eaba60 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.statistics: (nil) 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.use_fsync: 0 2023-06-10T20:53:14.413 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_log_file_size: 0 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.allow_fallocate: 1 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-10T20:53:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.use_direct_reads: 0 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.db_log_dir: 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_subcompactions: 1 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_background_flushes: -1 2023-06-10T20:53:14.415 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-10T20:53:14.416 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.write_buffer_manager: 0x55f5b6eb3770 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-10T20:53:14.417 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.unordered_write: 0 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.row_cache: None 2023-06-10T20:53:14.418 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.wal_filter: None 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.preserve_deletes: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.two_write_queues: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.atomic_flush: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T20:53:14.419 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.log_readahead_size: 0 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_background_jobs: 2 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_background_compactions: -1 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T20:53:14.420 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.max_open_files: -1 2023-06-10T20:53:14.421 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Compression algorithms supported: 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kZSTD supported: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kXpressCompression supported: 0 2023-06-10T20:53:14.422 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kLZ4Compression supported: 1 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kBZip2Compression supported: 0 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kZlibCompression supported: 1 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: kSnappyCompression supported: 1 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.388+0000 7f937b324880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.423 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.merge_operator: 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_filter: None 2023-06-10T20:53:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f5b6dbdd20) 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T20:53:14.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: index_type: 0 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: checksum: 1 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-06-10T20:53:14.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_cache: 0x55f5b6df6d10 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-06-10T20:53:14.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-06-10T20:53:14.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: format_version: 2 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-06-10T20:53:14.429 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: block_align: 0 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression: NoCompression 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-10T20:53:14.430 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.num_levels: 7 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T20:53:14.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:53:14.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T20:53:14.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T20:53:14.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-10T20:53:14.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T20:53:14.436 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.table_properties_collectors: 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.inplace_update_support: 0 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T20:53:14.437 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.bloom_locality: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.max_successive_merges: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-10T20:53:14.438 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.ttl: 2592000 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.389+0000 7f937b324880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.390+0000 7f937b324880 4 rocksdb: DB pointer 0x55f5b6ec1800 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.390+0000 7f93641af700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-10T20:53:14.439 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.390+0000 7f93641af700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T20:53:14.440 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:53:14.441 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:53:14.442 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.443 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:14.444 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:14.445 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.392+0000 7f937b324880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.392+0000 7f937b324880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-06-10T20:53:14.446 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph-mon: debug 2023-06-10T20:53:14.392+0000 7f937b324880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-06-10T20:53:14.905 INFO:teuthology.orchestra.run.smithi064.stderr:create mon.a on 2023-06-10T20:53:14.920 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-06-10T20:53:15.021 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-06-10T20:53:15.120 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target → /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target. 2023-06-10T20:53:15.120 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target → /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target. 2023-06-10T20:53:15.334 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Failed to reset failed state of unit ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service: Unit ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service not loaded. 2023-06-10T20:53:15.341 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target.wants/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service → /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@.service. 2023-06-10T20:53:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 systemd[1]: Starting Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:53:15.976 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: disabled 2023-06-10T20:53:15.984 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:53:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:firewalld.service is not enabled 2023-06-10T20:53:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:Not possible to enable service . firewalld.service is not available 2023-06-10T20:53:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mon to start... 2023-06-10T20:53:15.985 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mon... 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 bash[96187]: c4b01bfb9c7872348f9d753256b1c6e2533a7a6d63192da85af23a6722d0dac0 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 systemd[1]: Started Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.981364+0000 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 conmon[96228]: mon.a (mon.0) 0 : [INF] mkfs b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 conmon[96228]: cluster 2023-06-10T20 2023-06-10T20:53:16.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:15 smithi064 conmon[96228]: :53:15.974964+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-10T20:53:16.549 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: cluster: 2023-06-10T20:53:16.549 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: id: b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: services: 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.565187s) 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mgr: no daemons active 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-06-10T20:53:16.550 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: data: 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: pgs: 2023-06-10T20:53:16.551 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:17.254 INFO:teuthology.orchestra.run.smithi064.stderr:mon is available 2023-06-10T20:53:17.254 INFO:teuthology.orchestra.run.smithi064.stderr:Assimilating anything we can from ceph.conf... 2023-06-10T20:53:17.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.979790+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-10T20:53:17.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.980241+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} 2023-06-10T20:53:17.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.983623+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-06-10T20:53:17.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.991320+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-06-10T20:53:17.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: cluster 2023-06-10T20:53:15.991967+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-06-10T20:53:17.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: audit 2023-06-10T20:53:16.544933+0000 mon.a (mon.0 2023-06-10T20:53:17.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:16 smithi064 conmon[96228]: ) 7 : audit [DBG] from='client.? 172.21.15.64:0/3843133642' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-06-10T20:53:17.823 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [global] 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: fsid = b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: log_to_journald = false 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.64:3300,v1:172.21.15.64:6789] 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-06-10T20:53:17.824 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [mgr] 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [osd] 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-06-10T20:53:17.825 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-06-10T20:53:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:17 smithi064 conmon[96228]: audit 2023-06-10T20:53:17.816218+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.64:0/1760981054' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-06-10T20:53:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:17 smithi064 conmon[96228]: audit 2023-06-10T20:53:17.819286+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.64:0/1760981054' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-06-10T20:53:18.321 INFO:teuthology.orchestra.run.smithi064.stderr:Generating new minimal ceph.conf... 2023-06-10T20:53:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:18 smithi064 conmon[96228]: audit 2023-06-10T20:53:18.916438+0000 mon.a 2023-06-10T20:53:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:18 smithi064 conmon[96228]: (mon.0) 10 : audit [DBG] from='client.? 172.21.15.64:0/1713584703' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:53:19.513 INFO:teuthology.orchestra.run.smithi064.stderr:Restarting the monitor... 2023-06-10T20:53:19.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:19 smithi064 systemd[1]: Stopping Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:53:19.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:19 smithi064 bash[96738]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-mon.a found: no such container 2023-06-10T20:53:19.812 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:19 smithi064 conmon[96228]: debug 2023-06-10T20:53:19.807+0000 7fa75de94700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-10T20:53:20.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:19 smithi064 conmon[96228]: debug 2023-06-10T20:53:19.807+0000 7fa75de94700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-06-10T20:53:20.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 bash[96853]: c4b01bfb9c7872348f9d753256b1c6e2533a7a6d63192da85af23a6722d0dac0 2023-06-10T20:53:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service: Succeeded. 2023-06-10T20:53:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 systemd[1]: Stopped Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:53:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 systemd[1]: Starting Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:53:20.924 INFO:teuthology.orchestra.run.smithi064.stderr:Setting mon public_network to 172.21.0.0/20 2023-06-10T20:53:21.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.910+0000 7f7caf0d1880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.910+0000 7f7caf0d1880 0 ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable), process ceph-mon, pid 7 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.910+0000 7f7caf0d1880 0 pidfile_write: ignore empty --pid-file 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 bash[97184]: 27c52af4dbabdc27e40383bf71b2aee38f38993973229eb6a4ebc8d295de4232 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.914+0000 7f7caf0d1880 0 load: jerasure load: lrc load: isa 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: RocksDB version: 6.8.1 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Compile date May 28 2023 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: DB SUMMARY 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: CURRENT file: CURRENT 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: IDENTITY file: IDENTITY 2023-06-10T20:53:21.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86675 ; 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.error_if_exists: 0 2023-06-10T20:53:21.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.create_if_missing: 0 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.paranoid_checks: 1 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.env: 0x55ba9b11a080 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.fs: Posix File System 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.info_log: 0x55ba9d095960 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.statistics: (nil) 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.use_fsync: 0 2023-06-10T20:53:21.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_log_file_size: 0 2023-06-10T20:53:21.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T20:53:21.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T20:53:21.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-10T20:53:21.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-10T20:53:21.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.allow_fallocate: 1 2023-06-10T20:53:21.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.use_direct_reads: 0 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.db_log_dir: 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-06-10T20:53:21.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_subcompactions: 1 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_background_flushes: -1 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T20:53:21.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.write_buffer_manager: 0x55ba9dd90210 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T20:53:21.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-10T20:53:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T20:53:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-10T20:53:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-10T20:53:21.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-10T20:53:21.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.unordered_write: 0 2023-06-10T20:53:21.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T20:53:21.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T20:53:21.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T20:53:21.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T20:53:21.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.row_cache: None 2023-06-10T20:53:21.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.wal_filter: None 2023-06-10T20:53:21.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T20:53:21.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-10T20:53:21.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.preserve_deletes: 0 2023-06-10T20:53:21.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.two_write_queues: 0 2023-06-10T20:53:21.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-10T20:53:21.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.atomic_flush: 0 2023-06-10T20:53:21.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T20:53:21.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T20:53:21.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T20:53:21.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.log_readahead_size: 0 2023-06-10T20:53:21.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-10T20:53:21.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_background_jobs: 2 2023-06-10T20:53:21.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_background_compactions: -1 2023-06-10T20:53:21.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T20:53:21.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T20:53:21.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T20:53:21.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-10T20:53:21.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T20:53:21.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T20:53:21.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T20:53:21.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T20:53:21.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_open_files: -1 2023-06-10T20:53:21.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-10T20:53:21.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T20:53:21.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T20:53:21.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-10T20:53:21.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Compression algorithms supported: 2023-06-10T20:53:21.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T20:53:21.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kZSTD supported: 0 2023-06-10T20:53:21.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kXpressCompression supported: 0 2023-06-10T20:53:21.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-10T20:53:21.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kLZ4Compression supported: 1 2023-06-10T20:53:21.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kBZip2Compression supported: 0 2023-06-10T20:53:21.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kZlibCompression supported: 1 2023-06-10T20:53:21.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: kSnappyCompression supported: 1 2023-06-10T20:53:21.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T20:53:21.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-06-10T20:53:21.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.208 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-10T20:53:21.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T20:53:21.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.merge_operator: 2023-06-10T20:53:21.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_filter: None 2023-06-10T20:53:21.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-10T20:53:21.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T20:53:21.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T20:53:21.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ba9d03fd18) 2023-06-10T20:53:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cache_index_and_filter_blocks: 1 2023-06-10T20:53:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T20:53:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T20:53:21.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: pin_top_level_index_and_filter: 1 2023-06-10T20:53:21.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: index_type: 0 2023-06-10T20:53:21.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: data_block_index_type: 0 2023-06-10T20:53:21.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: index_shortening: 1 2023-06-10T20:53:21.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: data_block_hash_table_util_ratio: 0.750000 2023-06-10T20:53:21.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: hash_index_allow_collision: 1 2023-06-10T20:53:21.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: checksum: 1 2023-06-10T20:53:21.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: no_block_cache: 0 2023-06-10T20:53:21.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_cache: 0x55ba9d078f10 2023-06-10T20:53:21.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_cache_name: BinnedLRUCache 2023-06-10T20:53:21.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_cache_options: 2023-06-10T20:53:21.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: capacity : 536870912 2023-06-10T20:53:21.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: num_shard_bits : 4 2023-06-10T20:53:21.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: strict_capacity_limit : 0 2023-06-10T20:53:21.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: high_pri_pool_ratio: 0.000 2023-06-10T20:53:21.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_cache_compressed: (nil) 2023-06-10T20:53:21.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: persistent_cache: (nil) 2023-06-10T20:53:21.216 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_size: 4096 2023-06-10T20:53:21.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_size_deviation: 10 2023-06-10T20:53:21.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_restart_interval: 16 2023-06-10T20:53:21.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: index_block_restart_interval: 1 2023-06-10T20:53:21.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: metadata_block_size: 4096 2023-06-10T20:53:21.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: partition_filters: 0 2023-06-10T20:53:21.218 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: use_delta_encoding: 1 2023-06-10T20:53:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-10T20:53:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: whole_key_filtering: 1 2023-06-10T20:53:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: verify_compression: 0 2023-06-10T20:53:21.219 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: read_amp_bytes_per_bit: 0 2023-06-10T20:53:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: format_version: 2 2023-06-10T20:53:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: enable_index_compression: 1 2023-06-10T20:53:21.220 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: block_align: 0 2023-06-10T20:53:21.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-10T20:53:21.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-10T20:53:21.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression: NoCompression 2023-06-10T20:53:21.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-10T20:53:21.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-10T20:53:21.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T20:53:21.222 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.num_levels: 7 2023-06-10T20:53:21.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T20:53:21.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T20:53:21.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T20:53:21.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T20:53:21.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T20:53:21.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T20:53:21.224 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T20:53:21.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-10T20:53:21.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T20:53:21.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T20:53:21.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T20:53:21.227 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T20:53:21.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T20:53:21.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T20:53:21.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T20:53:21.228 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T20:53:21.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_mu 2023-06-10T20:53:21.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ltiplier_addtl[4]: 1 2023-06-10T20:53:21.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T20:53:21.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-10T20:53:21.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T20:53:21.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.table_properties_collectors: 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.inplace_update_support: 0 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T20:53:21.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.bloom_locality: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.max_successive_merges: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-10T20:53:21.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.ttl: 2592000 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.915+0000 7f7caf0d1880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.916+0000 7f7caf0d1880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.916+0000 7f7caf0d1880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.916+0000 7f7caf0d1880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430400917913, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-06-10T20:53:21.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.916+0000 7f7caf0d1880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.916+0000 7f7caf0d1880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.917+0000 7f7caf0d1880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430400918812, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83275, "table_properties": {"data_size": 81548, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75869, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1686430400, "oldest_key_time": 3, "file_creation_time": 0}} 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.917+0000 7f7caf0d1880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.917+0000 7f7caf0d1880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430400919202, "job": 1, "event": "recovery_finished"} 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.918+0000 7f7caf0d1880 4 rocksdb: DB pointer 0x55ba9d143800 2023-06-10T20:53:21.235 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.918+0000 7f7c95edc700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.918+0000 7f7c95edc700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** DB Stats ** 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T20:53:21.236 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** Compaction Stats [default] ** 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** Compaction Stats [default] ** 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:21.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Cumulative compaction: 0.00 GB write, 27.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Interval compaction: 0.00 GB write, 27.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:53:21.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** Compaction Stats [default] ** 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: L0 2/0 82.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Sum 2/0 82.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** Compaction Stats [default] ** 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.8 0.00 0.00 1 0.001 0 0 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:53:21.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Cumulative compaction: 0.00 GB write, 27.25 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:53:21.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 systemd[1]: Started Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.919+0000 7f7caf0d1880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.920+0000 7f7caf0d1880 1 mon.a@-1(???) e1 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.920+0000 7f7caf0d1880 0 mon.a@-1(???).mds e1 new map 2023-06-10T20:53:21.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.920+0000 7f7caf0d1880 0 mon.a@-1(???).mds e1 print_map 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: e1 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: legacy client fscid: -1 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: No filesystems configured 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.921+0000 7f7caf0d1880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.921+0000 7f7caf0d1880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.921+0000 7f7caf0d1880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.921+0000 7f7caf0d1880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:53:21.245 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: -10T20:53:18.916438+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.64:0/1713584703' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: debug 2023-06-10T20:53:20.921+0000 7f7caf0d1880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: T20:53:20.927616+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cluster 2023-06-10T20:53:20.927700+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0]} removed_ranks: {} 2023-06-10T20:53:21.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cluster 2023-06-10T20:53:20.928126+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-06-10T20:53:21.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cluster 2023-06-10T20:53:20.928168+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-06-10T20:53:21.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:20 smithi064 conmon[97224]: cluster 2023-06-10T20:53:20.929601+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-06-10T20:53:22.129 INFO:teuthology.orchestra.run.smithi064.stderr:Wrote config to /etc/ceph/ceph.conf 2023-06-10T20:53:22.129 INFO:teuthology.orchestra.run.smithi064.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-06-10T20:53:22.129 INFO:teuthology.orchestra.run.smithi064.stderr:Creating mgr... 2023-06-10T20:53:22.130 INFO:teuthology.orchestra.run.smithi064.stderr:Verifying port 9283 ... 2023-06-10T20:53:22.260 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Failed to reset failed state of unit ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service: Unit ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service not loaded. 2023-06-10T20:53:22.266 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: Created symlink /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c.target.wants/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service → /etc/systemd/system/ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@.service. 2023-06-10T20:53:22.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:22 smithi064 systemd[1]: Starting Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:53:22.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:22 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:53:22.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:22 smithi064 conmon[97224]: :53:21.533115+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.64:0/3882513948' entity='client.admin' 2023-06-10T20:53:22.976 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: disabled 2023-06-10T20:53:22.983 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:53:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:firewalld.service is not enabled 2023-06-10T20:53:22.984 INFO:teuthology.orchestra.run.smithi064.stderr:Not possible to enable service . firewalld.service is not available 2023-06-10T20:53:22.992 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: disabled 2023-06-10T20:53:22.999 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:53:23.000 INFO:teuthology.orchestra.run.smithi064.stderr:firewalld.service is not enabled 2023-06-10T20:53:23.000 INFO:teuthology.orchestra.run.smithi064.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-06-10T20:53:23.001 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mgr to start... 2023-06-10T20:53:23.001 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mgr... 2023-06-10T20:53:23.225 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:22 smithi064 bash[97635]: 081c9920c7713501fd0d72bddad5889e532d8b49cfaf0cfdb39f6a521da36d9f 2023-06-10T20:53:23.225 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:22 smithi064 systemd[1]: Started Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:53:23.225 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:23 smithi064 conmon[97677]: debug 2023-06-10T20:53:23.102+0000 7f1473380680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:53:23.225 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:23 smithi064 conmon[97677]: debug 2023-06-10T20:53:23.203+0000 7f1473380680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:53:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsid": "b618768e-07d0-11ee-9b1e-001a4aab830c", 2023-06-10T20:53:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "health": { 2023-06-10T20:53:23.622 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "checks": {}, 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mutes": [] 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum": [ 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 0 2023-06-10T20:53:23.623 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "a" 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "monmap": { 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:23.624 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osdmap": { 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-10T20:53:23.625 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgmap": { 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-10T20:53:23.626 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.627 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsmap": { 2023-06-10T20:53:23.628 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:23.628 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-10T20:53:23.628 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-10T20:53:23.628 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.628 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-10T20:53:23.629 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": false, 2023-06-10T20:53:23.629 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-10T20:53:23.629 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modules": [ 2023-06-10T20:53:23.629 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "iostat", 2023-06-10T20:53:23.630 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "nfs", 2023-06-10T20:53:23.630 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "restful" 2023-06-10T20:53:23.630 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:23.630 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:23.630 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "servicemap": { 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modified": "2023-06-10T20:53:15.981935+0000", 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-10T20:53:23.631 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:23.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:23 smithi064 conmon[97677]: debug 2023-06-10T20:53:23.668+0000 7f1473380680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:53:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:23 smithi064 conmon[97224]: audit 2023-06-10T20:53:23.613649+0000 2023-06-10T20:53:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:23 smithi064 conmon[97224]: mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.64:0/961742666' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-10T20:53:24.211 INFO:teuthology.orchestra.run.smithi064.stderr:mgr not available, waiting (1/15)... 2023-06-10T20:53:24.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:24 smithi064 conmon[97677]: debug 2023-06-10T20:53:24.648+0000 7f1473380680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:53:24.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:24 smithi064 conmon[97677]: debug 2023-06-10T20:53:24.744+0000 7f1473380680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:53:25.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:24 smithi064 conmon[97677]: debug 2023-06-10T20:53:24.990+0000 7f1473380680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:53:26.398 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97677]: debug 2023-06-10T20:53:26.129+0000 7f1473380680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:53:26.398 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97677]: debug 2023-06-10T20:53:26.298+0000 7f1473380680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:53:26.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97677]: debug 2023-06-10T20:53:26.394+0000 7f1473380680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:53:26.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97677]: debug 2023-06-10T20:53:26.508+0000 7f1473380680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:53:26.839 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsid": "b618768e-07d0-11ee-9b1e-001a4aab830c", 2023-06-10T20:53:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "health": { 2023-06-10T20:53:26.841 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "checks": {}, 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mutes": [] 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum": [ 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 0 2023-06-10T20:53:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:26.843 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-10T20:53:26.843 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "a" 2023-06-10T20:53:26.843 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:26.843 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_age": 5, 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "monmap": { 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osdmap": { 2023-06-10T20:53:26.844 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:26.845 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-10T20:53:26.845 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-10T20:53:26.845 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-10T20:53:26.847 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-10T20:53:26.847 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgmap": { 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-10T20:53:26.848 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsmap": { 2023-06-10T20:53:26.849 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": false, 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-10T20:53:26.850 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modules": [ 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "iostat", 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "nfs", 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "restful" 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "servicemap": { 2023-06-10T20:53:26.851 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:26.852 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modified": "2023-06-10T20:53:15.981935+0000", 2023-06-10T20:53:26.852 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:26.852 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:26.852 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-10T20:53:26.852 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:27.043 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97677]: debug 2023-06-10T20:53:26.730+0000 7f1473380680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:53:27.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97224]: audit 2023-06-10T20:53:26.835532+0000 mon.a (mon 2023-06-10T20:53:27.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:26 smithi064 conmon[97224]: .0) 8 : audit [DBG] from='client.? 172.21.15.64:0/3050040993' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-10T20:53:27.336 INFO:teuthology.orchestra.run.smithi064.stderr:mgr not available, waiting (2/15)... 2023-06-10T20:53:27.341 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:27 smithi064 conmon[97677]: debug 2023-06-10T20:53:27.102+0000 7f1473380680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:53:27.342 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:27 smithi064 conmon[97677]: debug 2023-06-10T20:53:27.220+0000 7f1473380680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:53:28.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.093+0000 7f1473380680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:53:28.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.191+0000 7f1473380680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:53:28.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.301+0000 7f1473380680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:53:28.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.511+0000 7f1473380680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:53:28.801 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.797+0000 7f1473380680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:53:29.128 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:28 smithi064 conmon[97677]: debug 2023-06-10T20:53:28.957+0000 7f1473380680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:53:29.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97677]: debug 2023-06-10T20:53:29.124+0000 7f1473380680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:53:29.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97677]: debug 2023-06-10T20:53:29.338+0000 7f1473380680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:53:29.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97677]: debug 2023-06-10T20:53:29.446+0000 7f1473380680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:53:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: cluster 2023-06-10T20:53: 2023-06-10T20:53:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 29.459261+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-06-10T20:53:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: cluster 2023-06-10T20:53:29. 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 461986+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00280548s) 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 10T20:53:29.463119+0000 mon.a 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53: 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 29.463340+0000 mon.a (mon.0) 12 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: : audit [DBG] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: T20:53:29.463482+0000 mon.a ( 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20: 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 53:29.463706+0000 mon.a (mon.0 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: ) 14 : audit [DBG] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 10T20:53:29.464086+0000 mon.a ( 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 06-10T20:53:29.478405+0000 2023-06-10T20:53:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: mon.a (mon.0) 16 : cluster [INF] 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: Manager daemon y is now available 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53:29. 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 489524+0000 mon.a (mon.0) 17 : audit 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: [INF] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:53:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: :53:29.494099+0000 mon.a (mon. 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 0) 18 : audit [INF] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53:29. 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 495547+0000 mon.a (mon.0) 19 : audit 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: [INF] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 53:29.501595+0000 mon.a (mon.0 2023-06-10T20:53:29.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: ) 20 : audit [INF] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' 2023-06-10T20:53:29.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 2023-06-10T20:53 2023-06-10T20:53:29.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: :29.506888+0000 mon.a (mon.0) 2023-06-10T20:53:29.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:29 smithi064 conmon[97224]: 21 : audit [INF] from='mgr.14100 172.21.15.64:0/4047457413' entity='mgr.y' 2023-06-10T20:53:29.916 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:29.916 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsid": "b618768e-07d0-11ee-9b1e-001a4aab830c", 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "health": { 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "checks": {}, 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mutes": [] 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-10T20:53:29.917 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum": [ 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 0 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "a" 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_age": 8, 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "monmap": { 2023-06-10T20:53:29.918 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osdmap": { 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-10T20:53:29.919 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-10T20:53:29.922 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-10T20:53:29.922 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-10T20:53:29.922 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-10T20:53:29.922 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.922 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgmap": { 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-10T20:53:29.923 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsmap": { 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.924 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": false, 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modules": [ 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "iostat", 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "nfs", 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "restful" 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:29.925 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:29.926 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.926 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "servicemap": { 2023-06-10T20:53:29.926 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:29.926 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modified": "2023-06-10T20:53:15.981935+0000", 2023-06-10T20:53:29.926 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:29.927 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:29.927 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-10T20:53:29.927 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:30.469 INFO:teuthology.orchestra.run.smithi064.stderr:mgr not available, waiting (3/15)... 2023-06-10T20:53:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:30 smithi064 conmon[97224]: audit 2023-06-10T20:53:29.909797+0000 mon.a (mon.0) 22 : audit [DBG] from='client.? 172.21.15.64:0/1239648732' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-10T20:53:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:30 smithi064 conmon[97224]: cluster 2023-06-10T20:53:30.464975+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.00579s) 2023-06-10T20:53:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:32 smithi064 conmon[97224]: cluster 2023-06-10T20:53:31.929471+0000 mon.a ( 2023-06-10T20:53:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:32 smithi064 conmon[97224]: mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-06-10T20:53:33.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:33.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:33.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsid": "b618768e-07d0-11ee-9b1e-001a4aab830c", 2023-06-10T20:53:33.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "health": { 2023-06-10T20:53:33.425 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "checks": {}, 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mutes": [] 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum": [ 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 0 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:33.426 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "a" 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "quorum_age": 12, 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "monmap": { 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-10T20:53:33.427 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osdmap": { 2023-06-10T20:53:33.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:33.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-10T20:53:33.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-10T20:53:33.428 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgmap": { 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-10T20:53:33.431 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.432 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "fsmap": { 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": true, 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-10T20:53:33.433 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modules": [ 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "iostat", 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "nfs", 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "restful" 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ], 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.434 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "servicemap": { 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "modified": "2023-06-10T20:53:15.981935+0000", 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "services": {} 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: }, 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-10T20:53:33.435 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:33.960 INFO:teuthology.orchestra.run.smithi064.stderr:mgr is available 2023-06-10T20:53:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:33 smithi064 conmon[97224]: audit 2023-06-10T20:53:33.420162+0000 mon.a (mon.0 2023-06-10T20:53:34.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:33 smithi064 conmon[97224]: ) 25 : audit [DBG] from='client.? 172.21.15.64:0/2865098632' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-10T20:53:34.759 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:34.759 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [global] 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: fsid = b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: log_to_journald = false 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-06-10T20:53:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [mgr] 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: [osd] 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-06-10T20:53:34.761 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-06-10T20:53:35.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:34 smithi064 conmon[97224]: audit 2023-06-10T20:53:34.754304+0000 mon.a (mon 2023-06-10T20:53:35.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:34 smithi064 conmon[97224]: .0) 26 : audit [INF] from='client.? 172.21.15.64:0/3043362379' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-06-10T20:53:35.351 INFO:teuthology.orchestra.run.smithi064.stderr:Enabling cephadm module... 2023-06-10T20:53:37.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:36 smithi064 conmon[97224]: audit 2023-06-10T20:53:36.213763+0000 mon.a (mon.0 2023-06-10T20:53:37.226 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:36 smithi064 conmon[97224]: ) 27 : audit [INF] from='client.? 172.21.15.64:0/454450141' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-06-10T20:53:37.226 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:36 smithi064 conmon[97677]: ignoring --setuser ceph since I am not root 2023-06-10T20:53:37.226 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:36 smithi064 conmon[97677]: ignoring --setgroup ceph since I am not root 2023-06-10T20:53:37.226 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97677]: debug 2023-06-10T20:53:37.123+0000 7f51d42a1680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:53:37.498 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97677]: debug 2023-06-10T20:53:37.222+0000 7f51d42a1680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:53:37.750 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97677]: debug 2023-06-10T20:53:37.680+0000 7f51d42a1680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:53:38.405 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:38.405 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 5, 2023-06-10T20:53:38.405 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": true, 2023-06-10T20:53:38.406 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "active_name": "y", 2023-06-10T20:53:38.406 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standby": 0 2023-06-10T20:53:38.406 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:38.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97224]: audit 2023-06-10T20:53:36.932687+0000 mon.a (mon.0) 28 : audit 2023-06-10T20:53:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97224]: [INF] from='client.? 172.21.15.64:0/454450141' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-06-10T20:53:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:37 smithi064 conmon[97224]: cluster 2023-06-10T20:53:36.932761+0000 mon.a (mon.0) 29 : cluster [DBG] mgrmap e5: y(active, since 7s) 2023-06-10T20:53:38.733 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:38 smithi064 conmon[97677]: debug 2023-06-10T20:53:38.635+0000 7f51d42a1680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:53:38.976 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for the mgr to restart... 2023-06-10T20:53:38.976 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mgr epoch 5... 2023-06-10T20:53:38.983 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:38 smithi064 conmon[97677]: debug 2023-06-10T20:53:38.730+0000 7f51d42a1680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:53:38.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:38 smithi064 conmon[97224]: audit 2023-06-10T20:53:38.401556+0000 mon.a (mon.0 2023-06-10T20:53:38.984 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:38 smithi064 conmon[97224]: ) 30 : audit [DBG] from='client.? 172.21.15.64:0/204825608' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-06-10T20:53:39.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:38 smithi064 conmon[97677]: debug 2023-06-10T20:53:38.980+0000 7f51d42a1680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:53:40.379 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:40 smithi064 conmon[97677]: debug 2023-06-10T20:53:40.120+0000 7f51d42a1680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:53:40.379 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:40 smithi064 conmon[97677]: debug 2023-06-10T20:53:40.280+0000 7f51d42a1680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:53:40.379 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:40 smithi064 conmon[97677]: debug 2023-06-10T20:53:40.375+0000 7f51d42a1680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:53:40.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:40 smithi064 conmon[97677]: debug 2023-06-10T20:53:40.483+0000 7f51d42a1680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:53:41.067 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:40 smithi064 conmon[97677]: debug 2023-06-10T20:53:40.700+0000 7f51d42a1680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:53:41.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:41 smithi064 conmon[97677]: debug 2023-06-10T20:53:41.063+0000 7f51d42a1680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:53:41.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:41 smithi064 conmon[97677]: debug 2023-06-10T20:53:41.179+0000 7f51d42a1680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:53:42.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.046+0000 7f51d42a1680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:53:42.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.143+0000 7f51d42a1680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:53:42.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.255+0000 7f51d42a1680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:53:42.785 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.466+0000 7f51d42a1680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:53:43.113 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.781+0000 7f51d42a1680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:53:43.114 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:42 smithi064 conmon[97677]: debug 2023-06-10T20:53:42.940+0000 7f51d42a1680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:53:43.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97677]: debug 2023-06-10T20:53:43.109+0000 7f51d42a1680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:53:43.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97677]: debug 2023-06-10T20:53:43.321+0000 7f51d42a1680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:53:43.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97677]: debug 2023-06-10T20:53:43.422+0000 7f51d42a1680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:53:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: cluster 2023-06-10T20:53:43.436270+0000 mon.a (mon 2023-06-10T20:53:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: .0) 31 : cluster [INF] Active manager daemon y restarted 2023-06-10T20:53:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: cluster 2023-06-10T20:53:43.436680+0000 mon.a (mon.0) 32 : cluster 2023-06-10T20:53:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: [INF] Activating manager daemon y 2023-06-10T20:53:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: cluster 2023-06 2023-06-10T20:53:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: -10T20:53:43.438871+0000 mon.a (mon.0) 2023-06-10T20:53:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:43 smithi064 conmon[97224]: 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-06-10T20:53:44.510 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:53:44.510 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-06-10T20:53:44.510 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "initialized": true 2023-06-10T20:53:44.510 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:53:44.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: cluster 2023-06-10T20:53:43.489078+0000 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: mon.a (mon.0) 34 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0524762s) 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.491478+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.492195+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.492917+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.493084+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.493230+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:53:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: cluster 2023-06-10T20:53:43.507324+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: cephadm 2023-06-10T20:53:43.519023+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.521638+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.637535+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.638647+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.640154+0000 mon.a (mon.0) 44 : audit 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.642574+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:53:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:43.656239+0000 mon.a (mon.0) 2023-06-10T20:53:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: 46 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:53:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: audit 2023-06-10T20:53:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: 2023-06-10T20:53:43.657573+0000 mon.a (mon.0) 47 : audit 2023-06-10T20:53:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:44 smithi064 conmon[97224]: [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:53:45.059 INFO:teuthology.orchestra.run.smithi064.stderr:mgr epoch 5 is available 2023-06-10T20:53:45.060 INFO:teuthology.orchestra.run.smithi064.stderr:Setting orchestrator backend to cephadm... 2023-06-10T20:53:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: audit 2023-06-10T20:53:44.494275+0000 2023-06-10T20:53:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-06-10T20:53:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:53:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: :53:44.494758+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e7: y(active, since 1.05815s) 2023-06-10T20:53:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:53:45.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: 06-10T20:53:44.505975+0000 mgr.y ( 2023-06-10T20:53:45.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:45 smithi064 conmon[97224]: mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-06-10T20:53:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:46 smithi064 conmon[97224]: cluster 2023-06-10T20:53:45.502812+0000 mon.a (mon.0) 49 2023-06-10T20:53:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:46 smithi064 conmon[97224]: : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-06-10T20:53:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:46 smithi064 conmon[97224]: audit 2023-06-10T20:53:46.011574+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:46 smithi064 conmon[97224]: audit 2023-06-10T20:53:46.065709+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:53:47.587 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: value unchanged 2023-06-10T20:53:47.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:47 smithi064 conmon[97224]: audit 2023-06-10T20:53:46.007413+0000 mgr.y (mgr.14118) 2023-06-10T20:53:47.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:47 smithi064 conmon[97224]: 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:48.083 INFO:teuthology.orchestra.run.smithi064.stderr:Generating ssh key... 2023-06-10T20:53:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:48 smithi064 conmon[97224]: audit 2023-06-10T20:53:47.583891+0000 mgr.y (mgr.14118) 5 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: audit 2023-06-10T20:53:48.866217+0000 2023-06-10T20:53:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: mgr.y (mgr.14118) 6 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: cephadm 2023-06-10T20:53 2023-06-10T20:53:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: :48.866412+0000 mgr.y (mgr.14118) 7 : cephadm [INF] Generating ssh key... 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: audit 2023-06-10T20:53:49. 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: 387997+0000 mon.a (mon.0) 52 : audit [INF] 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: audit 2023-06-10T20:53:49. 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: 390516+0000 mon.a (mon.0) 53 : audit 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97224]: [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:49.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: Generating public/private rsa key pair. 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: Your identification has been saved in /tmp/tmpg7oozdek/key. 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: Your public key has been saved in /tmp/tmpg7oozdek/key.pub. 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: The key fingerprint is: 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: SHA256:sNG5K060BtORwdEpHOr4w2Kv0RLRO/NMkb1VYTV+xa0 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: The key's randomart image is: 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: +---[RSA 3072]----+ 2023-06-10T20:53:49.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | o++ . +oo.o| 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | . .+*o. o . .+| 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | . o B.+ . ...| 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | + o * + E. | 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | o B = S | 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | = X . . | 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | = = B . | 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | . = = . | 2023-06-10T20:53:49.661 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: | ... . | 2023-06-10T20:53:49.662 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:49 smithi064 conmon[97677]: +----[SHA256]-----+ 2023-06-10T20:53:50.734 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiD/1sawNwLvbtm2Raixg0X9Z5r1v+S99zrjHIVCAAXq/u3HqXqFvtQMvjqc469IsL+LOOk6I+m5wwmjOAzOdUuu45COwEMQc/XsxJq0z53zqssg0qN6iBmbveczHXCb0kAPZYafHHJk0ZMmfAmmvYXHD98SiWKnbP2VH7UxzkhgZtLeza1PnM7Tvuxgnu3C8VtcIe2pFlWLKTnwPhUwvBLpX2WPuvn3H1O99J7AUC7kgAoOp0of/j8h3o1R224Y6re4rK7pLz5N/yW7dtXZS0V8BiWhXIh7XrBUE9A2b9ewzpL3sGiJxjyyYwsXjkw8SNsVXKNvEwZ9EfYyn4bOaqJ4Xmqsr9bnJ59lh0nr+3jO7NdcW3uRjPfk1tZ9HW45Pv0AefYgm/3ALPTfI4AGETUtLxc7TAegmr/3Yi+eeUAaHyMJDQn+D1+A/FouHF0N+Uday4Z0MGfZ1Da7xRGAzBiib1zirZCZaPQ0sLm6JZt5+C5ec4dqrYucPDvF37NmM= ceph-b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:53:51.358 INFO:teuthology.orchestra.run.smithi064.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-06-10T20:53:51.358 INFO:teuthology.orchestra.run.smithi064.stderr:Adding key to root@localhost authorized_keys... 2023-06-10T20:53:51.359 INFO:teuthology.orchestra.run.smithi064.stderr:Adding host smithi064... 2023-06-10T20:53:52.241 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:53:52 smithi064 conmon[97677]: [79B blob data] 2023-06-10T20:53:52.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:51 smithi064 conmon[97224]: audit 2023-06-10T20:53:50.730602+0000 mgr.y (mgr.14118 2023-06-10T20:53:52.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:51 smithi064 conmon[97224]: ) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:52.843 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: Added host 'smithi064' with addr '172.21.15.64' 2023-06-10T20:53:53.415 INFO:teuthology.orchestra.run.smithi064.stderr:Deploying unmanaged mon service... 2023-06-10T20:53:54.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:53 smithi064 conmon[97224]: audit 2023-06-10T20:53:52.135395+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi064", "addr": "172.21.15.64", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:54.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:53 smithi064 conmon[97224]: audit 2023-06-10T20:53:52.838304+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:54.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:53 smithi064 conmon[97224]: cephadm 2023-06-10T20:53:52.838671+0000 mgr.y (mgr.14118) 10 : cephadm [INF] Added host smithi064 2023-06-10T20:53:54.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:53 smithi064 conmon[97224]: audit 2023-06-10T20:53:52.851803+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:53:54.248 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: Scheduled mon update... 2023-06-10T20:53:54.890 INFO:teuthology.orchestra.run.smithi064.stderr:Deploying unmanaged mgr service... 2023-06-10T20:53:55.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:55 smithi064 conmon[97224]: audit 2023-06-10T20:53:54.241023+0000 mgr.y (mgr.14118) 11 2023-06-10T20:53:55.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:55 smithi064 conmon[97224]: : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:55.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:55 smithi064 conmon[97224]: cephadm 2023-06-10T20:53:54.242424+0000 mgr.y (mgr.14118) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-06-10T20:53:55.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:55 smithi064 conmon[97224]: audit 2023-06-10T20:53:54.244779+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:56.090 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-06-10T20:53:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:56 smithi064 conmon[97224]: audit 2023-06-10T20:53:55.511579+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:56 smithi064 conmon[97224]: audit 2023-06-10T20:53:55.784490+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:56 smithi064 conmon[97224]: audit 2023-06-10T20:53:56.086834+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:57 smithi064 conmon[97224]: audit 2023-06-10T20:53:56.082929+0000 2023-06-10T20:53:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:57 smithi064 conmon[97224]: mgr.y (mgr.14118) 13 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:53:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:57 smithi064 conmon[97224]: cephadm 2023-06-10T20:53:56.084394+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-06-10T20:53:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:57 smithi064 conmon[97224]: audit 2023-06-10T20:53:57.138058+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14118 172.21.15.64:0/4174356716' entity='mgr.y' 2023-06-10T20:53:58.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:58 smithi064 conmon[97224]: audit 2023-06-10T20:53 2023-06-10T20:53:58.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:53:58 smithi064 conmon[97224]: :57.922095+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.64:0/1137335600' entity='client.admin' 2023-06-10T20:54:00.082 INFO:teuthology.orchestra.run.smithi064.stderr:Enabling the dashboard module... 2023-06-10T20:54:00.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:00 smithi064 conmon[97224]: audit 2023-06-10T20:53:59.515827+0000 mon.a (mon 2023-06-10T20:54:00.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:00 smithi064 conmon[97224]: .0) 62 : audit [INF] from='client.? 172.21.15.64:0/2179651265' entity='client.admin' 2023-06-10T20:54:02.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:01 smithi064 conmon[97224]: audit 2023-06-10T20:54:00.955509+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.64:0/1235922133' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-06-10T20:54:02.240 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:01 smithi064 conmon[97677]: ignoring --setuser ceph since I am not root 2023-06-10T20:54:02.241 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:01 smithi064 conmon[97677]: ignoring --setgroup ceph since I am not root 2023-06-10T20:54:02.241 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97677]: debug 2023-06-10T20:54:02.138+0000 7f63a26cf680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:54:02.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97677]: debug 2023-06-10T20:54:02.235+0000 7f63a26cf680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:54:02.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97677]: debug 2023-06-10T20:54:02.696+0000 7f63a26cf680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:54:03.381 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:54:03.381 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "epoch": 9, 2023-06-10T20:54:03.382 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "available": true, 2023-06-10T20:54:03.382 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "active_name": "y", 2023-06-10T20:54:03.382 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "num_standby": 0 2023-06-10T20:54:03.382 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:54:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97224]: audit 2023-06-10T20:54:01.938869+0000 mon.a (mon.0 2023-06-10T20:54:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97224]: ) 64 : audit [INF] from='client.? 172.21.15.64:0/1235922133' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-06-10T20:54:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:02 smithi064 conmon[97224]: cluster 2023-06-10T20:54:01.938952+0000 mon.a (mon.0) 65 : cluster [DBG] mgrmap e9: y(active, since 18s) 2023-06-10T20:54:03.722 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:03 smithi064 conmon[97677]: debug 2023-06-10T20:54:03.621+0000 7f63a26cf680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:54:03.722 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:03 smithi064 conmon[97677]: debug 2023-06-10T20:54:03.717+0000 7f63a26cf680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:54:03.948 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for the mgr to restart... 2023-06-10T20:54:03.948 INFO:teuthology.orchestra.run.smithi064.stderr:Waiting for mgr epoch 9... 2023-06-10T20:54:03.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:03 smithi064 conmon[97224]: audit 2023-06-10T20:54:03.377575+0000 mon.a (mon. 2023-06-10T20:54:03.978 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:03 smithi064 conmon[97224]: 0) 66 : audit [DBG] from='client.? 172.21.15.64:0/326740416' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-06-10T20:54:04.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:03 smithi064 conmon[97677]: debug 2023-06-10T20:54:03.973+0000 7f63a26cf680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:54:05.380 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:05 smithi064 conmon[97677]: debug 2023-06-10T20:54:05.119+0000 7f63a26cf680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:54:05.381 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:05 smithi064 conmon[97677]: debug 2023-06-10T20:54:05.281+0000 7f63a26cf680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:54:05.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:05 smithi064 conmon[97677]: debug 2023-06-10T20:54:05.376+0000 7f63a26cf680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:54:05.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:05 smithi064 conmon[97677]: debug 2023-06-10T20:54:05.485+0000 7f63a26cf680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:54:06.073 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:05 smithi064 conmon[97677]: debug 2023-06-10T20:54:05.704+0000 7f63a26cf680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:54:06.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:06 smithi064 conmon[97677]: debug 2023-06-10T20:54:06.069+0000 7f63a26cf680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:54:06.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:06 smithi064 conmon[97677]: debug 2023-06-10T20:54:06.186+0000 7f63a26cf680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:54:07.267 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.054+0000 7f63a26cf680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:54:07.268 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.152+0000 7f63a26cf680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:54:07.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.263+0000 7f63a26cf680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:54:07.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.477+0000 7f63a26cf680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:54:08.100 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.764+0000 7f63a26cf680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:54:08.100 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:07 smithi064 conmon[97677]: debug 2023-06-10T20:54:07.927+0000 7f63a26cf680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:54:08.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:08 smithi064 conmon[97677]: debug 2023-06-10T20:54:08.095+0000 7f63a26cf680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:54:08.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:08 smithi064 conmon[97677]: debug 2023-06-10T20:54:08.309+0000 7f63a26cf680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:54:08.668 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:08 smithi064 conmon[97677]: debug 2023-06-10T20:54:08.411+0000 7f63a26cf680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:54:09.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:08 smithi064 conmon[97677]: [79B blob data] 2023-06-10T20:54:09.497 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: { 2023-06-10T20:54:09.497 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-06-10T20:54:09.497 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: "initialized": true 2023-06-10T20:54:09.497 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: } 2023-06-10T20:54:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: cluster 2023-06-10T20:54:08.425368+0000 mon.a (mon.0) 67 : cluster [INF] Active manager daemon y restarted 2023-06-10T20:54:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: cluster 2023-06-10T20:54:08.425797+0000 mon.a (mon.0) 68 : cluster [INF] Activating manager daemon y 2023-06-10T20:54:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: cluster 2023-06-10T20:54:08.427520+0000 mon.a (mon.0) 69 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: cluster 2023-06-10T20:54:08.478415+0000 mon.a (mon.0) 70 : cluster [DBG] mgrmap e10: y(active, starting, since 0.0527328s) 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.479327+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.479929+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.480885+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.481039+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.481175+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:54:09.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: cluster 2023-06-10T20:54:08.495000+0000 mon.a (mon.0) 76 : cluster [INF] Manager daemon y is now available 2023-06-10T20:54:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.621895+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.623416+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.651477+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:54:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:09 smithi064 conmon[97224]: audit 2023-06-10T20:54:08.653006+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:54:10.268 INFO:teuthology.orchestra.run.smithi064.stderr:mgr epoch 9 is available 2023-06-10T20:54:10.268 INFO:teuthology.orchestra.run.smithi064.stderr:Generating a dashboard self-signed certificate... 2023-06-10T20:54:10.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:10 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:54:10.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:10 smithi064 conmon[97224]: 54:09.482701+0000 mgr.y (mgr.14148) 1 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-06-10T20:54:10.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:10 smithi064 conmon[97224]: cluster 2023-06-10T20:54:09.482848+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e11: y(active, since 1.05716s) 2023-06-10T20:54:10.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:10 smithi064 conmon[97224]: audit 2023-06-10T20:54:09.493253+0000 mgr.y (mgr.14148) 2 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-06-10T20:54:10.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:10 smithi064 conmon[97224]: audit 2023-06-10T20:54:10.209805+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:11.274 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: Self-signed certificate created 2023-06-10T20:54:11.947 INFO:teuthology.orchestra.run.smithi064.stderr:Creating initial admin user... 2023-06-10T20:54:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:12 smithi064 conmon[97224]: audit 2023-06-10T20:54:11.107733 2023-06-10T20:54:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:12 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 3 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:12 smithi064 conmon[97224]: cluster 2023-06-10T20:54:11.211426+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-06-10T20:54:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:12 smithi064 conmon[97224]: audit 2023-06-10T20:54:11.269052+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:12 smithi064 conmon[97224]: audit 2023-06-10T20:54:11.270918+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:12.987 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$fPoLIS3aLzLc2LIdbot8Ge94amaUyL4oPheHftdU5LixQVCueptD.", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1686430452, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-06-10T20:54:13.546 INFO:teuthology.orchestra.run.smithi064.stderr:Fetching dashboard port number... 2023-06-10T20:54:14.333 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: 8443 2023-06-10T20:54:14.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:13 smithi064 conmon[97224]: audit 2023-06-10T20:54:12.738492+0000 mgr.y (mgr.14148) 4 : audit [DBG] from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:14.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:13 smithi064 conmon[97224]: audit 2023-06-10T20:54:12.983586+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:14.880 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: disabled 2023-06-10T20:54:14.888 INFO:teuthology.orchestra.run.smithi064.stderr:systemctl: inactive 2023-06-10T20:54:14.888 INFO:teuthology.orchestra.run.smithi064.stderr:firewalld.service is not enabled 2023-06-10T20:54:14.888 INFO:teuthology.orchestra.run.smithi064.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-06-10T20:54:14.890 INFO:teuthology.orchestra.run.smithi064.stderr:Ceph Dashboard is now available at: 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr: URL: https://smithi064.front.sepia.ceph.com:8443/ 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr: User: admin 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr: Password: up3f7r5cqu 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:14.891 INFO:teuthology.orchestra.run.smithi064.stderr:Enabling autotune for osd_memory_target 2023-06-10T20:54:15.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:14 smithi064 conmon[97224]: audit 2023-06-10T20:54:14.329809+0000 mon.a (mon 2023-06-10T20:54:15.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:14 smithi064 conmon[97224]: .0) 87 : audit [DBG] from='client.? 172.21.15.64:0/3552671607' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-06-10T20:54:16.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:15 smithi064 conmon[97224]: cluster 2023-06-10T20:54:14.990873+0000 mon.a (mon.0 2023-06-10T20:54:16.362 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:15 smithi064 conmon[97224]: ) 88 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-06-10T20:54:17.152 INFO:teuthology.orchestra.run.smithi064.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-06-10T20:54:17.771 INFO:teuthology.orchestra.run.smithi064.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-06-10T20:54:17.771 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.771 INFO:teuthology.orchestra.run.smithi064.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:17.771 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr:Or, if you are only running a single cluster on this host: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr:Please consider enabling telemetry to help improve Ceph: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.772 INFO:teuthology.orchestra.run.smithi064.stderr: ceph telemetry on 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr:For more information see: 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr: 2023-06-10T20:54:17.773 INFO:teuthology.orchestra.run.smithi064.stderr:Bootstrap complete. 2023-06-10T20:54:17.808 INFO:tasks.cephadm:Fetching config... 2023-06-10T20:54:17.808 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:17.809 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-06-10T20:54:17.825 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-06-10T20:54:17.826 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:17.826 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-06-10T20:54:17.885 INFO:tasks.cephadm:Fetching mon keyring... 2023-06-10T20:54:17.885 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:17.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/keyring of=/dev/stdout 2023-06-10T20:54:17.953 INFO:tasks.cephadm:Fetching pub ssh key... 2023-06-10T20:54:17.954 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:17.954 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-06-10T20:54:18.010 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-06-10T20:54:18.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiD/1sawNwLvbtm2Raixg0X9Z5r1v+S99zrjHIVCAAXq/u3HqXqFvtQMvjqc469IsL+LOOk6I+m5wwmjOAzOdUuu45COwEMQc/XsxJq0z53zqssg0qN6iBmbveczHXCb0kAPZYafHHJk0ZMmfAmmvYXHD98SiWKnbP2VH7UxzkhgZtLeza1PnM7Tvuxgnu3C8VtcIe2pFlWLKTnwPhUwvBLpX2WPuvn3H1O99J7AUC7kgAoOp0of/j8h3o1R224Y6re4rK7pLz5N/yW7dtXZS0V8BiWhXIh7XrBUE9A2b9ewzpL3sGiJxjyyYwsXjkw8SNsVXKNvEwZ9EfYyn4bOaqJ4Xmqsr9bnJ59lh0nr+3jO7NdcW3uRjPfk1tZ9HW45Pv0AefYgm/3ALPTfI4AGETUtLxc7TAegmr/3Yi+eeUAaHyMJDQn+D1+A/FouHF0N+Uday4Z0MGfZ1Da7xRGAzBiib1zirZCZaPQ0sLm6JZt5+C5ec4dqrYucPDvF37NmM= ceph-b618768e-07d0-11ee-9b1e-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-06-10T20:54:18.093 INFO:teuthology.orchestra.run.smithi064.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiD/1sawNwLvbtm2Raixg0X9Z5r1v+S99zrjHIVCAAXq/u3HqXqFvtQMvjqc469IsL+LOOk6I+m5wwmjOAzOdUuu45COwEMQc/XsxJq0z53zqssg0qN6iBmbveczHXCb0kAPZYafHHJk0ZMmfAmmvYXHD98SiWKnbP2VH7UxzkhgZtLeza1PnM7Tvuxgnu3C8VtcIe2pFlWLKTnwPhUwvBLpX2WPuvn3H1O99J7AUC7kgAoOp0of/j8h3o1R224Y6re4rK7pLz5N/yW7dtXZS0V8BiWhXIh7XrBUE9A2b9ewzpL3sGiJxjyyYwsXjkw8SNsVXKNvEwZ9EfYyn4bOaqJ4Xmqsr9bnJ59lh0nr+3jO7NdcW3uRjPfk1tZ9HW45Pv0AefYgm/3ALPTfI4AGETUtLxc7TAegmr/3Yi+eeUAaHyMJDQn+D1+A/FouHF0N+Uday4Z0MGfZ1Da7xRGAzBiib1zirZCZaPQ0sLm6JZt5+C5ec4dqrYucPDvF37NmM= ceph-b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:18.107 DEBUG:teuthology.orchestra.run.smithi152:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiD/1sawNwLvbtm2Raixg0X9Z5r1v+S99zrjHIVCAAXq/u3HqXqFvtQMvjqc469IsL+LOOk6I+m5wwmjOAzOdUuu45COwEMQc/XsxJq0z53zqssg0qN6iBmbveczHXCb0kAPZYafHHJk0ZMmfAmmvYXHD98SiWKnbP2VH7UxzkhgZtLeza1PnM7Tvuxgnu3C8VtcIe2pFlWLKTnwPhUwvBLpX2WPuvn3H1O99J7AUC7kgAoOp0of/j8h3o1R224Y6re4rK7pLz5N/yW7dtXZS0V8BiWhXIh7XrBUE9A2b9ewzpL3sGiJxjyyYwsXjkw8SNsVXKNvEwZ9EfYyn4bOaqJ4Xmqsr9bnJ59lh0nr+3jO7NdcW3uRjPfk1tZ9HW45Pv0AefYgm/3ALPTfI4AGETUtLxc7TAegmr/3Yi+eeUAaHyMJDQn+D1+A/FouHF0N+Uday4Z0MGfZ1Da7xRGAzBiib1zirZCZaPQ0sLm6JZt5+C5ec4dqrYucPDvF37NmM= ceph-b618768e-07d0-11ee-9b1e-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-06-10T20:54:18.151 INFO:teuthology.orchestra.run.smithi152.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCiD/1sawNwLvbtm2Raixg0X9Z5r1v+S99zrjHIVCAAXq/u3HqXqFvtQMvjqc469IsL+LOOk6I+m5wwmjOAzOdUuu45COwEMQc/XsxJq0z53zqssg0qN6iBmbveczHXCb0kAPZYafHHJk0ZMmfAmmvYXHD98SiWKnbP2VH7UxzkhgZtLeza1PnM7Tvuxgnu3C8VtcIe2pFlWLKTnwPhUwvBLpX2WPuvn3H1O99J7AUC7kgAoOp0of/j8h3o1R224Y6re4rK7pLz5N/yW7dtXZS0V8BiWhXIh7XrBUE9A2b9ewzpL3sGiJxjyyYwsXjkw8SNsVXKNvEwZ9EfYyn4bOaqJ4Xmqsr9bnJ59lh0nr+3jO7NdcW3uRjPfk1tZ9HW45Pv0AefYgm/3ALPTfI4AGETUtLxc7TAegmr/3Yi+eeUAaHyMJDQn+D1+A/FouHF0N+Uday4Z0MGfZ1Da7xRGAzBiib1zirZCZaPQ0sLm6JZt5+C5ec4dqrYucPDvF37NmM= ceph-b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:18.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-06-10T20:54:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:18 smithi064 conmon[97224]: audit 2023-06-10T20:54:17.146381+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.64:0/2000688676' entity='client.admin' 2023-06-10T20:54:21.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:19.826126+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:21.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.225495+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi064", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:54:21.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.229092+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.230682+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.234803+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.703491+0000 mon.a (mon.0) 95 : audit [INF] from='client.? 172.21.15.64:0/19134918' entity='client.admin' 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: -06-10T20:54:20.750244+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:21.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:20 smithi064 conmon[97224]: audit 2023-06-10T20:54:20.754667+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:21.491 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-06-10T20:54:21.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-06-10T20:54:23.784 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi152 2023-06-10T20:54:23.784 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:54:23.785 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.conf 2023-06-10T20:54:23.804 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:54:23.805 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:23.864 INFO:tasks.cephadm:Adding host smithi152 to orchestrator... 2023-06-10T20:54:23.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch host add smithi152 2023-06-10T20:54:24.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.207910+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:24.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.223870+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:24.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.225278+0000 mon.a (mon.0) 100 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:24.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.226047+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:24.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:23.227114+0000 mgr.y (mgr.14148) 6 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:54:24.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:23.250917+0000 mgr.y (mgr. 2023-06-10T20:54:24.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: 14148) 7 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:24.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.279226+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:24.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:24 smithi064 conmon[97224]: audit 2023-06-10T20:54:23.282038+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:26.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:25 smithi064 conmon[97677]: [90B blob data] 2023-06-10T20:54:26.342 INFO:teuthology.orchestra.run.smithi064.stdout:Added host 'smithi152' with addr '172.21.15.152' 2023-06-10T20:54:26.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97677]: [80B blob data] 2023-06-10T20:54:26.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch host ls --format=json 2023-06-10T20:54:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: audit 2023-06-10T20:54:25.621373+0000 mgr.y (mgr.14148) 8 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi152", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: audit 2023-06-10T20:54:26.338967+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: audit 2023-06-10T20:54:26.356420+0000 mon.a ( 2023-06-10T20:54:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: mon.0) 105 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: audit 2023-06-10T20:54:26 2023-06-10T20:54:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: .814076+0000 mon.a (mon.0) 106 : audit 2023-06-10T20:54:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:26 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:28.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:27 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:26.339326+0000 mgr.y (mgr.14148) 9 : cephadm [INF] Added host smithi152 2023-06-10T20:54:28.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:27 smithi064 conmon[97224]: audit 2023-06-10T20:54:27.163875+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:28.753 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:54:28.754 INFO:teuthology.orchestra.run.smithi064.stdout:[{"addr": "172.21.15.64", "hostname": "smithi064", "labels": [], "status": ""}, {"addr": "172.21.15.152", "hostname": "smithi152", "labels": [], "status": ""}] 2023-06-10T20:54:29.310 INFO:tasks.cephadm:Setting crush tunables to default 2023-06-10T20:54:29.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd crush tunables default 2023-06-10T20:54:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:29 smithi064 conmon[97224]: cluster 2023-06-10T20:54:28.482155+0000 2023-06-10T20:54:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:29 smithi064 conmon[97224]: mgr.y (mgr.14148) 10 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:29 smithi064 conmon[97224]: audit 2023-06-10T20:54:28.749460+0000 mgr.y (mgr.14148) 11 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:54:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:29.851421+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.193832+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.194474+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.195384+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.196129+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.245892+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:30 smithi064 conmon[97224]: audit 2023-06-10T20:54:30.251583+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:31.948 INFO:teuthology.orchestra.run.smithi064.stderr:adjusted tunables profile to default 2023-06-10T20:54:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:30.197204+0000 mgr.y (mgr. 2023-06-10T20:54:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: 14148) 12 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:54:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:30.216445+0000 mgr.y (mgr.14148) 13 : 2023-06-10T20:54:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: cluster 2023-06-10T20:54:32.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: 2023-06-10T20: 2023-06-10T20:54:32.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: 54:30.482426+0000 mgr.y 2023-06-10T20:54:32.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: (mgr.14148) 14 : cluster [DBG] 2023-06-10T20:54:32.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:32.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:54:32.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: 20:54:31.086611+0000 mon.a (mon 2023-06-10T20:54:32.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: .0) 115 : audit [INF] 2023-06-10T20:54:32.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:31 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/3282093810' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-06-10T20:54:32.518 INFO:tasks.cephadm:Adding mon.a on smithi064 2023-06-10T20:54:32.519 INFO:tasks.cephadm:Adding mon.c on smithi064 2023-06-10T20:54:32.519 INFO:tasks.cephadm:Adding mon.b on smithi152 2023-06-10T20:54:32.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply mon '3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi152:172.21.15.152=b' 2023-06-10T20:54:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:32 smithi064 conmon[97224]: audit 2023-06-10T20:54:31.939989+0000 mon.a (mon.0) 116 : 2023-06-10T20:54:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:32 smithi064 conmon[97224]: audit [INF] from='client.? 172.21.15.64:0/3282093810' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-06-10T20:54:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:32 smithi064 conmon[97224]: cluster 2023-06-10T20:54:31.940061+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:34.230 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mon update... 2023-06-10T20:54:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:33 smithi064 conmon[97224]: cluster 2023-06-10T20:54:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:33 smithi064 conmon[97224]: 2023-06-10T20:54:32.482862+0000 mgr.y (mgr.14148) 15 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:34.788 DEBUG:teuthology.orchestra.run.smithi064:mon.c> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.c.service 2023-06-10T20:54:34.791 DEBUG:teuthology.orchestra.run.smithi152:mon.b> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.b.service 2023-06-10T20:54:34.794 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-10T20:54:34.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mon dump -f json 2023-06-10T20:54:34.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:54:34.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:54:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34.220933+0000 mgr.y (mgr.14148 2023-06-10T20:54:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: ) 16 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi152:172.21.15.152=b", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:34.223655+0000 mgr.y (mgr.14148) 17 : cephadm [INF] Saving service mon spec with placement smithi064:172.21.15.64=a;smithi064:[v2:172.21.15.64:3301,v1:172.21.15.64:6790]=c;smithi152:172.21.15.152=b;count:3 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34.226259+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34.276140+0000 mon.a (mon.0) 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: 119 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023 2023-06-10T20:54:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: -06-10T20:54:34.277970+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34. 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: 278921+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34.285068+0000 mon.a ( 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: mon.0) 122 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:34.287294+0000 mon.a (mon.0 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: ) 123 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: audit 2023-06-10T20:54:35.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: 2023-06-10T20:54:34.288316+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:35.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:34.289200+0000 2023-06-10T20:54:35.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: mgr.y (mgr.14148) 18 : cephadm [INF] Deploying daemon mon.b on smithi152 2023-06-10T20:54:35.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: cluster 2023-06-10T20:54:34.483232+0000 mgr.y (mgr. 2023-06-10T20:54:35.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:35 smithi064 conmon[97224]: 14148) 19 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:37.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:37 smithi152 systemd[1]: Starting Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:54:37.819 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:54:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":1,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","modified":"2023-06-10T20:53:13.526882Z","created":"2023-06-10T20:53:13.526882Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-06-10T20:54:37.821 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 1 2023-06-10T20:54:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:37 smithi064 conmon[97224]: cluster 2023-06-10T20:54:36.483637+0000 mgr.y (mgr.14148) 20 : cluster 2023-06-10T20:54:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:37 smithi064 conmon[97224]: [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:37 smithi064 conmon[97224]: audit 2023-06-10T20:54:37.815919+0000 mon.a (mon.0) 125 : audit [DBG] from='client.? 172.21.15.152:0/823790927' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.688+0000 7f69b9937880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.688+0000 7f69b9937880 0 ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable), process ceph-mon, pid 7 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.688+0000 7f69b9937880 0 pidfile_write: ignore empty --pid-file 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.692+0000 7f69b9937880 0 load: jerasure load: lrc load: isa 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: RocksDB version: 6.8.1 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-10T20:54:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Compile date May 28 2023 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: DB SUMMARY 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: CURRENT file: CURRENT 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: IDENTITY file: IDENTITY 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-06-10T20:54:38.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.error_if_exists: 0 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.create_if_missing: 0 2023-06-10T20:54:38.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.paranoid_checks: 1 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.env: 0x5604bfea4080 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.fs: Posix File System 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.info_log: 0x5604c1f61960 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.statistics: (nil) 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.use_fsync: 0 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_log_file_size: 0 2023-06-10T20:54:38.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.allow_fallocate: 1 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.use_direct_reads: 0 2023-06-10T20:54:38.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.db_log_dir: 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_subcompactions: 1 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_background_flushes: -1 2023-06-10T20:54:38.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.write_buffer_manager: 0x5604c2c5c210 2023-06-10T20:54:38.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T20:54:38.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-10T20:54:38.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T20:54:38.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T20:54:38.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-10T20:54:38.954 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T20:54:38.954 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.unordered_write: 0 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T20:54:38.955 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.row_cache: None 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.wal_filter: None 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.preserve_deletes: 0 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.two_write_queues: 0 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-10T20:54:38.956 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.atomic_flush: 0 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.log_readahead_size: 0 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_background_jobs: 2 2023-06-10T20:54:38.957 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_background_compactions: -1 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T20:54:38.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_open_files: -1 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Compression algorithms supported: 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T20:54:38.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kZSTD supported: 0 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kXpressCompression supported: 0 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kLZ4Compression supported: 1 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kBZip2Compression supported: 0 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kZlibCompression supported: 1 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: kSnappyCompression supported: 1 2023-06-10T20:54:38.960 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.merge_operator: 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_filter: None 2023-06-10T20:54:38.961 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5604c1f0bd18) 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: cache_index_and_filter_blocks: 1 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T20:54:38.962 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: pin_top_level_index_and_filter: 1 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: index_type: 0 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: data_block_index_type: 0 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: index_shortening: 1 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: data_block_hash_table_util_ratio: 0.750000 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: hash_index_allow_collision: 1 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: checksum: 1 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: no_block_cache: 0 2023-06-10T20:54:38.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_cache: 0x5604c1f44f10 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_cache_name: BinnedLRUCache 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_cache_options: 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: capacity : 536870912 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: num_shard_bits : 4 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: strict_capacity_limit : 0 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: high_pri_pool_ratio: 0.000 2023-06-10T20:54:38.964 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_cache_compressed: (nil) 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: persistent_cache: (nil) 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_size: 4096 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_size_deviation: 10 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_restart_interval: 16 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: index_block_restart_interval: 1 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: metadata_block_size: 4096 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: partition_filters: 0 2023-06-10T20:54:38.965 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: use_delta_encoding: 1 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: whole_key_filtering: 1 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: verify_compression: 0 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: read_amp_bytes_per_bit: 0 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: format_version: 2 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: enable_index_compression: 1 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: block_align: 0 2023-06-10T20:54:38.966 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression: NoCompression 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.num_levels: 7 2023-06-10T20:54:38.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:54:38.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-10T20:54:38.969 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T20:54:38.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T20:54:38.971 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T20:54:38.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T20:54:38.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.table_properties_collectors: 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.inplace_update_support: 0 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T20:54:38.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.bloom_locality: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.max_successive_merges: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.ttl: 2592000 2023-06-10T20:54:38.975 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.693+0000 7f69b9937880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430478695472, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-06-10T20:54:38.976 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.694+0000 7f69b9937880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430478695953, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1686430478, "oldest_key_time": 3, "file_creation_time": 0}} 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.695+0000 7f69b9937880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.695+0000 7f69b9937880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430478696344, "job": 1, "event": "recovery_finished"} 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.695+0000 7f69b9937880 4 rocksdb: DB pointer 0x5604c200f800 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.696+0000 7f69a0742700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.696+0000 7f69a0742700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-10T20:54:38.977 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** DB Stats ** 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-10T20:54:38.978 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** Compaction Stats [default] ** 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.979 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** Compaction Stats [default] ** 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:38.980 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:54:38.981 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** Compaction Stats [default] ** 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.982 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** Compaction Stats [default] ** 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:38.983 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:54:38.984 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.696+0000 7f69b9937880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.696+0000 7f69b9937880 0 using public_addr v2:172.21.15.152:0/0 -> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.696+0000 7f69b9937880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.697+0000 7f69b9937880 1 mon.b@-1(???) e0 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 bash[97272]: a46036733d6f84427973cdcebacc3259be60cf6fed4e24399d79d4ce1c4203a9 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 systemd[1]: Started Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.718+0000 7f69a2f47700 0 mon.b@-1(synchronizing).mds e1 new map 2023-06-10T20:54:38.985 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.718+0000 7f69a2f47700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: e1 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: legacy client fscid: -1 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: No filesystems configured 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:38.986 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.718+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-06-10T20:54:38.987 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: cluster 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: 2023-06-10T20:54:36.483637+0000 mgr.y 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: (mgr.14148) 20 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: audit 2023 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: -06-10T20:54:37.815919+0000 2023-06-10T20:54:38.988 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: mon.a (mon.0) 125 : audit [DBG] from='client.? 172.21.15.152:0/823790927' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:38.989 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:38 smithi152 conmon[97365]: debug 2023-06-10T20:54:38.719+0000 7f69a2f47700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-06-10T20:54:39.648 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-10T20:54:39.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mon dump -f json 2023-06-10T20:54:44.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:38.484237+0000 mgr.y (mgr.14148) 21 : cluster 2023-06-10T20:54:44.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T20:54:38.726801+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:38.727048+0000 mon.a (mon. 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 0) 128 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T20:54:38.740534+0000 mon.a (mon.0) 129 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T20:54:39.724004+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:40.484520+0000 mgr.y (mgr.14148) 22 : cluster 2023-06-10T20:54:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: -10T20:54:40.724658+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20: 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 54:40.725438+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 20:54:41.724717+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 2023-06-10T20:54:42.484807+0000 mgr.y (mgr.14148) 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: T20:54:42.724524+0000 mon.a (mon.0) 133 : audit [DBG] 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T20:54:43.724608+0000 mon.a (mon 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: .0) 134 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 2023-06-10T20:54:43.754715+0000 mon.a (mon.0) 135 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: :54:43.760519+0000 mon.a (mon.0) 136 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: T20:54:43.773039+0000 mon.a (mon.0) 137 : cluster [DBG] fsmap 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54:43 2023-06-10T20:54:44.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: .773085+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20:54: 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 43.773299+0000 mon.a (mon.0) 139 : cluster [DBG] mgrmap e13: y(active, since 35s) 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: :54:43.776484+0000 mon.a (mon.0) 140 : cluster [INF] overall HEALTH_OK 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:44.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 10T20:54:43.781965+0000 mon.a (mon.0) 141 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:44.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 2023-06-10T20:54:44.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:54:44.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: 20:54:43.783022+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:44.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: audit 2023-06-10T20:54 2023-06-10T20:54:44.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:43 smithi064 conmon[97224]: :43.783866+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:38.484237+0000 mgr.y (mgr.14148) 21 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:38.726801+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:38.727048+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:38.740534+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:39.724004+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:40.484520+0000 mgr.y (mgr.14148) 22 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:40.724658+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:40.725438+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:41.724717+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:42.484807+0000 mgr.y (mgr.14148) 23 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: 2023-06-10T20:54:42.724524+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:43.724608+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.754715+0000 mon.a (mon.0) 135 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.760519+0000 mon.a (mon.0) 136 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.773039+0000 mon.a (mon.0) 137 : cluster [DBG] fsmap 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.773085+0000 mon.a (mon.0) 138 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.773299+0000 mon.a (mon.0) 139 : cluster [DBG] mgrmap e13: y(active, since 35s) 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: cluster 2023-06-10T20:54:43.776484+0000 mon.a (mon.0) 140 : cluster [INF] overall HEALTH_OK 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:43.781965+0000 mon.a (mon.0) 141 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:44.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:43.783022+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:44.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:43 smithi152 conmon[97365]: audit 2023-06-10T20:54:43.783866+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:54:44.396 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":2,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","modified":"2023-06-10T20:54:38.724225Z","created":"2023-06-10T20:53:13.526882Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-06-10T20:54:44.398 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 2 2023-06-10T20:54:45.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:44 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:43.784655+0000 mgr.y (mgr.14148) 24 : cephadm [INF] Deploying daemon mon.c on smithi064 2023-06-10T20:54:45.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:44 smithi064 conmon[97224]: audit 2023-06-10T20:54:44.392546+0000 mon.a (mon.0) 144 : audit [DBG] from='client.? 172.21.15.152:0/2709655759' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:45.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:44 smithi064 conmon[97224]: audit 2023-06-10T20:54:44.724740+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:44 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:43.784655+0000 mgr.y (mgr.14148) 24 : cephadm [INF] Deploying daemon mon.c on smithi064 2023-06-10T20:54:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:44 smithi152 conmon[97365]: audit 2023-06-10T20:54:44.392546+0000 mon.a (mon.0) 144 : audit [DBG] from='client.? 172.21.15.152:0/2709655759' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:45.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:44 smithi152 conmon[97365]: audit 2023-06-10T20:54:44.724740+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:45.972 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-10T20:54:45.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mon dump -f json 2023-06-10T20:54:46.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:45 smithi064 conmon[97224]: cluster 2023-06-10T20:54:44.485032 2023-06-10T20:54:46.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:45 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:46.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:45 smithi152 conmon[97365]: cluster 2023-06-10T20:54:44.485032+0000 mgr.y (mgr.14148) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:46.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 systemd[1]: Starting Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:54:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.762+0000 7fb779135880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.762+0000 7fb779135880 0 ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable), process ceph-mon, pid 7 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.762+0000 7fb779135880 0 pidfile_write: ignore empty --pid-file 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.766+0000 7fb779135880 0 load: jerasure load: lrc load: isa 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: RocksDB version: 6.8.1 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-10T20:54:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Compile date May 28 2023 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: DB SUMMARY 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: CURRENT file: CURRENT 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: IDENTITY file: IDENTITY 2023-06-10T20:54:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 636 ; 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.error_if_exists: 0 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.create_if_missing: 0 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.paranoid_checks: 1 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.env: 0x5617f6c7f080 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.fs: Posix File System 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.info_log: 0x5617f7955960 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-10T20:54:47.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.statistics: (nil) 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.use_fsync: 0 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_log_file_size: 0 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-10T20:54:47.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.allow_fallocate: 1 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.use_direct_reads: 0 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.db_log_dir: 2023-06-10T20:54:47.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_subcompactions: 1 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_background_flushes: -1 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T20:54:47.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T20:54:47.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T20:54:47.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T20:54:47.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-10T20:54:47.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.write_buffer_manager: 0x5617f86501e0 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-10T20:54:47.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.unordered_write: 0 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T20:54:47.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T20:54:47.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T20:54:47.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T20:54:47.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.row_cache: None 2023-06-10T20:54:47.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.wal_filter: None 2023-06-10T20:54:47.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T20:54:47.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-10T20:54:47.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.preserve_deletes: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.two_write_queues: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.atomic_flush: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T20:54:47.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.log_readahead_size: 0 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_background_jobs: 2 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_background_compactions: -1 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T20:54:47.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T20:54:47.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-10T20:54:47.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T20:54:47.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T20:54:47.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_open_files: -1 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T20:54:47.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Compression algorithms supported: 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kZSTD supported: 0 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kXpressCompression supported: 0 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kLZ4Compression supported: 1 2023-06-10T20:54:47.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kBZip2Compression supported: 0 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kZlibCompression supported: 1 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: kSnappyCompression supported: 1 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.merge_operator: 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_filter: None 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T20:54:47.178 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5617f78ffd18) 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: cache_index_and_filter_blocks: 1 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: pin_top_level_index_and_filter: 1 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: index_type: 0 2023-06-10T20:54:47.179 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: data_block_index_type: 0 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: index_shortening: 1 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: data_block_hash_table_util_ratio: 0.750000 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: hash_index_allow_collision: 1 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: checksum: 1 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: no_block_cache: 0 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_cache: 0x5617f7938f10 2023-06-10T20:54:47.180 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_cache_name: BinnedLRUCache 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_cache_options: 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: capacity : 536870912 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: num_shard_bits : 4 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: strict_capacity_limit : 0 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: high_pri_pool_ratio: 0.000 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_cache_compressed: (nil) 2023-06-10T20:54:47.181 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: persistent_cache: (nil) 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_size: 4096 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_size_deviation: 10 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_restart_interval: 16 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: index_block_restart_interval: 1 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: metadata_block_size: 4096 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: partition_filters: 0 2023-06-10T20:54:47.182 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: use_delta_encoding: 1 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: whole_key_filtering: 1 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: verify_compression: 0 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: read_amp_bytes_per_bit: 0 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: format_version: 2 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: enable_index_compression: 1 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: block_align: 0 2023-06-10T20:54:47.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression: NoCompression 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T20:54:47.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.num_levels: 7 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T20:54:47.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-10T20:54:47.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T20:54:47.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T20:54:47.188 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-10T20:54:47.189 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T20:54:47.190 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T20:54:47.191 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.table_properties_collectors: 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.inplace_update_support: 0 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T20:54:47.192 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.bloom_locality: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.max_successive_merges: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.ttl: 2592000 2023-06-10T20:54:47.193 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.767+0000 7fb779135880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 bash[107081]: 22815652c7a963e7e13d14de7e03464901864a8e394ac5692be60df86500820e 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430486769551, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-06-10T20:54:47.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430486770011, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1686430486, "oldest_key_time": 3, "file_creation_time": 0}} 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.768+0000 7fb779135880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.769+0000 7fb779135880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1686430486770416, "job": 1, "event": "recovery_finished"} 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.769+0000 7fb779135880 4 rocksdb: DB pointer 0x5617f7a03800 2023-06-10T20:54:47.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.769+0000 7fb75ff40700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.769+0000 7fb75ff40700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** DB Stats ** 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T20:54:47.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** Compaction Stats [default] ** 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:47.197 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:47.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.198 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** Compaction Stats [default] ** 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:47.199 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** Compaction Stats [default] ** 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.201 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** Compaction Stats [default] ** 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2023-06-10T20:54:47.202 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: AddFile(Keys): cumulative 0, interval 0 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:47.203 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ** File Read Latency Histogram By Level [default] ** 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.769+0000 7fb779135880 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2023-06-10T20:54:47.204 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.770+0000 7fb779135880 0 using public_addrv [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.770+0000 7fb779135880 0 starting mon.c rank -1 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.771+0000 7fb779135880 1 mon.c@-1(???) e0 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 systemd[1]: Started Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 0 mon.c@-1(synchronizing).mds e1 new map 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 0 mon.c@-1(synchronizing).mds e1 print_map 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: e1 2023-06-10T20:54:47.205 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: legacy client fscid: -1 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: No filesystems configured 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-06-10T20:54:47.206 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.788+0000 7fb762745700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.789+0000 7fb762745700 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.789+0000 7fb762745700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:47.207 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.789+0000 7fb762745700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:47.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.789+0000 7fb762745700 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-06-10T20:54:47.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: cluster 2023-06-10T20:54:47.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: 2023-06-10T20:54:44.485032+0000 mgr.y (mgr.14148 2023-06-10T20:54:47.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: ) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:47.208 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:46 smithi064 conmon[107122]: debug 2023-06-10T20:54:46.789+0000 7fb762745700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-06-10T20:54:52.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:46.485274+0000 mgr.y (mgr.14148) 26 : 2023-06-10T20:54:52.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:46.796999+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:52.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:46.797215+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:46.800019+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-10T20:54:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:46.810508+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:46.810636+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:47.793969+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:48.485761+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023 2023-06-10T20:54:52.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: -06-10T20:54:48.793908+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:48.795516+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-10T20:54:52.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:49.794203+0000 mon.a (mon.0) 153 : audit [DBG] 2023-06-10T20:54:52.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:50.486038+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:50.794283+0000 mon.a (mon.0) 154 : audit [DBG] 2023-06-10T20:54:52.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:51 2023-06-10T20:54:52.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: .794444+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:54:52.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: -06-10T20:54:51.802613+0000 mon.a (mon.0) 156 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:51.827815+0000 mon.a (mon. 2023-06-10T20:54:52.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T20:54:52.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:54:52.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 10T20:54:51.833394+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T20:54:52.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54: 2023-06-10T20:54:52.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 51.833887+0000 mon.a (mon.0) 159 : cluster [DBG] 2023-06-10T20:54:52.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: fsmap 2023-06-10T20:54:52.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T20:54:52.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 20:54:51.833926+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:52.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 2023-06-10T20:54:52.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54 2023-06-10T20:54:52.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: :51.834137+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-06-10T20:54:52.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: cluster 2023-06-10T20:54:52.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: 2023-06-10T20:54:51.837564+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-06-10T20:54:52.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:51.841243+0000 2023-06-10T20:54:52.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: mon.a (mon.0) 163 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:52.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: audit 2023-06-10T20:54:51.842684+0000 mon.a (mon.0) 164 2023-06-10T20:54:52.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:52.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T20:54:46.485274+0000 mgr.y 2023-06-10T20:54:52.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:46.796999+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:52.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T20:54:46.797215+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T20:54:46.800019+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-10T20:54:52.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:46.810508+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:52.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:46.810636+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:47.793969+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T20:54:48.485761+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:48.793908 2023-06-10T20:54:52.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: +0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T20:54:48.795516+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-10T20:54:52.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:49.794203+0000 mon.a (mon.0 2023-06-10T20:54:52.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: ) 153 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:54:52.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: -06-10T20:54:50. 2023-06-10T20:54:52.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 486038+0000 mgr.y (mgr.14148) 28 : 2023-06-10T20:54:52.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:52.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:50.794283+0000 mon.a 2023-06-10T20:54:52.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: (mon.0) 154 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:52.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:51.794444+0000 mon.a (mon 2023-06-10T20:54:52.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: .0) 155 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T20:54:52.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: T20:54:51.802613+0000 mon.a (mon 2023-06-10T20:54:52.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: .0) 156 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T20:54:52.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:51.827815+0000 mon.a (mon 2023-06-10T20:54:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: .0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T20:54:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T20:54:52.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: T20:54:51.833394+0000 mon.a (mon. 2023-06-10T20:54:52.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T20:54:52.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06-10T 2023-06-10T20:54:52.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 20:54:51.833887+0000 mon.a (mon.0 2023-06-10T20:54:52.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: ) 159 : cluster [DBG] fsmap 2023-06-10T20:54:52.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T20:54:52.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:51.833926+0000 mon.a (mon 2023-06-10T20:54:52.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: .0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:52.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T20:54:52.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:51.834137+0000 mon.a (mon 2023-06-10T20:54:52.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: .0) 161 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-06-10T20:54:52.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:54:52.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: -06-10T20:54:51.837564+0000 2023-06-10T20:54:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-06-10T20:54:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 10T20:54:51.841243+0000 mon.a (mon.0) 2023-06-10T20:54:52.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 163 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:52.183 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: audit 2023-06-10T20:54:52.183 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: 2023-06-10T20:54:51.842684+0000 mon.a 2023-06-10T20:54:52.183 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:51 smithi064 conmon[97224]: (mon.0) 164 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:52.184 INFO:teuthology.orchestra.run.smithi152.stdout: 2023-06-10T20:54:52.185 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":3,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","modified":"2023-06-10T20:54:46.794314Z","created":"2023-06-10T20:53:13.526882Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3300","nonce":0},{"type":"v1","addr":"172.21.15.64:6789","nonce":0}]},"addr":"172.21.15.64:6789/0","public_addr":"172.21.15.64:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:3301","nonce":0},{"type":"v1","addr":"172.21.15.64:6790","nonce":0}]},"addr":"172.21.15.64:6790/0","public_addr":"172.21.15.64:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-06-10T20:54:52.185 INFO:teuthology.orchestra.run.smithi152.stderr:dumped monmap epoch 3 2023-06-10T20:54:52.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:46.485274+0000 mgr.y (mgr.14148) 26 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:46.796999+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:54:52.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:46.797215+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:46.800019+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-10T20:54:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:46.810508+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:54:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:46.810636+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:47.793969+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:48.485761+0000 mgr.y (mgr.14148) 27 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:48.793908+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: 2023-06-10T20:54:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:48.795516+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-10T20:54:52.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:49.794203+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:50.486038+0000 mgr.y (mgr.14148) 28 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:52.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:50.794283+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:51.794444+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.802613+0000 mon.a (mon.0) 156 : cluster [INF] mon.a calling monitor election 2023-06-10T20:54:52.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.827815+0000 mon.a (mon.0) 157 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T20:54:52.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.833394+0000 mon.a (mon.0) 158 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T20:54:52.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.833887+0000 mon.a (mon.0) 159 : cluster [DBG] fsmap 2023-06-10T20:54:52.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.833926+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-10T20:54:52.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.834137+0000 mon.a (mon.0) 161 : cluster [DBG] mgrmap e13: y(active, since 43s) 2023-06-10T20:54:52.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: cluster 2023-06-10T20:54:51.837564+0000 mon.a (mon.0) 162 : cluster [INF] overall HEALTH_OK 2023-06-10T20:54:52.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:51.841243+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:52.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:51 smithi152 conmon[97365]: audit 2023-06-10T20:54:51.842684+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:52.706 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-06-10T20:54:52.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph config generate-minimal-conf 2023-06-10T20:54:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:52 smithi152 conmon[97365]: audit 2023-06-10T20:54:52.166350+0000 mon.a (mon.0) 165 : audit 2023-06-10T20:54:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:52 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.152:0/3624096790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:52 smithi152 conmon[97365]: audit 2023-06-10T20:54:52.794459+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:52 smithi152 conmon[97365]: audit 2023-06-10T20:54:52.826124+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[107122]: audit 2023-06-10T20:54:52.166350+0000 2023-06-10T20:54:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[107122]: mon.a (mon.0) 165 : audit [DBG] from='client.? 172.21.15.152:0/3624096790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[107122]: audit 2023-06-10T20:54:52.794459+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[107122]: audit 2023-06-10T20:54:52.826124+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[97224]: audit 2023-06-10T20:54:52.166350 2023-06-10T20:54:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[97224]: +0000 mon.a (mon.0) 165 : audit [DBG] from='client.? 172.21.15.152:0/3624096790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T20:54:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[97224]: audit 2023-06-10T20:54:52.794459+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:54:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:52 smithi064 conmon[97224]: audit 2023-06-10T20:54:52.826124+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:54:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: :54:52.486300+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:53.037404+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:53.038058+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:53.038720+0000 mon.a (mon.0 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: ) 170 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: 10T20:54:53.064208+0000 mon.a ( 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: -06-10T20:54:53.101795+0000 mon.a (mon 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: .0) 172 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: 2023-06-10T20:54:53.105189+0000 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: mon.a (mon.0) 173 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:54:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: 20:54:53.106558+0000 mon.a (mon.0) 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: 174 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:53.107156 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: +0000 mon.a (mon.0) 175 : audit [DBG] 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: 20:54:53.107751+0000 mon.a (mon.0) 176 : audit 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: audit 2023-06-10T20:54:53 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: .654808+0000 mon.a (mon.0) 177 : audit [INF] 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: cluster 2023-06-10T20:54: 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 52.486300+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06-10T20:54:53.037404+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06-10T20:54:53.038058+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06-10T20:54: 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 53.038720+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 54:53.064208+0000 mon.a (mon.0) 171 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: -10T20:54:53.101795+0000 mon.a (mon. 2023-06-10T20:54:54.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 0) 172 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: -06-10T20:54:53.105189+0000 mon.a ( 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: mon.0) 173 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 06-10T20:54:53.106558 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: +0000 mon.a (mon.0) 174 : audit 2023-06-10T20:54:54.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: :54:53.107156+0000 mon.a (mon.0) 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 175 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: -10T20:54:53.107751+0000 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: mon.a (mon.0) 176 : audit [DBG] 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:54:54.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: 06-10T20:54:53.654808+0000 mon.a ( 2023-06-10T20:54:54.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: mon.0) 177 : audit [INF] 2023-06-10T20:54:54.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:53 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: cluster 2023-06-10T20:54:52.486300+0000 mgr.y (mgr.14148) 29 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53.037404+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53.038058+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53.038720+0000 mon.a (mon.0) 170 : audit 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53.064208 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: +0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 2023-06-10T20:54:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 2023-06-10T20:54:53.101795+0000 mon.a (mon.0 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: ) 172 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 20:54:53.105189+0000 mon.a (mon.0 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: ) 173 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 20:54:53.106558+0000 mon.a (mon.0) 2023-06-10T20:54:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 174 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 20:54:53.107156+0000 mon.a (mon.0 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: ) 175 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53. 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 107751+0000 mon.a (mon.0) 176 : audit [DBG] 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:54.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: audit 2023-06-10T20:54:53.654808+0000 2023-06-10T20:54:54.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: mon.a (mon.0) 177 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:54.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:53 smithi152 conmon[97365]: 2023-06-10T20:54:54.755 INFO:teuthology.orchestra.run.smithi064.stdout:# minimal ceph.conf for b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:54.756 INFO:teuthology.orchestra.run.smithi064.stdout:[global] 2023-06-10T20:54:54.756 INFO:teuthology.orchestra.run.smithi064.stdout: fsid = b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T20:54:54.756 INFO:teuthology.orchestra.run.smithi064.stdout: mon_host = [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] 2023-06-10T20:54:55.111 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:53.039649+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:53.039876+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:53.106335+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:53.108263+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi064 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[107122]: audit 2023-06-10T20:54:54.751905+0000 mon.a (mon.0) 178 : audit [DBG] from='client.? 172.21.15.64:0/1547192197' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:53.039649+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:54:55.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:53.039876+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:54:55.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:53.106335+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-10T20:54:55.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:53.108263+0000 mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi064 2023-06-10T20:54:55.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:54 smithi064 conmon[97224]: audit 2023-06-10T20:54:54.751905+0000 mon.a (mon.0) 178 : audit [DBG] from='client.? 172.21.15.64:0/1547192197' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:55.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:53.039649+0000 mgr.y (mgr.14148) 30 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: cephadm 2023-06-10 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: T20:54:53.039876+0000 mgr.y (mgr.14148) 31 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:53.106335+0000 mgr.y (mgr.14148) 32 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:53.108263+0000 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: mgr.y (mgr.14148) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi064 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: audit 2023 2023-06-10T20:54:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: -06-10T20:54:54.751905+0000 mon.a ( 2023-06-10T20:54:55.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:54 smithi152 conmon[97365]: mon.0) 178 : audit [DBG] from='client.? 172.21.15.64:0/1547192197' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:55.621 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-06-10T20:54:55.621 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:55.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.conf 2023-06-10T20:54:55.651 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:55.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:55.719 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:54:55.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.conf 2023-06-10T20:54:55.755 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:54:55.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:54:55.825 INFO:tasks.cephadm:Adding mgr.y on smithi064 2023-06-10T20:54:55.826 INFO:tasks.cephadm:Adding mgr.x on smithi152 2023-06-10T20:54:55.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply mgr '2;smithi064=y;smithi152=x' 2023-06-10T20:54:56.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:55 smithi152 conmon[97365]: cluster 2023-06-10T20:54:54.486504+0000 mgr.y (mgr.14148) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:55 smithi152 conmon[97365]: audit 2023-06-10T20:54:55.156705+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:55 smithi152 conmon[97365]: audit 2023-06-10T20:54:55.157488+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:55 smithi152 conmon[97365]: audit 2023-06-10T20:54:55.158102+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:55 smithi152 conmon[97365]: audit 2023-06-10T20:54:55.158692+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[97224]: cluster 2023-06-10T20:54:54.486504+0000 mgr.y (mgr.14148) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[97224]: audit 2023-06-10T20:54:55.156705+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[97224]: audit 2023-06-10T20:54:55.157488+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[97224]: audit 2023-06-10T20:54:55.158102+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[97224]: audit 2023-06-10T20:54:55.158692+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[107122]: cluster 2023-06-10T20:54:54.486504+0000 mgr.y (mgr.14148) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[107122]: audit 2023-06-10T20:54:55.156705+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[107122]: audit 2023-06-10T20:54:55.157488+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[107122]: audit 2023-06-10T20:54:55.158102+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:55 smithi064 conmon[107122]: audit 2023-06-10T20:54:55.158692+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:57.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:55.157250+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-10T20:54:57.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: cephadm 2023 2023-06-10T20:54:57.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: -06-10T20:54:55.159245+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi064 2023-06-10T20:54:57.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: audit 2023-06-10T20:54:56.592479+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:57.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: audit 2023-06-10T20:54:56.593228+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:57.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: audit 2023-06-10T20:54:56.593931+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:57.265 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:56 smithi152 conmon[97365]: audit 2023-06-10T20:54:56.594598+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:55.157250+0000 mgr.y (mgr. 2023-06-10T20:54:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: 14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-10T20:54:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:55.159245+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi064 2023-06-10T20:54:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: audit 2023-06-10T20:54:56.592479+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: audit 2023-06-10T20:54:56.593228+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: audit 2023-06-10T20:54:56.593931+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[97224]: audit 2023-06-10T20:54:56.594598+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:55 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: .157250+0000 mgr.y (mgr.14148) 35 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-10T20:54:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:55.159245+0000 mgr.y (mgr.14148) 36 : cephadm [INF] Reconfiguring daemon mon.c on smithi064 2023-06-10T20:54:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: audit 2023-06-10T20:54:56.592479+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: audit 2023-06-10T20:54:56.593228+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T20:54:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: audit 2023-06-10T20:54:56.593931+0000 mon.a (mon.0) 185 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T20:54:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:56 smithi064 conmon[107122]: audit 2023-06-10T20:54:56.594598+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:57.958 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled mgr update... 2023-06-10T20:54:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:57 smithi152 conmon[97365]: cluster 2023-06-10T20:54:56.486763+0000 mgr.y (mgr.14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:57 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:56.592961+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-10T20:54:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:57 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:56.595203+0000 mgr.y (mgr.14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi152 2023-06-10T20:54:58.400 DEBUG:teuthology.orchestra.run.smithi152:mgr.x> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service 2023-06-10T20:54:58.402 INFO:tasks.cephadm:Deploying OSDs... 2023-06-10T20:54:58.403 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:54:58.403 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/scratch_devs of=/dev/stdout 2023-06-10T20:54:58.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: cluster 2023-06-10T20:54:56.486763+0000 mgr.y ( 2023-06-10T20:54:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: mgr.14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:56.592961+0000 mgr.y (mgr.14148 2023-06-10T20:54:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: ) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-10T20:54:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:56.595203+0000 mgr.y ( 2023-06-10T20:54:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[97224]: mgr.14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi152 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: cluster 2023-06-10T20:54:56.486763+0000 mgr.y (mgr. 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: 14148) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:56. 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: 592961+0000 mgr.y (mgr.14148) 38 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: cephadm 2023-06 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: -10T20:54:56.595203+0000 mgr.y (mgr. 2023-06-10T20:54:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:57 smithi064 conmon[107122]: 14148) 39 : cephadm [INF] Reconfiguring daemon mon.b on smithi152 2023-06-10T20:54:58.424 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-06-10T20:54:58.424 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_1 2023-06-10T20:54:58.434 INFO:journalctl@ceph.mgr.x.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:54:58.485 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-06-10T20:54:58.485 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:58.485 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 374418 Links: 1 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-06-10 20:54:46.189711482 +0000 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-06-10 20:46:56.089451748 +0000 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-06-10 20:46:56.089451748 +0000 2023-06-10T20:54:58.486 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-06-10T20:54:58.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-06-10T20:54:58.557 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-06-10T20:54:58.558 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-06-10T20:54:58.558 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000288406 s, 1.8 MB/s 2023-06-10T20:54:58.559 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-06-10T20:54:58.619 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_2 2023-06-10T20:54:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-06-10T20:54:58.679 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 378315 Links: 1 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-06-10 20:54:46.189711482 +0000 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-06-10 20:46:56.619441374 +0000 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-06-10 20:46:56.619441374 +0000 2023-06-10T20:54:58.680 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-06-10T20:54:58.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-06-10T20:54:58.750 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-06-10T20:54:58.750 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-06-10T20:54:58.750 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000320649 s, 1.6 MB/s 2023-06-10T20:54:58.751 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-06-10T20:54:58.810 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_3 2023-06-10T20:54:58.869 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-06-10T20:54:58.870 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:58.870 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 376669 Links: 1 2023-06-10T20:54:58.870 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:58.870 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:58.870 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-06-10 20:54:46.189711482 +0000 2023-06-10T20:54:58.871 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-06-10 20:46:57.155430883 +0000 2023-06-10T20:54:58.871 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-06-10 20:46:57.155430883 +0000 2023-06-10T20:54:58.871 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-06-10T20:54:58.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-06-10T20:54:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-06-10T20:54:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-06-10T20:54:58.938 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000241187 s, 2.1 MB/s 2023-06-10T20:54:58.940 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-06-10T20:54:58.999 DEBUG:teuthology.orchestra.run.smithi064:> stat /dev/vg_nvme/lv_4 2023-06-10T20:54:59.057 INFO:teuthology.orchestra.run.smithi064.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-06-10T20:54:59.057 INFO:teuthology.orchestra.run.smithi064.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:59.057 INFO:teuthology.orchestra.run.smithi064.stdout:Device: 6h/6d Inode: 372227 Links: 1 2023-06-10T20:54:59.057 INFO:teuthology.orchestra.run.smithi064.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:59.058 INFO:teuthology.orchestra.run.smithi064.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:59.058 INFO:teuthology.orchestra.run.smithi064.stdout:Access: 2023-06-10 20:54:46.189711482 +0000 2023-06-10T20:54:59.058 INFO:teuthology.orchestra.run.smithi064.stdout:Modify: 2023-06-10 20:46:57.682420568 +0000 2023-06-10T20:54:59.058 INFO:teuthology.orchestra.run.smithi064.stdout:Change: 2023-06-10 20:46:57.682420568 +0000 2023-06-10T20:54:59.058 INFO:teuthology.orchestra.run.smithi064.stdout: Birth: - 2023-06-10T20:54:59.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-06-10T20:54:59.123 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records in 2023-06-10T20:54:59.123 INFO:teuthology.orchestra.run.smithi064.stderr:1+0 records out 2023-06-10T20:54:59.123 INFO:teuthology.orchestra.run.smithi064.stderr:512 bytes copied, 0.000196387 s, 2.6 MB/s 2023-06-10T20:54:59.124 DEBUG:teuthology.orchestra.run.smithi064:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-06-10T20:54:59.182 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:54:59.182 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/scratch_devs of=/dev/stdout 2023-06-10T20:54:59.199 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-06-10T20:54:59.199 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_1 2023-06-10T20:54:59.256 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-06-10T20:54:59.256 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 6h/6d Inode: 377904 Links: 1 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2023-06-10 20:54:37.584180022 +0000 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2023-06-10 20:46:43.277924330 +0000 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2023-06-10 20:46:43.277924330 +0000 2023-06-10T20:54:59.257 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: - 2023-06-10T20:54:59.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-06-10T20:54:59.319 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.947883+0000 mgr.y (mgr. 2023-06-10T20:54:59.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: 14148) 40 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:59.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:57.950144+0000 mgr.y (mgr.14148) 41 : cephadm [INF] Saving service mgr spec with placement smithi064=y;smithi152=x;count:2 2023-06-10T20:54:59.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.953678+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.986398+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.987488+0000 mon.a (mon.0) 189 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.989292+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.990219+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.995570+0000 mon.a (mon.0) 192 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:57.997142+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:58.000499+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:58.001367+0000 mon.a (mon.0) 195 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:54:59.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:58 smithi152 conmon[97365]: audit 2023-06-10T20:54:58.002433+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.322 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2023-06-10T20:54:59.322 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2023-06-10T20:54:59.323 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000219815 s, 2.3 MB/s 2023-06-10T20:54:59.323 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-06-10T20:54:59.382 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_2 2023-06-10T20:54:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:57.947883+0000 mgr.y (mgr.14148 2023-06-10T20:54:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: ) 40 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:57.950144+0000 mgr.y (mgr.14148) 41 : cephadm [INF] Saving service mgr spec with placement smithi064=y;smithi152=x;count:2 2023-06-10T20:54:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:57. 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 953678+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: T20:54:57.986398+0000 mon.a (mon.0) 188 : audit 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 10T20:54:57.987488+0000 mon.a (mon 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: .0) 189 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: -06-10T20:54:57.989292+0000 mon.a ( 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 2023-06-10T20:54:57.990219+0000 mon.a 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: (mon.0) 191 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 2023-06-10T20:54:57.995570+0000 mon.a 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: (mon.0) 192 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 2023-06-10T20:54:57.997142+0000 mon.a 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 2023-06-10T20 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: :54:58.000499+0000 mon.a (mon.0) 2023-06-10T20:54:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 194 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-10T20:54:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:54:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 06-10T20:54:58.001367+0000 mon.a (mon.0) 2023-06-10T20:54:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 195 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:54:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: audit 2023-06-10T20:54:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: 2023-06-10T20:54:58.002433 2023-06-10T20:54:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: +0000 mon.a (mon.0) 196 : audit [DBG] 2023-06-10T20:54:59.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10T20:54 2023-06-10T20:54:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: :57.947883+0000 mgr.y (mgr.14148) 40 : audit [DBG] from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi064=y;smithi152=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:54:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:57.950144+0000 mgr.y (mgr.14148) 41 : 2023-06-10T20:54:59.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: cephadm [INF] Saving service mgr spec with placement smithi064=y;smithi152=x;count:2 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: T20:54:57.953678+0000 mon.a 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: (mon.0) 187 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 20:54:57.986398+0000 mon.a 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: (mon.0) 188 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 20:54:57.987488+0000 mon.a ( 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: mon.0) 189 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: :54:57.989292+0000 mon.a ( 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: mon.0) 190 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: T20:54:57.990219+0000 mon.a (mon.0) 191 : 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 10T20:54:57.995570 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: +0000 mon.a (mon.0) 192 : audit [INF] 2023-06-10T20:54:59.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: -10T20:54:57.997142+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 20:54:58.000499+0000 mon.a (mon.0) 194 : audit [INF] 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023 2023-06-10T20:54:59.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: -06-10T20:54:58.001367+0000 mon.a (mon.0 2023-06-10T20:54:59.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: ) 195 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:54:59.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: audit 2023 2023-06-10T20:54:59.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: -06-10T20:54:58 2023-06-10T20:54:59.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: .002433+0000 mon.a (mon.0) 2023-06-10T20:54:59.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:58 smithi064 conmon[107122]: 196 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:54:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 6h/6d Inode: 364496 Links: 1 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2023-06-10 20:54:37.584180022 +0000 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2023-06-10 20:46:43.789918680 +0000 2023-06-10T20:54:59.441 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2023-06-10 20:46:43.789918680 +0000 2023-06-10T20:54:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: - 2023-06-10T20:54:59.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-06-10T20:54:59.515 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2023-06-10T20:54:59.515 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2023-06-10T20:54:59.516 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000233423 s, 2.2 MB/s 2023-06-10T20:54:59.517 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-06-10T20:54:59.576 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_3 2023-06-10T20:54:59.634 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-06-10T20:54:59.634 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:59.634 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 6h/6d Inode: 380978 Links: 1 2023-06-10T20:54:59.634 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:59.635 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:59.635 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2023-06-10 20:54:59.566927390 +0000 2023-06-10T20:54:59.635 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2023-06-10 20:46:44.314912886 +0000 2023-06-10T20:54:59.635 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2023-06-10 20:46:44.314912886 +0000 2023-06-10T20:54:59.635 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: - 2023-06-10T20:54:59.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-06-10T20:54:59.703 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2023-06-10T20:54:59.703 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2023-06-10T20:54:59.704 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000214649 s, 2.4 MB/s 2023-06-10T20:54:59.704 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-06-10T20:54:59.762 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_4 2023-06-10T20:54:59.820 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-06-10T20:54:59.820 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-10T20:54:59.820 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 6h/6d Inode: 381164 Links: 1 2023-06-10T20:54:59.820 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-10T20:54:59.820 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2023-06-10T20:54:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2023-06-10 20:54:59.566927390 +0000 2023-06-10T20:54:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2023-06-10 20:46:44.817907336 +0000 2023-06-10T20:54:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2023-06-10 20:46:44.817907336 +0000 2023-06-10T20:54:59.821 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: - 2023-06-10T20:54:59.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-06-10T20:54:59.883 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:54:59 smithi152 systemd[1]: Starting Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:54:59.885 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2023-06-10T20:54:59.885 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2023-06-10T20:54:59.885 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000224915 s, 2.3 MB/s 2023-06-10T20:54:59.886 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-06-10T20:54:59.944 INFO:tasks.cephadm:Deploying osd.0 on smithi064 with /dev/vg_nvme/lv_4... 2023-06-10T20:54:59.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-06-10T20:55:00.163 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:59 smithi152 conmon[97365]: cephadm 2023-06-10T20:54:58.003313+0000 2023-06-10T20:55:00.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:59 smithi152 conmon[97365]: mgr.y (mgr.14148) 42 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T20:55:00.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:54:59 smithi152 conmon[97365]: cluster 2023-06-10T20:54:58.487064+0000 mgr.y (mgr.14148) 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[97224]: cephadm 2023-06-10T20:54:58.003313+0000 mgr.y (mgr. 2023-06-10T20:55:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[97224]: 14148) 42 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T20:55:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[97224]: cluster 2023-06-10T20:54:58.487064+0000 mgr.y (mgr.14148) 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:00.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[97224]: 2023-06-10T20:55:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[107122]: cephadm 2023-06-10T20:54:58.003313+0000 mgr.y 2023-06-10T20:55:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[107122]: (mgr.14148) 42 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T20:55:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:55:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:54:59 smithi064 conmon[107122]: 10T20:54:58.487064+0000 mgr.y (mgr.14148) 43 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:00.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 conmon[99677]: debug 2023-06-10T20:55:00.159+0000 7f733ed83700 1 -- 172.21.15.152:0/383500729 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d2356f24e0 con 0x55d2356c9400 2023-06-10T20:55:00.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 bash[99589]: 9977c174f9de461041ad64dd7e78064f725235659cf465e801bdb717916c0917 2023-06-10T20:55:00.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 systemd[1]: Started Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:55:00.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 conmon[99677]: debug 2023-06-10T20:55:00.289+0000 7f734f366680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:55:00.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 conmon[99677]: debug 2023-06-10T20:55:00.372+0000 7f734f366680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:55:01.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:00 smithi152 conmon[99677]: debug 2023-06-10T20:55:00.794+0000 7f734f366680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:55:01.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:00.230050+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.448 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:00.231827+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: cluster 2023-06-10T20:55:00.487344+0000 mgr.y (mgr.14148) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.087686+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.088316+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.089076+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.095487+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.097328+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.098048+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[107122]: audit 2023-06-10T20:55:01.098656+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:00.230050 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: +0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:00.231827+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:01.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: cluster 2023-06-10T20:55:00.487344+0000 mgr.y (mgr.14148) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:01.087686+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:01.088316+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:01.089076+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55:01 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: .095487+0000 mon.a (mon.0) 202 : 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20:55 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: :01.097328+0000 mon.a (mon.0) 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: 203 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: 55:01.098048+0000 mon.a (mon.0) 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: 204 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:55:01.452 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: :55:01.098656+0000 mon.a (mon.0 2023-06-10T20:55:01.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:01 smithi064 conmon[97224]: ) 205 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:00.230050+0000 mon.a ( 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: mon.0) 197 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:00.231827+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: cluster 2023-06-10T20:55:00.487344+0000 mgr.y (mgr.14148) 44 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.087686+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.088316+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.089076+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:01.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.095487+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:01.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.097328+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T20:55:01.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.098048+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T20:55:01.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[97365]: audit 2023-06-10T20:55:01.098656+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:01.853 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[99677]: debug 2023-06-10T20:55:01.556+0000 7f734f366680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:55:01.854 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[99677]: debug 2023-06-10T20:55:01.639+0000 7f734f366680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:55:01.965 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-06-10T20:55:01.965 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-06-10T20:55:01.966 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-06-10T20:55:01.966 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-06-10T20:55:01.966 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110564 s, 948 MB/s 2023-06-10T20:55:01.966 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-06-10T20:55:02.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:01 smithi152 conmon[99677]: debug 2023-06-10T20:55:01.849+0000 7f734f366680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:55:02.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:02 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:01.097066+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-10T20:55:02.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:02 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:01.099245+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi064 2023-06-10T20:55:02.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:02 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:01.097066+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-10T20:55:02.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:02 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:01.099245+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi064 2023-06-10T20:55:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:01.097066+0000 mgr.y (mgr.14148) 45 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-10T20:55:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[97365]: cephadm 2023-06-10T20:55: 2023-06-10T20:55:02.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[97365]: 01.099245+0000 mgr.y (mgr.14148) 46 : cephadm [INF] Reconfiguring daemon mgr.y on smithi064 2023-06-10T20:55:02.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi064:vg_nvme/lv_4 2023-06-10T20:55:03.086 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[99677]: debug 2023-06-10T20:55:02.789+0000 7f734f366680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:55:03.086 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[99677]: debug 2023-06-10T20:55:02.916+0000 7f734f366680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:55:03.086 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:02 smithi152 conmon[99677]: debug 2023-06-10T20:55:02.993+0000 7f734f366680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:55:03.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[99677]: debug 2023-06-10T20:55:03.082+0000 7f734f366680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:55:03.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[99677]: debug 2023-06-10T20:55:03.277+0000 7f734f366680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:55:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: cluster 2023-06-10T20:55:02 2023-06-10T20:55:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: .487689+0000 mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:02.796690+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:02.797773+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:02.799139+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:02.799851+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:02.803380+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[97224]: audit 2023-06-10T20:55:03.658930+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: cluster 2023-06-10T20:55 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: :02.487689+0000 mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:02.796690+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:02.797773+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:02.799139+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:02.799851+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:03.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:02.803380+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:03 smithi064 conmon[107122]: audit 2023-06-10T20:55:03.658930+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[99677]: debug 2023-06-10T20:55:03.551+0000 7f734f366680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:55:03.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[99677]: debug 2023-06-10T20:55:03.649+0000 7f734f366680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:55:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: cluster 2023-06-10T20:55:02.487689+0000 mgr.y (mgr.14148) 47 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023-06-10T20:55: 2023-06-10T20:55:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 02.796690+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023-06-10T20:55:02.797773+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 2023-06-10T20:55:02.799139+0000 mon.a (mon.0) 208 : audit 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023-06-10T20:55: 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 02.799851+0000 mon.a (mon.0) 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 209 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 2023-06-10T20:55: 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 02.803380+0000 mon.a (mon.0) 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 210 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 06-10T20:55:03. 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 658930+0000 mon.a (mon.0) 2023-06-10T20:55:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:03 smithi152 conmon[97365]: 211 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:04.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[99677]: debug 2023-06-10T20:55:04.364+0000 7f734f366680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:55:04.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[99677]: debug 2023-06-10T20:55:04.445+0000 7f734f366680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:55:04.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[99677]: debug 2023-06-10T20:55:04.535+0000 7f734f366680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:55:04.970 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[99677]: debug 2023-06-10T20:55:04.705+0000 7f734f366680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:55:04.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[97365]: audit 2023-06-10T20:55:04.617575+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:04.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[97365]: audit 2023-06-10T20:55 2023-06-10T20:55:04.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[97365]: :04.619821+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:04.970 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[97365]: audit 2023-06-10T20:55:04.620397+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:05.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[97224]: audit 2023-06-10T20:55:04.617575+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:05.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[97224]: audit 2023-06-10T20:55:04.619821+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:05.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[97224]: audit 2023-06-10T20:55:04.620397+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:05.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[107122]: audit 2023-06-10T20:55:04.617575+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:05.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[107122]: audit 2023-06-10T20:55:04.619821+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:05.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:04 smithi064 conmon[107122]: audit 2023-06-10T20:55:04.620397+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:05.229 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:04 smithi152 conmon[99677]: debug 2023-06-10T20:55:04.965+0000 7f734f366680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:55:05.229 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[99677]: debug 2023-06-10T20:55:05.094+0000 7f734f366680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:55:05.480 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[99677]: debug 2023-06-10T20:55:05.225+0000 7f734f366680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:55:05.481 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[99677]: debug 2023-06-10T20:55:05.394+0000 7f734f366680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:55:05.805 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[99677]: debug 2023-06-10T20:55:05.476+0000 7f734f366680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:55:06.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: cluster 2023-06-10T20:55:04.487959 2023-06-10T20:55:06.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:06.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: audit 2023-06-10T20:55:04.614753+0000 mgr.y (mgr.14148) 49 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:06.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: cluster 2023-06-10T20:55:05.491585+0000 mon.a (mon.0) 215 : cluster [DBG] Standby manager daemon x started 2023-06-10T20:55:06.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: audit 2023-06-10T20:55:05. 2023-06-10T20:55:06.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 492485+0000 mon.b (mon.1) 3 : audit 2023-06-10T20:55:06.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: audit 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 2023-06-10T20:55 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: :05.492796+0000 mon.b (mon.1) 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 4 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 10T20:55:05.493725+0000 mon.b ( 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:55:06.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 06-10T20:55:05. 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 493983+0000 mon.b (mon.1) 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[97224]: 6 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: cluster 2023-06-10T20:55:04. 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 487959+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:06.135 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: audit 2023-06-10T20:55:04. 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 614753+0000 mgr.y (mgr.14148) 49 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: cluster 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 2023-06-10T20:55:05.491585+0000 mon.a ( 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: mon.0) 215 : cluster [DBG] Standby manager daemon x started 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: -10T20:55:05.492485+0000 mon.b ( 2023-06-10T20:55:06.136 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: T20:55:05.492796+0000 mon.b (mon. 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 1) 4 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: :55:05.493725+0000 mon.b (mon.1 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: ) 5 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:55:06.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:55:06.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: 55:05.493983+0000 mon.b (mon.1) 6 2023-06-10T20:55:06.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:05 smithi064 conmon[107122]: : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:55:06.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: cluster 2023-06-10T20:55:04.487959+0000 mgr.y (mgr.14148) 48 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: audit 2023-06-10T20:55:04.614753+0000 mgr.y (mgr.14148) 49 : audit [DBG] from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: cluster 2023-06-10T20:55:05.491585+0000 mon.a (mon 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: .0) 215 : cluster [DBG] Standby manager daemon x started 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: T20:55:05.492485+0000 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:55:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: audit 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: 2023-06-10T20:55:05.492796+0000 mon.b 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: -10T20:55:05.493725+0000 mon.b (mon 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: .1) 5 : audit [DBG] from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: audit 2023-06-10T20:55:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: 2023-06-10T 2023-06-10T20:55:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: 20:55:05.493983+0000 mon.b 2023-06-10T20:55:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: (mon.1) 6 : audit [DBG] 2023-06-10T20:55:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:05 smithi152 conmon[97365]: from='mgr.? 172.21.15.152:0/2833820742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:55:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[97224]: cluster 2023-06-10T20:55:05.810780+0000 mon.a ( 2023-06-10T20:55:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[97224]: mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-06-10T20:55:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[97224]: audit 2023-06-10T20:55:05.810919+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:55:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[107122]: cluster 2023-06-10T20:55:05 2023-06-10T20:55:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[107122]: .810780+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-06-10T20:55:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:06 smithi064 conmon[107122]: audit 2023-06-10T20:55:05.810919+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:55:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:06 smithi152 conmon[97365]: cluster 2023-06-10T20:55:05.810780+0000 mon.a (mon.0) 216 : cluster [DBG] mgrmap e14: y(active, since 57s), standbys: x 2023-06-10T20:55:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:06 smithi152 conmon[97365]: audit 2023-06-10T20:55:05.810919+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:55:08.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: cluster 2023-06-10T20:55:06.488339+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:08.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: audit 2023-06-10T20:55:06.854432+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.64:0/2006757221' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: audit 2023-06-10T20:55:06.854723+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: audit 2023-06-10T20:55:08.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: 2023-06-10T20:55:06.857874+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]': finished 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: cluster 2023-06-10T20:55:06.857921+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: audit 2023-06-10T20:55:06.858007+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[97224]: audit 2023-06-10T20:55:07.449963+0000 mon.a (mon.0) 222 : audit [DBG] from='client.? 172.21.15.64:0/867677276' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: cluster 2023-06-10T20:55:06.488339+0000 mgr.y (mgr.14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: audit 2023-06-10T20:55:06.854432+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.64:0/2006757221' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: audit 2023-06-10T20:55:06.854723+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: audit 2023-06-10T20:55:06.857874+0000 mon.a (mon.0) 219 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]': finished 2023-06-10T20:55:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: cluster 2023-06-10T20:55:06.857921+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-10T20:55:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: audit 2023-06-10T20:55:06.858007+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:07 smithi064 conmon[107122]: audit 2023-06-10T20:55:07.449963+0000 mon.a (mon.0) 222 : audit [DBG] from='client.? 172.21.15.64:0/867677276' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: cluster 2023-06-10T20:55:06.488339+0000 mgr.y (mgr. 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 14148) 50 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: audit 2023-06-10T20:55:06.854432+0000 mon.c (mon.2) 2 : audit [INF] 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: from='client.? 172.21.15.64:0/2006757221' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: T20:55:06.854723+0000 mon.a (mon.0) 218 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]: dispatch 2023-06-10T20:55:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 10T20:55:06.857874+0000 mon.a (mon.0) 219 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17854ddb-2190-4089-8177-63bd7a59d79a"}]': finished 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: cluster 2023-06-10T20:55: 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 06.857921+0000 mon.a (mon.0) 220 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: audit 2023-06-10T20:55:06.858007 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: +0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: audit 2023-06-10T20:55:08.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 2023-06-10T20:55:07.449963+0000 mon.a (mon. 2023-06-10T20:55:08.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:07 smithi152 conmon[97365]: 0) 222 : audit [DBG] from='client.? 172.21.15.64:0/867677276' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:09 smithi064 conmon[97224]: cluster 2023-06-10T20:55:08.488646+0000 mgr.y (mgr.14148 2023-06-10T20:55:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:09 smithi064 conmon[97224]: ) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:10.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:09 smithi064 conmon[107122]: cluster 2023-06-10T20:55:08.488646+0000 2023-06-10T20:55:10.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:09 smithi064 conmon[107122]: mgr.y (mgr.14148) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:09 smithi152 conmon[97365]: cluster 2023-06-10T20:55:08.488646+0000 mgr.y (mgr.14148 2023-06-10T20:55:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:09 smithi152 conmon[97365]: ) 51 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:12.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:11 smithi064 conmon[97224]: cluster 2023-06-10T20:55:10.488924+0000 mgr.y (mgr.14148) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:12.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:11 smithi064 conmon[107122]: cluster 2023-06-10T20:55:10.488924 2023-06-10T20:55:12.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:11 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:11 smithi152 conmon[97365]: cluster 2023-06-10T20:55:10.488924+0000 mgr.y (mgr. 2023-06-10T20:55:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:11 smithi152 conmon[97365]: 14148) 52 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[97224]: audit 2023-06-10T20:55:12.881708+0000 mon.a (mon.0) 223 2023-06-10T20:55:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T20:55:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[97224]: audit 2023-06-10T20:55:12.882434+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[107122]: audit 2023-06-10T20:55:12.881708+0000 mon.a 2023-06-10T20:55:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[107122]: (mon.0) 223 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T20:55:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:12 smithi064 conmon[107122]: audit 2023-06-10T20:55:12.882434+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:12 smithi152 conmon[97365]: audit 2023-06-10T20:55:12.881708+0000 2023-06-10T20:55:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:12 smithi152 conmon[97365]: mon.a (mon.0) 223 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T20:55:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:12 smithi152 conmon[97365]: audit 2023-06-10T20:55 2023-06-10T20:55:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:12 smithi152 conmon[97365]: :12.882434+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:14.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[97224]: cluster 2023-06-10T20:55:12.489196+0000 mgr.y (mgr 2023-06-10T20:55:14.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[97224]: .14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:14.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:12.883005+0000 mgr.y (mgr.14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi064 2023-06-10T20:55:14.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[107122]: cluster 2023-06-10T20:55:12.489196+0000 mgr.y (mgr. 2023-06-10T20:55:14.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[107122]: 14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:14.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:12.883005+0000 mgr.y (mgr 2023-06-10T20:55:14.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:13 smithi064 conmon[107122]: .14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi064 2023-06-10T20:55:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:13 smithi152 conmon[97365]: cluster 2023-06-10T20:55:12.489196+0000 2023-06-10T20:55:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:13 smithi152 conmon[97365]: mgr.y (mgr.14148) 53 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:13 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:12.883005+0000 2023-06-10T20:55:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:13 smithi152 conmon[97365]: mgr.y (mgr.14148) 54 : cephadm [INF] Deploying daemon osd.0 on smithi064 2023-06-10T20:55:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:15 smithi064 conmon[97224]: cluster 2023-06-10T20:55:14 2023-06-10T20:55:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:15 smithi064 conmon[97224]: .489428+0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:15 smithi064 conmon[107122]: cluster 2023-06-10T20:55:14.489428 2023-06-10T20:55:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:15 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:15 smithi152 conmon[97365]: cluster 2023-06-10T20:55:14.489428+0000 mgr.y (mgr.14148) 2023-06-10T20:55:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:15 smithi152 conmon[97365]: 55 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[97224]: cluster 2023-06-10T20:55:16.489645+0000 mgr.y (mgr.14148) 56 2023-06-10T20:55:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[97224]: : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[97224]: audit 2023-06-10T20:55:17.776814+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:18.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[97224]: audit 2023-06-10T20:55:17.797100+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: cluster 2023-06-10T20:55:16.489645 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: -10T20:55:17.776814+0000 mon.a (mon.0) 225 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: 06-10T20:55:17.797100+0000 mon.a (mon.0 2023-06-10T20:55:18.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:17 smithi064 conmon[107122]: ) 226 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: cluster 2023-06-10T20:55:16.489645+0000 mgr.y (mgr.14148 2023-06-10T20:55:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: ) 56 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: audit 2023-06-10T20:55:17.776814+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: 2023-06-10T20:55:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: audit 2023-06-10T20:55:17.797100 2023-06-10T20:55:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:17 smithi152 conmon[97365]: +0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:20.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: cluster 2023-06-10T20:55:18.489877+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:20.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: audit 2023-06-10T20:55:19.754131+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: audit 2023-06-10T20:55:19.754791+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:20.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: audit 2023-06-10T20:55:19.755418+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:20.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: audit 2023-06-10T20:55:20.354 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[97224]: 2023-06-10T20:55:19.763078+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[107122]: cluster 2023-06-10T20:55:18.489877+0000 mgr.y (mgr.14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:20.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[107122]: audit 2023-06-10T20:55:19.754131+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[107122]: audit 2023-06-10T20:55:19.754791+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:20.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[107122]: audit 2023-06-10T20:55:19.755418+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:20.355 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:19 smithi064 conmon[107122]: audit 2023-06-10T20:55:19.763078+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: cluster 2023-06-10T20:55:18.489877+0000 mgr.y (mgr. 2023-06-10T20:55:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: 14148) 57 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: audit 2023-06-10T20:55:19.754131+0000 mon.a (mon.0) 227 : audit [INF] 2023-06-10T20:55:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: 55:19.754791+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: T20:55:19.755418+0000 mon.a (mon.0) 229 : 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: audit 2023-06-10T20:55:19. 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: 763078+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:19 smithi152 conmon[97365]: 2023-06-10T20:55:20.798 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 0 on host 'smithi064' 2023-06-10T20:55:21.350 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0.service 2023-06-10T20:55:21.352 INFO:tasks.cephadm:Deploying osd.1 on smithi064 with /dev/vg_nvme/lv_3... 2023-06-10T20:55:21.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-06-10T20:55:21.403 INFO:journalctl@ceph.osd.0.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:55:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[97224]: cluster 2023-06-10T20:55:20.490190+0000 mgr.y (mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[97224]: audit 2023-06-10T20:55:20.793421+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[97224]: audit 2023-06-10T20:55:20.830968+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[97224]: audit 2023-06-10T20:55:20.971307+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T20:55:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[107122]: cluster 2023-06-10T20:55:20.490190+0000 mgr.y (mgr.14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[107122]: audit 2023-06-10T20:55:20.793421+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[107122]: audit 2023-06-10T20:55:20.830968+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[107122]: audit 2023-06-10T20:55:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:21 smithi064 conmon[107122]: 2023-06-10T20:55:20.971307+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: cluster 2023-06-10T20:55:20.490190+0000 mgr.y (mgr. 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: 14148) 58 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: audit 2023-06-10T20:55:20.793421+0000 mon.a (mon.0) 231 : audit 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:55:22.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: 20:55:20.830968+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:22.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: audit 2023-06-10T20:55:22.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: 2023-06-10T20:55:20.971307+0000 mon.a (mon.0 2023-06-10T20:55:22.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:21 smithi152 conmon[97365]: ) 233 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T20:55:23.253 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[112223]: debug 2023-06-10T20:55:22.971+0000 7f2889049700 -1 osd.0 0 waiting for initial osdmap 2023-06-10T20:55:23.254 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[112223]: debug 2023-06-10T20:55:22.974+0000 7f2881fbf700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:55:23.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[97224]: audit 2023-06-10T20:55:21.967373+0000 mon.a (mon.0) 2023-06-10T20:55:23.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[97224]: 234 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T20:55:23.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[97224]: cluster 2023-06-10T20:55:21.967448+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[97224]: audit 2023-06-10T20:55:21.967580+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[97224]: audit 2023-06-10T20:55:21.967941+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[107122]: audit 2023-06-10T20:55:21.967373+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[107122]: cluster 2023-06-10T20:55:21.967448+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[107122]: audit 2023-06-10T20:55:21.967580+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:23.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:22 smithi064 conmon[107122]: audit 2023-06-10T20:55:21.967941+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: audit 2023-06-10T20:55:21.967373+0000 mon.a 2023-06-10T20:55:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: (mon.0) 234 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T20:55:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: cluster 2023-06-10T20:55:21.967448+0000 mon.a ( 2023-06-10T20:55:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-10T20:55:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: audit 2023-06-10T20:55:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: 2023-06-10T20:55:21.967580+0000 mon.a (mon.0) 236 : audit [DBG] 2023-06-10T20:55:23.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:23.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: audit 2023-06-10T20:55:21.967941+0000 2023-06-10T20:55:23.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:22 smithi152 conmon[97365]: mon.a (mon.0) 237 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:23.992 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-06-10T20:55:23.992 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-06-10T20:55:23.992 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-06-10T20:55:23.993 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-06-10T20:55:23.993 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115046 s, 911 MB/s 2023-06-10T20:55:23.993 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-06-10T20:55:24.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: cluster 2023-06-10T20:55:21.935310+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:24.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: cluster 2023-06-10T20:55:21.935444+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: cluster 2023-06-10T20:55:22.490552+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: audit 2023-06-10T20:55:22.968610+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: cluster 2023-06-10T20:55:22.968697+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: audit 2023-06-10T20:55:22.969146+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[107122]: audit 2023-06-10T20:55:22.973329+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: cluster 2023-06-10T20:55:21.935310+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: cluster 2023-06-10T20:55:21.935444+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: cluster 2023-06-10T20:55:22.490552+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: audit 2023-06-10T20:55:22.968610+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: cluster 2023-06-10T20:55:22.968697+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: audit 2023-06-10T20:55:22.969146+0000 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: mon.a (mon.0) 240 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:23 smithi064 conmon[97224]: audit 2023-06-10T20:55:22.973329+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: cluster 2023-06-10T20:55:21.935310+0000 osd.0 2023-06-10T20:55:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: cluster 2023-06-10T20:55:21.935444+0000 osd.0 (osd 2023-06-10T20:55:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: .0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: cluster 2023 2023-06-10T20:55:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: -06-10T20:55:22.490552+0000 mgr.y (mgr.14148) 59 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: audit 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: 2023-06-10T20:55:22.968610+0000 mon.a (mon.0) 238 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: : audit [INF] from='osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: 55:22.968697+0000 mon.a (mon.0) 239 : cluster 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-10T20:55:24.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: audit 2023-06-10T20:55:22 2023-06-10T20:55:24.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: .969146+0000 mon.a (mon.0) 240 : audit [DBG] 2023-06-10T20:55:24.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: audit 2023-06-10T20:55:22. 2023-06-10T20:55:24.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: 973329+0000 mon.a (mon.0) 241 : audit [DBG] 2023-06-10T20:55:24.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:23 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:24.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi064:vg_nvme/lv_3 2023-06-10T20:55:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[97224]: audit 2023-06-10T20:55:23.974851+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[97224]: cluster 2023-06-10T20:55:23.978633+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417] boot 2023-06-10T20:55:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[97224]: cluster 2023-06-10T20:55:23.978681+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-10T20:55:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[97224]: audit 2023-06-10T20:55:23.979058+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: audit 2023-06-10T20:55:23. 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: 974851+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: cluster 2023-06-10T20:55:23.978633+0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417] boot 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: cluster 2023-06-10T20:55:23.978681+0000 mon.a (mon.0) 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: -10T20:55:23.979058+0000 mon.a (mon 2023-06-10T20:55:25.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:24 smithi064 conmon[107122]: .0) 245 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: audit 2023-06-10T20:55:23.974851+0000 mon.a 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: (mon.0) 242 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: cluster 2023-06-10T20:55:23.978633 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: +0000 mon.a (mon.0) 243 : cluster [INF] osd.0 [v2:172.21.15.64:6802/2269841417,v1:172.21.15.64:6803/2269841417] boot 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: cluster 2023-06-10T20:55:23.978681 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: +0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: audit 2023-06-10T20:55:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: 2023-06-10T20:55:23.979058+0000 mon.a (mon 2023-06-10T20:55:25.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:24 smithi152 conmon[97365]: .0) 245 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:55:26.150 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: cluster 2023-06-10T20:55:24.490798+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: cluster 2023-06-10T20:55:24.986630+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023-06-10T20:55:25.138151+0000 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: -06-10T20:55:25.470545+0000 mon.a ( 2023-06-10T20:55:26.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: mon.0) 248 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: -10T20:55:25.477557+0000 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: mon.a (mon.0) 249 : audit [INF] 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023-06-10T20:55: 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: 25.478335+0000 mon.a (mon.0) 250 2023-06-10T20:55:26.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: 55:25.479036+0000 mon.a (mon.0 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: ) 251 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: 20:55:25.485345+0000 mon.a (mon 2023-06-10T20:55:26.153 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[107122]: .0) 252 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: 06-10T20:55:24.490798+0000 mgr.y (mgr.14148) 60 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: cluster 2023-06-10T20:55:24.986630+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.138151+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.470545+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.477557+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.154 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.478335+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:26.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.479036+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:26.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:25 smithi064 conmon[97224]: audit 2023-06-10T20:55:25.485345+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: cluster 2023-06-10T20:55:24.490798+0000 mgr.y (mgr. 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: 14148) 60 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: cluster 2023-06-10T20:55:24.986630+0000 mon.a (mon.0) 246 : cluster [DBG] 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: osdmap e9: 1 total, 1 up, 1 in 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: 20:55:25.138151+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: -06-10T20:55:25.470545+0000 mon.a (mon.0 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: ) 248 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: 20:55:25.477557+0000 mon.a (mon.0) 249 : audit 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023-06-10T20:55:25.478335 2023-06-10T20:55:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: +0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: T20:55:25.479036+0000 mon.a (mon.0) 251 : audit 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: audit 2023-06-10T20:55:25 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: .485345+0000 mon.a (mon.0) 252 : audit [INF] 2023-06-10T20:55:26.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:25 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:27.336 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:25.130836+0000 mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:27.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:25.471162+0000 mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 16144M 2023-06-10T20:55:27.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[107122]: audit 2023-06-10T20:55:26.788217+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:27.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[107122]: audit 2023-06-10T20:55:26.790527+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:27.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[107122]: audit 2023-06-10T20:55:26.791117+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:25.130836+0000 mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:25.471162+0000 mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 16144M 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: audit 2023-06-10T20:55:26.788217+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: audit 2023-06-10T20:55:26.790527+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: audit 2023-06-10T20:55:26.791117+0000 mon.a (mon.0) 255 : 2023-06-10T20:55:27.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:26 smithi064 conmon[97224]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:25.130836+0000 2023-06-10T20:55:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: mgr.y (mgr.14148) 61 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:25. 2023-06-10T20:55:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: 471162+0000 mgr.y (mgr.14148) 62 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 16144M 2023-06-10T20:55:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: audit 2023-06-10T20:55:26.788217+0000 2023-06-10T20:55:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: mon.a (mon.0) 253 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:27.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:27.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: 55:26.790527+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:27.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: audit 2023-06-10T20:55:27.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: 2023-06-10T20:55:26.791117+0000 mon.a (mon.0 2023-06-10T20:55:27.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:26 smithi152 conmon[97365]: ) 255 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:28.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[97224]: cluster 2023-06-10T20:55:26.491069+0000 mgr.y ( 2023-06-10T20:55:28.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[97224]: mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:28.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[97224]: audit 2023-06-10T20:55:26.785600+0000 mgr.y (mgr.14148) 64 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:28.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[107122]: cluster 2023-06-10T20:55:26.491069+0000 mgr.y ( 2023-06-10T20:55:28.354 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[107122]: mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:28.354 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[107122]: audit 2023-06-10T20:55:26.785600+0000 mgr.y (mgr. 2023-06-10T20:55:28.354 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:27 smithi064 conmon[107122]: 14148) 64 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:27 smithi152 conmon[97365]: cluster 2023-06-10T20:55:26.491069+0000 mgr.y 2023-06-10T20:55:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:27 smithi152 conmon[97365]: (mgr.14148) 63 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:27 smithi152 conmon[97365]: audit 2023-06-10T20:55:26.785600+0000 2023-06-10T20:55:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:27 smithi152 conmon[97365]: mgr.y (mgr.14148) 64 : audit [DBG] from='client.14241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:30.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: cluster 2023-06-10T20:55:28.491339+0000 mgr.y (mgr.14148) 65 2023-06-10T20:55:30.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:30.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: audit 2023-06-10T20:55:29.072793+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.64:0/1012878076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: audit 2023-06-10T20:55:29.073067+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: audit 2023-06-10T20:55:29.076826+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]': finished 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: cluster 2023-06-10T20:55:29.076885+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: audit 2023-06-10T20:55:29.076983+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[97224]: audit 2023-06-10T20:55:29.682304+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.64:0/333046306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: cluster 2023-06-10T20:55:28.491339+0000 2023-06-10T20:55:30.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: mgr.y (mgr.14148) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: audit 2023-06-10T20:55:29.072793+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.64:0/1012878076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: audit 2023-06-10T20:55:29.073067+0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: audit 2023-06-10T20:55:29.076826+0000 mon.a (mon.0) 257 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]': finished 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: cluster 2023-06-10T20:55:29.076885+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: audit 2023-06-10T20:55:29.076983+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:29 smithi064 conmon[107122]: audit 2023-06-10T20:55:29.682304+0000 mon.a (mon.0) 260 : audit [DBG] from='client.? 172.21.15.64:0/333046306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: cluster 2023-06-10T20:55:28.491339+0000 mgr.y (mgr.14148) 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: audit 2023-06-10T20:55:29.072793+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.64:0/1012878076' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: audit 2023-06-10T20:55:29.073067 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: +0000 mon.a (mon.0) 256 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]: dispatch 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: -10T20:55:29.076826+0000 mon.a (mon.0) 257 2023-06-10T20:55:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0f26f57-1eae-44eb-b91a-24db061d2f83"}]': finished 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: cluster 2023-06-10T20:55:29 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: .076885+0000 mon.a (mon.0) 258 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: audit 2023-06-10T20:55:29.076983 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: +0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: audit 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: 2023-06-10T20:55:29.682304+0000 mon.a ( 2023-06-10T20:55:30.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:29 smithi152 conmon[97365]: mon.0) 260 : audit [DBG] from='client.? 172.21.15.64:0/333046306' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:31 smithi064 conmon[97224]: cluster 2023-06-10T20:55:30.491831+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:32.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:31 smithi064 conmon[107122]: cluster 2023-06-10T20:55:30.491831+0000 mgr.y (mgr.14148) 66 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:31 smithi152 conmon[97365]: cluster 2023-06-10T20:55:30.491831+0000 mgr.y (mgr.14148) 66 2023-06-10T20:55:32.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:31 smithi152 conmon[97365]: : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:34.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:33 smithi064 conmon[97224]: cluster 2023-06-10T20:55:32.492477 2023-06-10T20:55:34.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:33 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:34.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:33 smithi064 conmon[107122]: cluster 2023-06-10T20:55:32.492477+0000 2023-06-10T20:55:34.133 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:33 smithi064 conmon[107122]: mgr.y (mgr.14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:33 smithi152 conmon[97365]: cluster 2023-06-10T20:55:32.492477+0000 mgr.y (mgr. 2023-06-10T20:55:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:33 smithi152 conmon[97365]: 14148) 67 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:36.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[107122]: cluster 2023-06-10T20:55:34.492885+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:36.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[107122]: audit 2023-06-10T20:55:35.139433+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T20:55:36.373 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[107122]: audit 2023-06-10T20:55:35.140214+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:36.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[97224]: cluster 2023-06-10T20:55:34.492885+0000 mgr.y (mgr.14148) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:36.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[97224]: audit 2023-06-10T20:55:35.139433+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T20:55:36.373 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:35 smithi064 conmon[97224]: audit 2023-06-10T20:55:35.140214+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: cluster 2023-06-10T20:55:34.492885+0000 mgr.y (mgr.14148 2023-06-10T20:55:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: ) 68 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: audit 2023-06-10T20:55:35.139433+0000 mon.a (mon.0) 261 : audit 2023-06-10T20:55:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T20:55:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:35 smithi152 conmon[97365]: 55:35.140214+0000 mon.a (mon.0) 262 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:37.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:36 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:35.140865+0000 mgr.y (mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi064 2023-06-10T20:55:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:36 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:35.140865+0000 mgr.y (mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi064 2023-06-10T20:55:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:36 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:35.140865+0000 2023-06-10T20:55:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:36 smithi152 conmon[97365]: mgr.y (mgr.14148) 69 : cephadm [INF] Deploying daemon osd.1 on smithi064 2023-06-10T20:55:38.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:37 smithi064 conmon[107122]: cluster 2023-06-10T20:55:36.493208 2023-06-10T20:55:38.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:37 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:38.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:37 smithi064 conmon[97224]: cluster 2023-06-10T20:55:36.493208 2023-06-10T20:55:38.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:37 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:37 smithi152 conmon[97365]: cluster 2023-06-10T20:55:36.493208+0000 2023-06-10T20:55:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:37 smithi152 conmon[97365]: mgr.y (mgr.14148) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:39 smithi064 conmon[97224]: cluster 2023-06-10T20:55:38.493474+0000 mgr.y ( 2023-06-10T20:55:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:39 smithi064 conmon[97224]: mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:40.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:39 smithi064 conmon[107122]: cluster 2023-06-10T20:55: 2023-06-10T20:55:40.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:39 smithi064 conmon[107122]: 38.493474+0000 mgr.y (mgr.14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:39 smithi152 conmon[97365]: cluster 2023-06-10T20:55:38.493474+0000 mgr.y (mgr. 2023-06-10T20:55:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:39 smithi152 conmon[97365]: 14148) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:41.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:55:41.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[97224]: 10T20:55:39.965914+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:41.142 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[97224]: audit 2023-06-10T20:55:39.977583+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:41.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: audit 2023-06-10T20:55:39 2023-06-10T20:55:41.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: .965914+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:41.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: audit 2023-06-10T20:55:41.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: 2023-06-10T20:55:39. 2023-06-10T20:55:41.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: 977583+0000 mon.a (mon.0) 264 : 2023-06-10T20:55:41.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:40 smithi064 conmon[107122]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:40 smithi152 conmon[97365]: audit 2023-06-10T20:55:39.965914+0000 mon.a (mon.0) 263 : 2023-06-10T20:55:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:40 smithi152 conmon[97365]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:40 smithi152 conmon[97365]: audit 2023-06-10T20:55:39.977583+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: cluster 2023-06-10T20:55:40.493864+0000 mgr.y (mgr.14148) 72 : cluster 2023-06-10T20:55:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: audit 2023-06-10T20:55:41.766900+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: audit 2023-06-10T20:55:41.767646+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: audit 2023-06-10T20:55:41.768280+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[97224]: audit 2023-06-10T20:55:41.772548+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: cluster 2023-06-10T20:55 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: :40.493864+0000 mgr.y (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: audit 2023-06-10T20:55:41.766900+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: audit 2023-06-10T20:55:41.767646+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:42.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: audit 2023-06-10T20:55:41.768280+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:42.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:41 smithi064 conmon[107122]: audit 2023-06-10T20:55:41.772548+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: cluster 2023-06-10T20:55:40.493864+0000 mgr.y 2023-06-10T20:55:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: (mgr.14148) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: audit 2023-06-10T20:55:41.766900+0000 mon.a ( 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: mon.0) 265 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: audit 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: 2023-06-10T20:55:41.767646+0000 mon.a (mon.0) 266 : audit [DBG] 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: audit 2023-06-10T20:55:41.768280+0000 2023-06-10T20:55:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: mon.a (mon.0) 267 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:55:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: 10T20:55:41.772548+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:41 smithi152 conmon[97365]: 2023-06-10T20:55:43.102 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 1 on host 'smithi064' 2023-06-10T20:55:43.822 DEBUG:teuthology.orchestra.run.smithi064:osd.1> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.1.service 2023-06-10T20:55:43.825 INFO:tasks.cephadm:Deploying osd.2 on smithi064 with /dev/vg_nvme/lv_2... 2023-06-10T20:55:43.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-06-10T20:55:43.856 INFO:journalctl@ceph.osd.1.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:55:44.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[107122]: cluster 2023-06-10T20:55:42 2023-06-10T20:55:44.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[107122]: .494234+0000 mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:44.110 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[107122]: audit 2023-06-10T20:55:43.098987+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[107122]: audit 2023-06-10T20:55:43.140683+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[107122]: audit 2023-06-10T20:55:43.173064+0000 mon.a (mon.0) 271 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[97224]: cluster 2023-06-10T20:55:42. 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[97224]: 494234+0000 mgr.y (mgr.14148) 73 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[97224]: audit 2023-06-10T20:55:43.098987+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:44.111 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[97224]: audit 2023-06-10T20:55:43.140683+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:44.112 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:43 smithi064 conmon[97224]: audit 2023-06-10T20:55:43.173064+0000 mon.a (mon.0) 271 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T20:55:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: cluster 2023-06-10T20:55:42.494234+0000 mgr.y (mgr.14148) 73 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: audit 2023-06-10T20:55:43.098987+0000 mon.a (mon.0) 269 : audit [INF] 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: -10T20:55:43.140683+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:55:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: 06-10T20:55:43.173064+0000 mon.a (mon.0) 271 : 2023-06-10T20:55:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:43 smithi152 conmon[97365]: audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T20:55:45.138 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[118035]: debug 2023-06-10T20:55:45.113+0000 7f43f7449700 -1 osd.1 0 waiting for initial osdmap 2023-06-10T20:55:45.138 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[118035]: debug 2023-06-10T20:55:45.116+0000 7f43f33c5700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: audit 2023-06-10T20:55:44 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: .103004+0000 mon.a (mon.0) 272 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: cluster 2023-06-10T20:55:44.103083+0000 mon.a (mon.0) 273 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: audit 2023-06-10T20:55:44.103221+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: audit 2023-06-10T20:55:44.103739+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:45.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[107122]: cluster 2023-06-10T20:55:44.494494+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: audit 2023-06-10T20:55:44.103004+0000 mon.a ( 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: mon.0) 272 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: cluster 2023-06-10T20:55:44.103083+0000 mon.a (mon.0) 273 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: audit 2023-06-10T20:55:44.103221+0000 mon.a (mon.0) 274 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: audit 2023-06-10T20:55:44.103739+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:45.140 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:45 smithi064 conmon[97224]: cluster 2023-06-10T20:55:44.494494+0000 mgr.y (mgr.14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:45.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: audit 2023-06-10T20:55:44.103004+0000 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: mon.a (mon.0) 272 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: 55:44.103083+0000 mon.a (mon.0) 273 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: 55:44.103221+0000 mon.a (mon.0) 274 : audit 2023-06-10T20:55:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: audit 2023-06-10T20:55:44. 2023-06-10T20:55:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: 103739+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:55:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: cluster 2023-06-10T20:55:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: 2023-06-10T20:55:44.494494+0000 mgr.y (mgr. 2023-06-10T20:55:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:45 smithi152 conmon[97365]: 14148) 74 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-10T20:55:46.357 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[107122]: audit 2023-06-10T20:55:45.110620+0000 mon.a (mon.0) 276 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:46.357 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[107122]: cluster 2023-06-10T20:55:45.110699+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-10T20:55:46.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[107122]: audit 2023-06-10T20:55:45.111733+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[107122]: audit 2023-06-10T20:55:45.115212+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[97224]: audit 2023-06-10T20:55:45.110620+0000 mon.a (mon.0) 276 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:46.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[97224]: cluster 2023-06-10T20:55:45.110699+0000 mon.a (mon.0) 277 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-10T20:55:46.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[97224]: audit 2023-06-10T20:55:45.111733+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:46 smithi064 conmon[97224]: audit 2023-06-10T20:55:45.115212+0000 mon.a (mon.0) 279 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: audit 2023-06-10T20:55:45.110620+0000 mon.a (mon 2023-06-10T20:55:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: .0) 276 : audit [INF] from='osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:55:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: cluster 2023-06-10T20:55:45.110699+0000 mon.a (mon.0 2023-06-10T20:55:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: ) 277 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-10T20:55:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:55:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: -10T20:55:45.111733+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: audit 2023 2023-06-10T20:55:46.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: -06-10T20:55:45.115212+0000 mon.a (mon.0) 2023-06-10T20:55:46.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:46 smithi152 conmon[97365]: 279 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:46.518 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-06-10T20:55:46.519 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-06-10T20:55:46.519 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-06-10T20:55:46.519 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-06-10T20:55:46.519 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124124 s, 845 MB/s 2023-06-10T20:55:46.519 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-06-10T20:55:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: cluster 2023-06-10T20:55:44.193061+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: cluster 2023-06-10T20:55:44.193128+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: cluster 2023-06-10T20:55:46.111571+0000 mon.a (mon.0) 280 : cluster [INF] osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478] boot 2023-06-10T20:55:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: cluster 2023-06-10T20:55:46.111611+0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-10T20:55:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: audit 2023-06-10T20:55:46.112635+0000 mon.a (mon.0) 282 : 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: cluster 2023-06-10T20:55:46.494820+0000 mgr.y ( 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[97224]: mgr.14148) 75 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: cluster 2023-06-10T20:55 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: :44.193061+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:47.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: cluster 2023-06-10T20:55:44.193128+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: cluster 2023-06-10T20:55:46.111571+0000 mon.a 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: (mon.0) 280 : cluster [INF] osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478] boot 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: -10T20:55:46.111611+0000 mon.a (mon.0) 281 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:55:47.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: 06-10T20:55:46.112635+0000 mon.a (mon.0) 282 2023-06-10T20:55:47.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:47.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: cluster 2023-06-10T20:55:47.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: 2023-06-10T20:55:46.494820+0000 mgr.y (mgr.14148) 2023-06-10T20:55:47.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:47 smithi064 conmon[107122]: 75 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: cluster 2023-06-10T20:55:44.193061 2023-06-10T20:55:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: +0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: 55:44.193128+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: 55:46.111571+0000 mon.a (mon.0) 280 : cluster [INF] 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: osd.1 [v2:172.21.15.64:6810/3223495478,v1:172.21.15.64:6811/3223495478] boot 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: cluster 2023-06-10T20:55:46.111611 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: +0000 mon.a (mon.0) 281 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-10T20:55:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: audit 2023-06-10T20:55:46 2023-06-10T20:55:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: .112635+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:55:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: cluster 2023-06-10T20:55: 2023-06-10T20:55:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: 46.494820+0000 mgr.y (mgr.14148) 75 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:47 smithi152 conmon[97365]: 2023-06-10T20:55:47.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi064:vg_nvme/lv_2 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: cluster 2023-06-10T20:55:47.123498+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:47.605686+0000 mgr.y (mgr.14148) 76 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.610127+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.611093+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.611854+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.358 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:47.612367+0000 mgr.y (mgr.14148) 77 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 8072M 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: 55:47.616061+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.617000+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.617683+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[97224]: audit 2023-06-10T20:55:47.622169+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.359 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: cluster 2023-06-10T20:55:47.123498+0000 mon.a (mon. 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: 0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: cephadm 2023-06-10T20:55:47.605686+0000 mgr.y (mgr.14148) 76 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06-10T20:55:47.610127+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06-10T20:55:47.611093+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06-10T20:55:47.611854+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: cephadm 2023-06-10T20:55: 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: 47.612367+0000 mgr.y (mgr.14148) 77 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 8072M 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: 10T20:55:47.616061+0000 mon.a (mon.0) 287 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: -06-10T20:55:47.617000+0000 2023-06-10T20:55:48.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: mon.a (mon.0) 288 : audit [DBG] 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: 10T20:55:47.617683+0000 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: mon.a (mon.0) 289 : audit [INF] 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: T20:55:47.622169+0000 mon.a (mon.0) 290 : 2023-06-10T20:55:48.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:48 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: cluster 2023-06-10T20:55:47.123498+0000 mon.a ( 2023-06-10T20:55:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: mon.0) 283 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-10T20:55:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:47.605686+0000 mgr.y (mgr.14148) 76 : 2023-06-10T20:55:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:55:48.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:55:48.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 06-10T20:55:47.610127+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023 2023-06-10T20:55:48.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: -06-10T20:55:47.611093+0000 mon.a (mon.0) 2023-06-10T20:55:48.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 285 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 55:47.611854+0000 mon.a (mon.0) 286 : audit 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:47. 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 612367+0000 mgr.y (mgr.14148) 77 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 8072M 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:55:48.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 55:47.616061+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 10T20:55:47.617000+0000 mon.a (mon.0) 288 : audit 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023-06-10T20:55:47. 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 617683+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: 2023-06-10T20:55:48.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: audit 2023-06-10T20:55:47.622169+0000 2023-06-10T20:55:48.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:48 smithi152 conmon[97365]: mon.a (mon.0) 290 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:55:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:49 smithi064 conmon[97224]: cluster 2023-06-10T20:55:48.495081+0000 mgr.y (mgr. 2023-06-10T20:55:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:49 smithi064 conmon[97224]: 14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:49 smithi064 conmon[107122]: cluster 2023-06-10T20:55:48. 2023-06-10T20:55:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:49 smithi064 conmon[107122]: 495081+0000 mgr.y (mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:49 smithi152 conmon[97365]: cluster 2023-06-10T20:55:48.495081+0000 mgr.y ( 2023-06-10T20:55:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:49 smithi152 conmon[97365]: mgr.14148) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:50.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[97224]: audit 2023-06-10T20:55:49.405821+0000 mgr.y (mgr.14148) 79 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:50.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[97224]: audit 2023-06-10T20:55:49.408407+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:50.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[97224]: audit 2023-06-10T20:55:49.410634+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:50.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[97224]: audit 2023-06-10T20:55:49.411292+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:50.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[107122]: audit 2023-06-10T20:55:49.405821+0000 mgr.y (mgr.14148) 79 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:50.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[107122]: audit 2023-06-10T20:55:49.408407+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:50.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[107122]: audit 2023-06-10T20:55:49.410634+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:50.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:50 smithi064 conmon[107122]: audit 2023-06-10T20:55:49.411292+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: audit 2023-06-10T20:55:49.405821+0000 mgr.y (mgr 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: .14148) 79 : audit [DBG] from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: audit 2023-06-10T20:55: 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: 49.408407+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: audit 2023-06-10T20:55:49.410634 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: +0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:55:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: audit 2023 2023-06-10T20:55:50.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: -06-10T20:55:49.411292+0000 mon.a (mon.0) 2023-06-10T20:55:50.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:50 smithi152 conmon[97365]: 293 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: cluster 2023-06-10T20:55:50.495335+0000 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: audit 2023-06-10T20:55:51.741315+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.64:0/3870408875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: audit 2023-06-10T20:55:51.741675+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: audit 2023-06-10T20:55:51.745581+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]': finished 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: cluster 2023-06-10T20:55:51.745652+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-06-10T20:55:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[97224]: audit 2023-06-10T20:55:51.745774+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: cluster 2023-06-10T20:55:50. 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: 495335+0000 mgr.y (mgr.14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: audit 2023-06-10T20:55:51.741315+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.64:0/3870408875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: audit 2023-06-10T20:55:51.741675+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: audit 2023-06-10T20:55:51.745581+0000 mon.a (mon.0) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]': finished 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: cluster 2023-06-10T20:55:51.745652+0000 mon.a (mon.0) 296 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-06-10T20:55:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:51 smithi064 conmon[107122]: audit 2023-06-10T20:55:51.745774+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:55:52.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: cluster 2023-06-10T20:55:50.495335+0000 mgr.y (mgr. 2023-06-10T20:55:52.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: 14148) 80 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: audit 2023-06-10T20:55:51.741315+0000 mon.c (mon.2) 4 : audit [INF] from='client.? 172.21.15.64:0/3870408875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: audit 2023-06-10T20:55:51. 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: 741675+0000 mon.a (mon.0) 294 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]: dispatch 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: audit 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: 2023-06-10T20:55:51.745581+0000 mon.a (mon.0 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: ) 295 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4a979da-bdc4-46f7-aaa3-e912753f1272"}]': finished 2023-06-10T20:55:52.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: cluster 2023-06-10T20 2023-06-10T20:55:52.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: :55:51.745652+0000 mon.a (mon.0) 296 : cluster [DBG] 2023-06-10T20:55:52.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: osdmap e15: 3 total, 2 up, 3 in 2023-06-10T20:55:52.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: audit 2023-06-10T20:55:51.745774 2023-06-10T20:55:52.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:51 smithi152 conmon[97365]: +0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:55:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:52 smithi064 conmon[107122]: audit 2023-06-10T20:55:52 2023-06-10T20:55:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:52 smithi064 conmon[107122]: .331605+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.64:0/562904691' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:53.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:52 smithi064 conmon[97224]: audit 2023-06-10T20:55:52.331605+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.64:0/562904691' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:52 smithi152 conmon[97365]: audit 2023-06-10T20:55:52.331605+0000 2023-06-10T20:55:53.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:52 smithi152 conmon[97365]: mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.64:0/562904691' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:55:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:53 smithi064 conmon[97224]: cluster 2023-06-10T20:55:52.495683+0000 mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:54.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:53 smithi064 conmon[107122]: cluster 2023-06-10T20:55:52.495683+0000 2023-06-10T20:55:54.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:53 smithi064 conmon[107122]: mgr.y (mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:53 smithi152 conmon[97365]: cluster 2023-06-10T20:55:52.495683+0000 mgr.y 2023-06-10T20:55:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:53 smithi152 conmon[97365]: (mgr.14148) 81 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:56.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:55 smithi064 conmon[97224]: cluster 2023-06-10T20:55:54.496167+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:56.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:55 smithi064 conmon[107122]: cluster 2023-06-10T20:55:54.496167+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:55 smithi152 conmon[97365]: cluster 2023-06-10T20:55:54.496167+0000 mgr.y (mgr.14148) 82 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:57 smithi064 conmon[107122]: cluster 2023-06-10T20:55:56.496516+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:57 smithi064 conmon[97224]: cluster 2023-06-10T20:55:56.496516+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:57 smithi152 conmon[97365]: cluster 2023-06-10T20:55:56.496516+0000 mgr.y (mgr.14148) 83 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:55:59.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:58 smithi064 conmon[107122]: audit 2023-06-10T20:55:58.037516+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T20:55:59.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:58 smithi064 conmon[107122]: audit 2023-06-10T20:55:58.038328+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:59.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:58 smithi064 conmon[97224]: audit 2023-06-10T20:55:58.037516+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T20:55:59.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:58 smithi064 conmon[97224]: audit 2023-06-10T20:55:58.038328+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:55:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:58 smithi152 conmon[97365]: audit 2023-06-10T20:55:58.037516+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T20:55:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:58 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:55:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:58 smithi152 conmon[97365]: T20:55:58.038328+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[97224]: cephadm 2023-06-10T20:55:58.038963+0000 mgr.y (mgr 2023-06-10T20:56:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[97224]: .14148) 84 : cephadm [INF] Deploying daemon osd.2 on smithi064 2023-06-10T20:56:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[97224]: cluster 2023-06-10T20:55:58.497043+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: cephadm 2023-06-10T20:55: 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: 58.038963+0000 mgr.y (mgr.14148) 84 : cephadm [INF] Deploying daemon osd.2 on smithi064 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: cluster 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: 2023-06-10T20: 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: 55:58.497043+0000 mgr.y 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: (mgr.14148) 85 : cluster 2023-06-10T20:56:00.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:55:59 smithi064 conmon[107122]: [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:59 smithi152 conmon[97365]: cephadm 2023-06-10T20:55:58.038963+0000 mgr.y (mgr.14148) 84 : cephadm [INF] Deploying daemon osd.2 on smithi064 2023-06-10T20:56:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:59 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T20:56:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:55:59 smithi152 conmon[97365]: 20:55:58.497043+0000 mgr.y (mgr.14148) 85 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:02.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:01 smithi064 conmon[97224]: cluster 2023-06-10T20: 2023-06-10T20:56:02.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:01 smithi064 conmon[97224]: 56:00.497411+0000 mgr.y (mgr.14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:02.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:01 smithi064 conmon[107122]: cluster 2023-06-10T20: 2023-06-10T20:56:02.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:01 smithi064 conmon[107122]: 56:00.497411+0000 mgr.y (mgr.14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:01 smithi152 conmon[97365]: cluster 2023-06-10T20:56:00.497411+0000 mgr.y (mgr 2023-06-10T20:56:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:01 smithi152 conmon[97365]: .14148) 86 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[107122]: cluster 2023-06-10T20:56:02.497741 2023-06-10T20:56:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[107122]: audit 2023-06-10T20:56:03.254951+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[107122]: audit 2023-06-10T20:56:03.267480+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[97224]: cluster 2023-06-10T20:56:02.497741+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[97224]: audit 2023-06-10T20:56 2023-06-10T20:56:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[97224]: :03.254951+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:03 smithi064 conmon[97224]: audit 2023-06-10T20:56:03.267480+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:03 smithi152 conmon[97365]: cluster 2023-06-10T20:56:02.497741+0000 mgr.y (mgr.14148) 87 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:03 smithi152 conmon[97365]: audit 2023-06-10T20:56 2023-06-10T20:56:04.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:03 smithi152 conmon[97365]: :03.254951+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:04.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:03 smithi152 conmon[97365]: audit 2023-06-10T20:56:03.267480+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:06.232 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 2 on host 'smithi064' 2023-06-10T20:56:06.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: cluster 2023-06-10T20:56:04.497963 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: audit 2023-06-10T20:56:05.278004+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: audit 2023-06-10T20:56:05.278708+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: audit 2023-06-10T20:56:05.279347+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[97224]: audit 2023-06-10T20:56:05.283218+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: cluster 2023-06-10T20:56:04.497963 2023-06-10T20:56:06.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:06.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: audit 2023-06-10T20:56:05.278004+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: audit 2023-06-10T20:56:05.278708+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:06.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: audit 2023-06-10T20:56:05.279347+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:06.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:05 smithi064 conmon[107122]: audit 2023-06-10T20:56:05.283218+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: cluster 2023-06-10T20:56:04.497963+0000 mgr.y (mgr.14148) 88 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: audit 2023-06-10T20:56:05 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: .278004+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: audit 2023-06-10T20:56:05.278708+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: audit 2023-06-10T20:56:05.279347+0000 mon.a (mon.0) 304 : audit 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:06.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:05 smithi152 conmon[97365]: audit 2023-06-10T20:56:05.283218+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:06.881 DEBUG:teuthology.orchestra.run.smithi064:osd.2> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.2.service 2023-06-10T20:56:06.883 INFO:tasks.cephadm:Deploying osd.3 on smithi064 with /dev/vg_nvme/lv_1... 2023-06-10T20:56:06.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-06-10T20:56:06.917 INFO:journalctl@ceph.osd.2.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:56:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[97224]: audit 2023-06-10T20:56:06.228054+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[97224]: audit 2023-06-10T20:56:06.250804+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[97224]: audit 2023-06-10T20:56:06.440397+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T20:56:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[97224]: cluster 2023-06-10T20:56:06.498434+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:07.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:56:07.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: :56:06.228054+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: audit 2023-06-10T20:56:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: 2023-06- 2023-06-10T20:56:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: 10T20:56:06.250804 2023-06-10T20:56:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: +0000 mon.a (mon.0) 307 2023-06-10T20:56:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: 06-10T20:56:06.440397+0000 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: mon.a (mon.0) 308 : audit [INF] 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: cluster 2023-06-10T20:56:06 2023-06-10T20:56:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: .498434+0000 mgr.y (mgr.14148) 89 : 2023-06-10T20:56:07.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:07 smithi064 conmon[107122]: cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:07 smithi152 conmon[97365]: audit 2023-06-10T20:56:06.228054+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:07.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:07 smithi152 conmon[97365]: audit 2023-06-10T20:56:06.250804+0000 2023-06-10T20:56:07.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:07 smithi152 conmon[97365]: mon.a (mon.0) 307 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:07.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:07 smithi152 conmon[97365]: audit 2023-06-10T20:56:06.440397+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T20:56:07.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:07 smithi152 conmon[97365]: cluster 2023-06-10T20:56:06.498434+0000 mgr.y (mgr.14148) 89 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:08.658 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[123976]: debug 2023-06-10T20:56:08.243+0000 7fe5620e9700 -1 osd.2 0 waiting for initial osdmap 2023-06-10T20:56:08.658 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[123976]: debug 2023-06-10T20:56:08.250+0000 7fe55c862700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[97224]: audit 2023-06-10T20:56:07 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[97224]: .235886+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[97224]: cluster 2023-06-10T20:56:07.235959+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[97224]: audit 2023-06-10T20:56:07.236067+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[97224]: audit 2023-06-10T20:56:07.236533+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: audit 2023-06-10T20:56:07. 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: 235886+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: cluster 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: 2023-06-10T20:56:07.235959+0000 mon.a (mon.0) 310 : cluster 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: -10T20:56:07.236067+0000 mon.a (mon.0) 311 2023-06-10T20:56:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:08.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: audit 2023-06-10T20:56:08.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: 2023-06-10T20:56:07.236533+0000 mon.a (mon.0) 312 : 2023-06-10T20:56:08.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:08 smithi064 conmon[107122]: audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: audit 2023-06-10T20:56:07.235886+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T20:56:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:56:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: 56:07.235959+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-10T20:56:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: audit 2023-06-10T20:56:07.236067+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: audit 2023-06-10T20:56:07 2023-06-10T20:56:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: .236533+0000 mon.a (mon.0) 312 : audit 2023-06-10T20:56:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:08 smithi152 conmon[97365]: [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:09.660 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-06-10T20:56:09.660 INFO:teuthology.orchestra.run.smithi064.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-06-10T20:56:09.661 INFO:teuthology.orchestra.run.smithi064.stderr: stderr: 10+0 records in 2023-06-10T20:56:09.661 INFO:teuthology.orchestra.run.smithi064.stderr:10+0 records out 2023-06-10T20:56:09.661 INFO:teuthology.orchestra.run.smithi064.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.011074 s, 947 MB/s 2023-06-10T20:56:09.661 INFO:teuthology.orchestra.run.smithi064.stderr:--> Zapping successful for: 2023-06-10T20:56:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[97224]: audit 2023-06-10T20:56:08.240290+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[97224]: cluster 2023-06-10T20:56:08.240363+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-10T20:56:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[97224]: audit 2023-06-10T20:56:08.240714+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[97224]: audit 2023-06-10T20:56:08.246802+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[97224]: cluster 2023-06-10T20:56:08.498703+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: audit 2023-06-10T20:56:08.240290+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: cluster 2023-06-10T20:56:08.240363+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: audit 2023-06-10T20:56:08.240714+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: audit 2023-06-10T20:56:08.246802+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: cluster 2023-06-10T20:56 2023-06-10T20:56:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:09 smithi064 conmon[107122]: :08.498703+0000 mgr.y (mgr.14148) 90 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: audit 2023-06-10T20:56:08.240290+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: cluster 2023-06-10T20 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: :56:08.240363+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: audit 2023-06-10T20:56:08.240714+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: 10T20:56:08.246802+0000 mon.a 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: (mon.0) 316 : audit [DBG] 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: cluster 2023-06-10 2023-06-10T20:56:10.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: T20:56:08.498703+0000 mgr.y (mgr 2023-06-10T20:56:10.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: .14148) 90 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-10T20:56:10.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:09 smithi152 conmon[97365]: 2023-06-10T20:56:10.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi064:vg_nvme/lv_1 2023-06-10T20:56:10.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: :56:07.400295+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: cluster 2023-06-10T20:56:07.400343+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10T20:56:09.244069+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: cluster 2023-06-10T20:56:09.252457+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224] boot 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: cluster 2023-06-10T20:56:09.252495+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10T20:56:09.252854+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.968 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 10T20:56:09.253286+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 06-10T20:56:10.249101+0000 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: mon.a (mon.0) 322 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: -10T20:56:10.249186 2023-06-10T20:56:10.969 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: +0000 mon.a (mon.0) 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 323 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 56:10.249946+0000 mon.a (mon. 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 0) 324 : audit [INF] 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:10.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 06-10T20:56:10. 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 317702+0000 mon.a (mon.0) 325 : audit 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: -06-10T20:56 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: :10.318737+0000 mon.a (mon 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: .0) 326 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:10.971 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 06-10T20:56: 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 10.319409+0000 mon.a (mon.0 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: ) 327 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: T20:56:10.319964+0000 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: mon.a (mon.0) 328 2023-06-10T20:56:10.972 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: cluster 2023-06-10T20:56: 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 07.400295+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: cluster 2023-06-10T20:56:07.400343+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:09.244069+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: cluster 2023-06-10T20:56:09.252457+0000 mon.a (mon.0) 318 : cluster [INF] osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224] boot 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: cluster 2023-06-10T20:56:09.252495+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:09.252854+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:10.973 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:09.253286+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.249101+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: cluster 2023-06-10T20:56:10.249186+0000 mon.a (mon.0) 323 : cluster [DBG] 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: osdmap e19: 3 total, 3 up, 3 in 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.249946+0000 mon.a (mon.0) 324 : audit [INF] 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 2023-06-10T20 2023-06-10T20:56:10.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: :56:10.317702+0000 mon.a ( 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: mon.0) 325 : audit [INF] 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 2023-06-10T20:56:10 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: .318737+0000 mon.a (mon.0) 326 : 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.975 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 2023-06-10T20 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: :56:10.319409+0000 mon.a (mon 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: .0) 327 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 2023-06-10T20:56:10 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: .319964+0000 mon.a (mon.0) 328 : 2023-06-10T20:56:10.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: -10T20:56:10.326637+0000 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: mon.a (mon.0) 329 : audit [INF] 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:10.977 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: T20:56:10.326637 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: +0000 mon.a (mon.0) 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 329 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: -06-10T20:56:10.327576 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: +0000 mon.a (mon.0) 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 330 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:10.978 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: -06-10T20:56:10.328167 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: +0000 mon.a (mon.0) 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 331 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: audit 2023-06-10T20:56 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: :10.334864+0000 mon.a (mon.0) 332 : audit 2023-06-10T20:56:10.979 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 56:10.327576+0000 mon.a (mon.0) 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 330 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 10T20:56:10.328167+0000 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: mon.a (mon.0) 331 : audit [INF] 2023-06-10T20:56:10.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:10.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:56:10.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: 06-10T20:56:10.334864 2023-06-10T20:56:10.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: +0000 mon.a (mon.0) 332 : audit 2023-06-10T20:56:10.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:10 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: cluster 2023-06-10T20:56:07.400295+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: cluster 2023-06-10T20:56: 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 07.400343+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023-06-10T20:56:09.244069+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: cluster 2023-06-10T20:56:09.252457+0000 2023-06-10T20:56:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 318 : cluster [INF] 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: osd.2 [v2:172.21.15.64:6818/208030224,v1:172.21.15.64:6819/208030224] boot 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: cluster 2023-06-10T20 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: :56:09.252495+0000 mon.a (mon.0 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: ) 319 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:09.252854+0000 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 320 : audit 2023-06-10T20:56:11.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 2023-06-10T20:56:09 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: .253286+0000 mon.a (mon.0) 321 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:10.249101+0000 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 322 : audit 2023-06-10T20:56:11.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: cluster 2023-06 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: -10T20:56:10.249186+0000 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 323 : cluster [DBG] 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: osdmap e19: 3 total, 3 up, 3 in 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:10.249946+0000 2023-06-10T20:56:11.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 324 : audit [INF] 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: -06-10T20:56:10.317702 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: +0000 mon.a (mon.0) 325 : audit 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 2023-06-10T20:56:10. 2023-06-10T20:56:11.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 318737+0000 mon.a (mon.0) 326 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:10.319409+0000 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 327 : audit 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:10.319964+0000 2023-06-10T20:56:11.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 328 : audit [INF] 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 2023-06-10T20:56:10. 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 326637+0000 mon.a (mon.0) 329 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: -06-10T20:56:10.327576 2023-06-10T20:56:11.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: +0000 mon.a (mon.0) 330 : audit 2023-06-10T20:56:11.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:11.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:11.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: 06-10T20:56:10.328167+0000 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: mon.a (mon.0) 331 : audit [INF] 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: audit 2023 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: -06-10T20:56:10.334864 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: +0000 mon.a (mon.0) 332 : audit 2023-06-10T20:56:11.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:10 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:12.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[97224]: cephadm 2023-06-10T20:56:10.312737+0000 mgr.y (mgr.14148) 91 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[97224]: cephadm 2023-06-10T20:56:10.320412+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 5381M 2023-06-10T20:56:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[97224]: cluster 2023-06-10T20:56:10.499008+0000 mgr.y (mgr.14148) 93 : cluster [DBG] pgmap v70: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[97224]: audit 2023-06-10T20:56:11.251395+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:56:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[97224]: cluster 2023-06-10T20:56:11.251479+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-10T20:56:12.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: cephadm 2023-06-10T20:56:10.312737+0000 mgr.y (mgr.14148) 91 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: cephadm 2023-06-10T20:56:10.320412+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 5381M 2023-06-10T20:56:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: cluster 2023-06-10T20:56:10.499008+0000 mgr.y (mgr.14148) 93 : cluster [DBG] pgmap v70: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: audit 2023-06-10T20:56:11.251395+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:56:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:56:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:11 smithi064 conmon[107122]: 10T20:56:11.251479+0000 mon.a (mon.0) 334 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-10T20:56:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:10.312737+0000 mgr.y (mgr.14148) 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: 91 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:10.320412+0000 mgr.y (mgr.14148) 92 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 5381M 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: cluster 2023-06-10T20:56: 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: 10.499008+0000 mgr.y (mgr.14148) 93 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: : cluster [DBG] pgmap v70: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: -10T20:56:11.251395+0000 mon.a (mon 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: .0) 333 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: cluster 2023- 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: 06-10T20:56:11.251479+0000 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: mon.a (mon.0) 334 : cluster 2023-06-10T20:56:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:11 smithi152 conmon[97365]: [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-10T20:56:13.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:13.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: 12.213953+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:13.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:13.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: 12.216233+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:13.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: 2023-06-10T20:56:13.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: audit 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: 2023-06-10T20:56:12.216851+0000 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: mon.a (mon.0) 337 : audit [DBG] 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: T20:56:12.251252+0000 mon.a (mon 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: .0) 338 : cluster [DBG] 2023-06-10T20:56:13.098 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[107122]: osdmap e21: 3 total, 3 up, 3 in 2023-06-10T20:56:13.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[97224]: audit 2023-06-10T20:56:12.213953+0000 2023-06-10T20:56:13.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[97224]: mon.a (mon.0) 335 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:13.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[97224]: audit 2023-06-10T20:56:12.216233+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:13.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[97224]: audit 2023-06-10T20:56:12.216851+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:13.099 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:12 smithi064 conmon[97224]: cluster 2023-06-10T20:56:12.251252+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-06-10T20:56:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: audit 2023-06-10T20:56:12.213953+0000 mon.a (mon.0) 335 : 2023-06-10T20:56:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: audit 2023-06-10T20:56:12.216233+0000 mon.a (mon.0) 336 : audit [INF] 2023-06-10T20:56:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: audit 2023-06-10T20:56 2023-06-10T20:56:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: :12.216851+0000 mon.a (mon.0 2023-06-10T20:56:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: ) 337 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: cluster 2023- 2023-06-10T20:56:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: 06-10T20:56:12.251252+0000 mon.a 2023-06-10T20:56:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: (mon.0) 338 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-06-10T20:56:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:12 smithi152 conmon[97365]: 2023-06-10T20:56:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:13 smithi064 conmon[97224]: audit 2023-06-10T20:56:12.211370+0000 mgr.y (mgr.14148) 94 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:13 smithi064 conmon[97224]: cluster 2023-06-10T20:56:12.499295+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:14.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:13 smithi064 conmon[107122]: audit 2023-06-10T20:56:12.211370+0000 mgr.y (mgr.14148) 94 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:13 smithi064 conmon[107122]: cluster 2023-06-10T20:56:12.499295+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:13 smithi152 conmon[97365]: audit 2023-06-10T20:56:12.211370+0000 mgr.y (mgr.14148) 94 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi064:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:13 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:56:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:13 smithi152 conmon[97365]: 56:12.499295+0000 mgr.y (mgr.14148) 95 : cluster [DBG] pgmap v73: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:15.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: audit 2023-06-10T20:56:14.482742+0000 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.64:0/2096122718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: audit 2023-06-10T20:56:14.483057+0000 mon.a (mon.0) 339 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: audit 2023-06-10T20:56:14.488826+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]': finished 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: cluster 2023-06-10T20:56:14.488893+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[97224]: audit 2023-06-10T20:56:14.489008+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: 56:14.482742+0000 mon.c (mon.2) 6 : audit [INF] from='client.? 172.21.15.64:0/2096122718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: 10T20:56:14.483057+0000 mon.a (mon.0) 339 : audit [INF] 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: 14.488826+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]': finished 2023-06-10T20:56:15.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T20:56:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: 20:56:14.488893+0000 mon.a (mon.0) 341 : cluster [DBG] 2023-06-10T20:56:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: osdmap e22: 4 total, 3 up, 4 in 2023-06-10T20:56:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: -06-10T20:56:14.489008+0000 mon.a (mon.0) 342 : audit [DBG] 2023-06-10T20:56:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:14 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: audit 2023-06-10T20:56:14.482742+0000 mon.c (mon 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: .2) 6 : audit [INF] from='client.? 172.21.15.64:0/2096122718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: audit 2023-06-10T20:56:14.483057+0000 mon.a (mon.0) 339 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]: dispatch 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: audit 2023 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: -06-10T20:56:14.488826 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: +0000 mon.a (mon.0) 340 : audit 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efd43bfa-889e-4c76-997c-0fd3570fbc6e"}]': finished 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:56:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: 56:14.488893+0000 mon.a (mon.0) 341 : cluster [DBG] 2023-06-10T20:56:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: osdmap e22: 4 total, 3 up, 4 in 2023-06-10T20:56:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: 06-10T20:56:14.489008+0000 mon.a ( 2023-06-10T20:56:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: mon.0) 342 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:14 smithi152 conmon[97365]: 2023-06-10T20:56:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[97224]: cluster 2023-06-10T20:56:14.499572+0000 mgr.y (mgr.14148) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[97224]: 2023-06-10T20:56:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[97224]: audit 2023-06-10T20:56:15.111212+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.64:0/718517746' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[107122]: cluster 2023-06-10T20:56:14.499572+0000 mgr.y ( 2023-06-10T20:56:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[107122]: mgr.14148) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:56:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:15 smithi064 conmon[107122]: T20:56:15.111212+0000 mon.a (mon.0) 343 : audit [DBG] from='client.? 172.21.15.64:0/718517746' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:15 smithi152 conmon[97365]: cluster 2023-06-10T20:56:14.499572+0000 mgr.y (mgr.14148) 96 : cluster [DBG] pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:15 smithi152 conmon[97365]: audit 2023-06-10T20:56:15.111212+0000 mon.a ( 2023-06-10T20:56:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:15 smithi152 conmon[97365]: mon.0) 343 : audit [DBG] from='client.? 172.21.15.64:0/718517746' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:17 smithi064 conmon[107122]: cluster 2023-06-10T20:56:16 2023-06-10T20:56:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:17 smithi064 conmon[107122]: .500047+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:17 smithi064 conmon[97224]: cluster 2023-06-10T20:56:16.500047+0000 mgr.y (mgr.14148) 97 2023-06-10T20:56:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:17 smithi064 conmon[97224]: : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:17 smithi152 conmon[97365]: cluster 2023-06-10T20:56:16.500047+0000 mgr.y (mgr.14148) 97 : cluster [DBG] pgmap v76: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:19 smithi064 conmon[97224]: cluster 2023-06-10T20:56:18.500475+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:19 smithi064 conmon[107122]: cluster 2023-06-10T20: 2023-06-10T20:56:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:19 smithi064 conmon[107122]: 56:18.500475+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:20.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:19 smithi152 conmon[97365]: cluster 2023-06-10T20:56:18.500475+0000 mgr.y (mgr.14148) 98 : cluster [DBG] pgmap v77: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:21.256 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:20 smithi064 conmon[107122]: audit 2023-06-10T20:56:20.541538+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T20:56:21.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:20 smithi064 conmon[107122]: audit 2023-06-10T20:56:20.542241+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:21.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:20 smithi064 conmon[97224]: audit 2023-06-10T20:56:20.541538+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T20:56:21.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:20 smithi064 conmon[97224]: audit 2023-06-10T20:56:20.542241+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:20 smithi152 conmon[97365]: audit 2023-06-10T20:56:20.541538+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T20:56:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:20 smithi152 conmon[97365]: audit 2023-06-10T20:56:20.542241+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:22.079 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[97224]: cluster 2023-06-10T20:56:20 2023-06-10T20:56:22.079 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[97224]: .500809+0000 mgr.y (mgr.14148) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:22.079 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[97224]: cephadm 2023-06-10T20:56:20.542826+0000 mgr.y (mgr.14148) 100 : cephadm [INF] Deploying daemon osd.3 on smithi064 2023-06-10T20:56:22.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[107122]: cluster 2023-06-10T20:56:20.500809 2023-06-10T20:56:22.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:22.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:21 smithi064 conmon[107122]: cephadm 2023-06-10T20:56:20.542826+0000 mgr.y (mgr.14148) 100 : cephadm [INF] Deploying daemon osd.3 on smithi064 2023-06-10T20:56:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:21 smithi152 conmon[97365]: cluster 2023-06-10T20:56:20.500809+0000 mgr.y (mgr.14148) 99 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:21 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:20.542826+0000 mgr.y (mgr.14148) 100 : cephadm [INF] Deploying daemon osd.3 on smithi064 2023-06-10T20:56:24.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:23 smithi064 conmon[97224]: cluster 2023-06-10T20:56:22.501265+0000 mgr.y (mgr.14148) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:24.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:23 smithi064 conmon[97224]: 2023-06-10T20:56:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:23 smithi064 conmon[107122]: cluster 2023-06-10T20:56:22.501265+0000 2023-06-10T20:56:24.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:23 smithi064 conmon[107122]: mgr.y (mgr.14148) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:24.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:23 smithi152 conmon[97365]: cluster 2023-06-10T20:56:22.501265+0000 mgr.y (mgr.14148) 101 : cluster [DBG] pgmap v79: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:26.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[107122]: cluster 2023-06-10T20:56:24.501767+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:26.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[107122]: audit 2023-06-10T20:56:25.626543+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:26.262 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[107122]: audit 2023-06-10T20:56:25.628666+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:26.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[97224]: cluster 2023-06-10T20:56 2023-06-10T20:56:26.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[97224]: :24.501767+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:26.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[97224]: audit 2023-06-10T20:56:25.626543+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:26.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:25 smithi064 conmon[97224]: audit 2023-06-10T20:56:25.628666+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:25 smithi152 conmon[97365]: cluster 2023-06-10T20:56:24.501767+0000 mgr.y (mgr.14148) 102 : cluster [DBG] pgmap v80: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:25 smithi152 conmon[97365]: audit 2023-06-10T20:56:25. 2023-06-10T20:56:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:25 smithi152 conmon[97365]: 626543+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:25 smithi152 conmon[97365]: audit 2023-06-10T20:56:25.628666+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:28.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:27 smithi064 conmon[97224]: cluster 2023-06-10T20:56:26.502086+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:28.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:27 smithi064 conmon[107122]: cluster 2023-06-10T20:56:26.502086+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:27 smithi152 conmon[97365]: cluster 2023-06-10T20:56:26.502086+0000 mgr.y (mgr.14148) 103 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:28.871 INFO:teuthology.orchestra.run.smithi064.stdout:Created osd(s) 3 on host 'smithi064' 2023-06-10T20:56:29.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:29.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 27.996449+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 2023-06-10T20:56:28.325577+0000 mon.a (mon.0) 349 : 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 56:28.326403+0000 mon.a (mon.0) 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 350 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 06-10T20:56:28.333059+0000 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: mon.a (mon.0) 351 : audit [INF] 2023-06-10T20:56:29.079 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20:56 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: :28.827128+0000 mon.c (mon.2) 7 : 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit [INF] from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 2023-06-10T20:56:28.827457+0000 mon.a (mon.0) 352 : audit 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.080 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20:56:28 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: .867824+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: :56:28.900085+0000 mon.a (mon.0) 354 : audit [DBG] 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:27.996449+0000 mon.a ( 2023-06-10T20:56:29.081 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:28.325577+0000 mon.a (mon.0) 349 : audit [DBG] 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 2023-06-10T20:56:28.326403 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: +0000 mon.a (mon.0) 350 : audit 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:29.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 2023-06- 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 10T20:56:28.333059+0000 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: mon.a (mon.0) 351 : audit 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 2023-06-10 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: T20:56:28.827128+0000 2023-06-10T20:56:29.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: mon.c (mon.2) 7 : audit [INF] 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 56:28.827457+0000 mon.a ( 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: mon.0) 352 : audit [INF] 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: 10T20:56:28.867824 2023-06-10T20:56:29.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: +0000 mon.a (mon.0) 353 2023-06-10T20:56:29.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: audit 2023-06-10T20:56:28 2023-06-10T20:56:29.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: .900085+0000 mon.a (mon.0) 354 : audit [DBG] 2023-06-10T20:56:29.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:28 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:29.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023-06-10T20:56:27.996449+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023-06-10T20:56: 2023-06-10T20:56:29.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 28.325577+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:29.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023-06-10T20:56:28.326403+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:29.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023-06-10T20:56:28.333059 2023-06-10T20:56:29.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: +0000 mon.a (mon.0) 351 : audit [INF] 2023-06-10T20:56:29.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:56:29.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 20:56:28.827128+0000 mon.c (mon. 2023-06-10T20:56:29.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 2) 7 : audit [INF] from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:29.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 06-10T20:56:28.827457+0000 2023-06-10T20:56:29.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: mon.a (mon.0) 352 : audit 2023-06-10T20:56:29.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T20:56:29.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:29.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 06-10T20:56:28.867824 2023-06-10T20:56:29.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: +0000 mon.a (mon.0) 353 : audit 2023-06-10T20:56:29.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:29.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:29.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: 06-10T20:56:28.900085+0000 2023-06-10T20:56:29.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: mon.a (mon.0) 354 : audit 2023-06-10T20:56:29.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:28 smithi152 conmon[97365]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:29.551 DEBUG:teuthology.orchestra.run.smithi064:osd.3> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3.service 2023-06-10T20:56:29.554 INFO:tasks.cephadm:Deploying osd.4 on smithi152 with /dev/vg_nvme/lv_4... 2023-06-10T20:56:29.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-06-10T20:56:29.584 INFO:journalctl@ceph.osd.3.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:56:30.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: cluster 2023-06-10T20:56:28.502323+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:30.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.347695+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T20:56:30.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: cluster 2023-06-10T20:56:29.347780+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-10T20:56:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.347920+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.348308+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[107122]: audit 2023-06-10T20:56:29.348611+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: cluster 2023-06-10T20:56:28. 2023-06-10T20:56:30.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: 502323+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:30.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.347695+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T20:56:30.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: cluster 2023-06-10T20:56:29.347780+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-10T20:56:30.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.347920+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:30.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.348308+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[97224]: audit 2023-06-10T20:56:29.348611+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: cluster 2023-06-10T20:56:28.502323+0000 mgr.y (mgr.14148) 104 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:30.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: audit 2023-06-10T20:56:29.347695+0000 mon.a (mon.0) 355 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T20:56:30.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: cluster 2023-06-10T20:56:29.347780+0000 mon.a (mon.0) 356 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-10T20:56:30.381 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: audit 2023-06-10T20:56:29.347920+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:30.381 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: audit 2023-06-10T20:56:29.348308+0000 mon.c (mon.2) 8 : audit [INF] from='osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.381 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:30 smithi152 conmon[97365]: audit 2023-06-10T20:56:29.348611+0000 mon.a (mon.0) 358 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T20:56:30.608 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[129869]: debug 2023-06-10T20:56:30.347+0000 7f65727bd700 -1 osd.3 0 waiting for initial osdmap 2023-06-10T20:56:30.609 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 20:56:30 smithi064 conmon[129869]: debug 2023-06-10T20:56:30.352+0000 7f656af32700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:56:31.496 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-06-10T20:56:31.496 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-06-10T20:56:31.497 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2023-06-10T20:56:31.497 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2023-06-10T20:56:31.497 INFO:teuthology.orchestra.run.smithi152.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111302 s, 942 MB/s 2023-06-10T20:56:31.497 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2023-06-10T20:56:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: audit 2023-06-10T20:56:30.343862+0000 mon.a ( 2023-06-10T20:56:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: cluster 2023-06-10T20:56:30.343963+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: audit 2023-06-10T20:56:30.344068+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: audit 2023-06-10T20:56:30.349490+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[97224]: cluster 2023-06-10T20:56:30.502702+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: 56:30.343862+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: cluster 2023-06-10T20:56:30.343963+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-10T20:56:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: audit 2023-06-10T20:56:30.344068+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: audit 2023-06-10T20:56:30.349490+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:31 smithi064 conmon[107122]: cluster 2023-06-10T20:56:30.502702+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:31.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: audit 2023-06-10T20:56:30.343862+0000 mon.a (mon. 2023-06-10T20:56:31.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: 0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]': finished 2023-06-10T20:56:31.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: cluster 2023-06-10T20:56:30.343963+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-10T20:56:31.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: audit 2023-06-10T20:56:30.344068+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: audit 2023-06-10T20:56:30.349490+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:31.689 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:31 smithi152 conmon[97365]: cluster 2023-06-10T20:56:30.502702+0000 mgr.y (mgr.14148) 105 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-10T20:56:32.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi152:vg_nvme/lv_4 2023-06-10T20:56:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[97224]: cluster 2023-06-10T20:56:29.868442+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[97224]: cluster 2023-06-10T20:56:29.868486+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[97224]: cluster 2023-06-10T20:56:31.347763+0000 mon.a (mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195] boot 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[97224]: cluster 2023-06-10T20:56:31.347821+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[97224]: audit 2023-06-10T20:56:31.348276+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: cluster 2023-06-10T20:56:29.868442 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: +0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: cluster 2023-06-10T20:56:32.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: 2023-06-10T20:56:29.868486+0000 osd.3 (osd.3) 2 : cluster [DBG] 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: purged_snaps scrub ok 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: -06-10T20:56:31.347763+0000 mon.a ( 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195] boot 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: cluster 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: 2023-06-10T20:56:31. 2023-06-10T20:56:32.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: 347821+0000 mon.a (mon.0) 364 : cluster 2023-06-10T20:56:32.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-10T20:56:32.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:32.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: 06-10T20:56:31.348276+0000 mon.a ( 2023-06-10T20:56:32.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:32 smithi064 conmon[107122]: mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:32.683 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:32 smithi152 conmon[97365]: cluster 2023-06-10T20:56:29.868442+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:32.683 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:32 smithi152 conmon[97365]: cluster 2023-06-10T20:56:29.868486+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:32.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:32 smithi152 conmon[97365]: cluster 2023-06-10T20:56:31.347763+0000 mon.a (mon.0) 363 : cluster [INF] osd.3 [v2:172.21.15.64:6826/791323195,v1:172.21.15.64:6827/791323195] boot 2023-06-10T20:56:32.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:32 smithi152 conmon[97365]: cluster 2023-06-10T20:56:31.347821+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-10T20:56:32.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:32 smithi152 conmon[97365]: audit 2023-06-10T20:56:31.348276+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:56:33.485 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: cluster 2023-06-10T20:56:32.357300+0000 mon.a (mon.0) 366 2023-06-10T20:56:33.485 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:32.392955+0000 mgr.y (mgr.14148) 106 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.397509+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.398476+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.399242+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.399965+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.400704+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:32.401214+0000 mgr.y (mgr.14148) 107 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 4036M 2023-06-10T20:56:33.487 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.404667+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.487 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.405569+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:33.487 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.406231+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:33.487 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: audit 2023-06-10T20:56:32.410567+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.487 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:33 smithi152 conmon[97365]: cluster 2023-06-10T20:56:32.503188+0000 mgr.y (mgr.14148) 108 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: cluster 2023-06-10T20:56:32.357300+0000 mon.a (mon 2023-06-10T20:56:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: .0) 366 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: cephadm 2023-06-10T20:56 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: :32.392955+0000 mgr.y (mgr.14148) 106 : cephadm 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: -06-10T20:56:32.397509 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: +0000 mon.a (mon.0) 367 : 2023-06-10T20:56:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 06-10T20:56:32.398476+0000 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: mon.a (mon.0) 368 : 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: -06-10T20:56:32.399242 2023-06-10T20:56:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: +0000 mon.a (mon.0) 369 : 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: -06-10T20:56:32. 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 399965+0000 mon.a (mon.0 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: ) 370 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 2023-06-10T20:56:32 2023-06-10T20:56:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: .400704+0000 mon.a (mon.0 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: ) 371 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: cephadm 2023 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: -06-10T20:56:32 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: .401214+0000 mgr.y (mgr.14148 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: ) 107 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 4036M 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: -10T20:56:32.404667+0000 mon.a (mon.0) 372 : audit [INF] 2023-06-10T20:56:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 56:32.405569+0000 mon.a (mon.0) 373 : audit 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023-06-10T20:56:32.406231+0000 mon.a ( 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: mon.0) 374 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 06-10T20:56:32.410567+0000 mon.a (mon. 2023-06-10T20:56:33.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 0) 375 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: 06-10T20:56:32.503188+0000 mgr.y (mgr 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[97224]: .14148) 108 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: cluster 2023-06-10T20:56:32.357300 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: +0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-10T20:56:33.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: cephadm 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56:32.392955+0000 mgr.y (mgr.14148 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: ) 106 : cephadm [INF] Detected new or changed devices on smithi064 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56: 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 32.397509+0000 mon.a (mon.0) 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 367 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:33.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 06-10T20:56:32.398476+0000 mon.a (mon 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: .0) 368 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 20:56:32.399242+0000 mon.a (mon. 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 0) 369 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: -06-10T20:56:32 2023-06-10T20:56:33.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: .399965+0000 mon.a (mon.0) 370 : 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 10T20:56:32.400704+0000 mon.a 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: (mon.0) 371 : audit 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T20:56:33.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 10T20:56:32.401214+0000 mgr.y ( 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: mgr.14148) 107 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 4036M 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56: 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 32.404667+0000 mon.a (mon 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: .0) 372 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: :56:32.405569+0000 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: mon.a (mon.0) 373 : audit 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56:32.406231+0000 mon.a 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: (mon.0) 374 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 2023-06-10T20:56:33.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:33.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: 06-10T20:56:32.410567+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:33.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T20:56:33.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: T20:56:32.503188+0000 mgr.y (mgr.14148) 108 : cluster [DBG] 2023-06-10T20:56:33.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:33 smithi064 conmon[107122]: pgmap v88: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:34.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:34 smithi152 conmon[97365]: audit 2023-06-10T20:56:33.788462+0000 mgr.y (mgr.14148) 109 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:34.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:34 smithi152 conmon[97365]: audit 2023-06-10T20:56:33.791570+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:34.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:34 smithi152 conmon[97365]: audit 2023-06-10T20:56:33.794465+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:34.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:34 smithi152 conmon[97365]: audit 2023-06-10T20:56:33.795310+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[97224]: audit 2023-06-10T20:56:33.788462+0000 mgr.y (mgr.14148) 2023-06-10T20:56:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[97224]: 109 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:34.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[97224]: audit 2023-06-10T20:56:33.791570+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:34.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[97224]: audit 2023-06-10T20:56:33.794465+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:34.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[97224]: audit 2023-06-10T20:56:33.795310+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.788462+0000 mgr.y (mgr 2023-06-10T20:56:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: .14148) 109 : audit [DBG] from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: audit 2023-06-10T20:56:33.791570+0000 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: mon.a (mon.0) 376 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: audit 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: 2023-06- 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: 10T20:56:33.794465+0000 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: mon.a (mon.0) 377 : audit [INF] 2023-06-10T20:56:34.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: audit 2023-06-10T20:56:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:56:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: T20:56:33.795310+0000 mon.a 2023-06-10T20:56:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: (mon.0) 378 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:34 smithi064 conmon[107122]: 2023-06-10T20:56:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:35 smithi152 conmon[97365]: cluster 2023-06-10T20:56:34.503612+0000 mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:35 smithi064 conmon[97224]: cluster 2023-06-10T20:56:34.503612+0000 mgr.y (mgr.14148) 2023-06-10T20:56:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:35 smithi064 conmon[97224]: 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:35 smithi064 conmon[107122]: cluster 2023-06-10T20:56:34.503612+0000 2023-06-10T20:56:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:35 smithi064 conmon[107122]: mgr.y (mgr.14148) 110 : cluster [DBG] pgmap v89: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:36 smithi152 conmon[97365]: audit 2023-06-10T20:56:35.839100+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.152:0/2802457743' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:36 smithi152 conmon[97365]: audit 2023-06-10T20:56:35.839700+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:36 smithi152 conmon[97365]: audit 2023-06-10T20:56:35.844626+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]': finished 2023-06-10T20:56:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:36 smithi152 conmon[97365]: cluster 2023-06-10T20:56:35.844737+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-10T20:56:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:36 smithi152 conmon[97365]: audit 2023-06-10T20:56:35.844896+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: audit 2023-06-10T20:56:35.839100+0000 mon.b (mon. 2023-06-10T20:56:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: 1) 7 : audit [INF] from='client.? 172.21.15.152:0/2802457743' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: audit 2023-06-10T20:56:35.839700+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: audit 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: 2023-06-10T20:56:35.844626+0000 mon.a (mon.0) 380 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]': finished 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: cluster 2023-06-10T 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: 20:56:35.844737+0000 mon.a (mon.0 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: ) 381 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: 10T20:56:35.844896+0000 mon.a ( 2023-06-10T20:56:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[97224]: mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: audit 2023-06-10T20:56:35.839100+0000 mon.b (mon 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: .1) 7 : audit [INF] from='client.? 172.21.15.152:0/2802457743' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: 56:35.839700+0000 mon.a (mon.0) 379 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]: dispatch 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:56:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: -10T20:56:35.844626+0000 mon.a (mon.0) 380 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e5f714f-9f72-4bd3-9e4d-97286fbfb88e"}]': finished 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: -10T20:56:35.844737+0000 mon.a (mon.0) 381 : 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: audit 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: 2023-06-10T20:56:35.844896+0000 mon.a 2023-06-10T20:56:36.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:36 smithi064 conmon[107122]: (mon.0) 382 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:37 smithi152 conmon[97365]: audit 2023-06-10T20:56:36.443522+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.152:0/2750805646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:37.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:37 smithi152 conmon[97365]: cluster 2023-06-10T20:56:36.503991+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:37.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[97224]: audit 2023-06-10T20:56:36.443522+0000 mon.b 2023-06-10T20:56:37.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[97224]: (mon.1) 8 : audit [DBG] from='client.? 172.21.15.152:0/2750805646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[97224]: cluster 2023-06-10T20:56:36.503991+0000 mgr.y (mgr.14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[107122]: audit 2023-06-10T20:56:36.443522+0000 mon.b (mon. 2023-06-10T20:56:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[107122]: 1) 8 : audit [DBG] from='client.? 172.21.15.152:0/2750805646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[107122]: cluster 2023-06-10T20:56:36.503991+0000 mgr.y 2023-06-10T20:56:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:37 smithi064 conmon[107122]: (mgr.14148) 111 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:39.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:39 smithi152 conmon[97365]: cluster 2023-06-10T20:56:38.504528+0000 mgr.y (mgr. 2023-06-10T20:56:39.826 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:39 smithi152 conmon[97365]: 14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:39 smithi064 conmon[97224]: cluster 2023-06-10T20:56:38.504528+0000 2023-06-10T20:56:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:39 smithi064 conmon[97224]: mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:39.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:39 smithi064 conmon[107122]: cluster 2023-06-10T20:56:38 2023-06-10T20:56:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:39 smithi064 conmon[107122]: .504528+0000 mgr.y (mgr.14148) 112 : cluster [DBG] pgmap v92: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:42.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:41 smithi152 conmon[97365]: cluster 2023-06-10T20:56:40.505016+0000 mgr.y (mgr.14148) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:42.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:41 smithi152 conmon[97365]: audit 2023-06-10T20:56:41.706799+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T20:56:42.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:41 smithi152 conmon[97365]: audit 2023-06-10T20:56:41.707729+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[97224]: cluster 2023-06-10T20:56:40.505016+0000 mgr.y (mgr.14148) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[97224]: audit 2023-06-10T20:56:41.706799+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T20:56:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[97224]: T20:56:41.707729+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: cluster 2023-06-10T20:56:40 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: .505016+0000 mgr.y (mgr.14148) 113 : cluster [DBG] pgmap v93: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: :56:41.706799+0000 mon.a ( 2023-06-10T20:56:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: mon.0) 383 : audit [INF] 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: -06-10T20:56: 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: 41.707729+0000 mon.a (mon. 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: 0) 384 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:41 smithi064 conmon[107122]: 2023-06-10T20:56:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:42 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:41.708507+0000 mgr.y (mgr 2023-06-10T20:56:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:42 smithi152 conmon[97365]: .14148) 114 : cephadm [INF] Deploying daemon osd.4 on smithi152 2023-06-10T20:56:43.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:42 smithi064 conmon[97224]: cephadm 2023-06-10T20:56:41.708507+0000 mgr.y (mgr.14148) 114 : cephadm [INF] 2023-06-10T20:56:43.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:42 smithi064 conmon[97224]: Deploying daemon osd.4 on smithi152 2023-06-10T20:56:43.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:42 smithi064 conmon[107122]: cephadm 2023-06-10T20:56: 2023-06-10T20:56:43.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:42 smithi064 conmon[107122]: 41.708507+0000 mgr.y (mgr.14148) 114 : cephadm [INF] Deploying daemon osd.4 on smithi152 2023-06-10T20:56:44.189 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:43 smithi152 conmon[97365]: cluster 2023-06-10T20:56:42.505439+0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:43 smithi064 conmon[97224]: cluster 2023-06-10T20:56:42.505439+0000 mgr.y (mgr.14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:43 smithi064 conmon[107122]: cluster 2023-06-10T20:56:42.505439+0000 mgr.y (mgr 2023-06-10T20:56:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:43 smithi064 conmon[107122]: .14148) 115 : cluster [DBG] pgmap v94: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:46.237 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:45 smithi152 conmon[97365]: cluster 2023-06-10T20:56:44.505891+0000 mgr.y (mgr.14148) 116 : cluster [DBG] 2023-06-10T20:56:46.237 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:45 smithi152 conmon[97365]: pgmap v95: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:45 smithi064 conmon[97224]: cluster 2023-06-10T20:56:44.505891+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:46.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:45 smithi064 conmon[107122]: cluster 2023-06-10T20:56:44.505891+0000 mgr.y (mgr.14148) 116 : cluster [DBG] pgmap v95: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:47 smithi152 conmon[97365]: audit 2023-06-10T20:56:46.276386+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:47 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:56:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:47 smithi152 conmon[97365]: 06-10T20:56:46.313402+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:47.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:47 smithi152 conmon[97365]: cluster 2023-06-10T20:56:46.506333+0000 mgr.y (mgr.14148) 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: audit 2023-06-10T20:56:46.276386+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: audit 2023-06-10T20:56:46.313402+0000 mon.a (mon 2023-06-10T20:56:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: .0) 386 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: cluster 2023-06-10T20:56:47.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: 2023-06-10T20:56:46. 2023-06-10T20:56:47.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: 506333+0000 mgr.y (mgr.14148) 2023-06-10T20:56:47.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[97224]: 117 : cluster [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:47.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: audit 2023-06-10T20:56:46.276386+0000 2023-06-10T20:56:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: mon.a (mon.0) 385 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: audit 2023-06-10T20:56:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: 2023-06-10T20:56:46.313402 2023-06-10T20:56:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: +0000 mon.a (mon.0) 386 2023-06-10T20:56:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:56:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: -10T20:56:46.506333+0000 2023-06-10T20:56:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: mgr.y (mgr.14148) 117 : cluster 2023-06-10T20:56:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:47 smithi064 conmon[107122]: [DBG] pgmap v96: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:48.862 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 4 on host 'smithi152' 2023-06-10T20:56:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:47.920721+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:48.232598+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:48.233651+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:49.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:48.239095+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:48.858155+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:48 smithi152 conmon[97365]: audit 2023-06-10T20:56:48.858477+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:49.390 DEBUG:teuthology.orchestra.run.smithi152:osd.4> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4.service 2023-06-10T20:56:49.395 INFO:tasks.cephadm:Deploying osd.5 on smithi152 with /dev/vg_nvme/lv_3... 2023-06-10T20:56:49.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-06-10T20:56:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023-06-10T20:56:47.920721+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023-06-10T20:56:48.232598+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023 2023-06-10T20:56:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: -06-10T20:56:48.233651+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:56:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: 20:56:48.239095+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:56:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: -10T20:56:48.858155+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: audit 2023-06-10T20:56:48.858477+0000 2023-06-10T20:56:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[97224]: mon.a (mon.0) 392 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023-06-10T20:56:47.920721+0000 mon.a (mon 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: .0) 387 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023-06-10T20:56:48 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: .232598+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: -06-10T20:56:48 2023-06-10T20:56:49.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: .233651+0000 mon.a (mon.0) 389 : 2023-06-10T20:56:49.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:49.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:56:49.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 06-10T20:56:48 2023-06-10T20:56:49.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: .239095+0000 mon.a (mon.0) 390 : 2023-06-10T20:56:49.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023-06-10T20:56:49.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 2023-06-10T20:56: 2023-06-10T20:56:49.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 48.858155+0000 mon.a (mon.0) 2023-06-10T20:56:49.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 391 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:49.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: audit 2023-06-10T20:56:49.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 2023-06-10T20:56 2023-06-10T20:56:49.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: :48.858477+0000 mon.a (mon.0) 2023-06-10T20:56:49.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:48 smithi064 conmon[107122]: 392 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:56:49.424 INFO:journalctl@ceph.osd.4.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:56:49.668 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 20:56:49 smithi152 conmon[103728]: debug 2023-06-10T20:56:49.482+0000 7f1f4db80200 -1 osd.4 0 log_to_monitors {default=true} 2023-06-10T20:56:50.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:49 smithi152 conmon[97365]: cluster 2023-06-10T20:56:48.506670 2023-06-10T20:56:50.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:49 smithi152 conmon[97365]: +0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:50.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:49 smithi152 conmon[97365]: audit 2023-06-10T20:56:49.487027+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:50.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:49 smithi152 conmon[97365]: audit 2023-06-10T20:56:49.487424+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[97224]: cluster 2023-06-10T20:56:48.506670+0000 mgr.y (mgr.14148) 118 : 2023-06-10T20:56:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[97224]: cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[97224]: audit 2023-06-10T20:56:49.487027+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[97224]: audit 2023-06-10T20:56:49.487424+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[107122]: cluster 2023-06-10T20:56:48.506670+0000 mgr.y (mgr.14148) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[107122]: 2023-06-10T20:56:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[107122]: audit 2023-06-10T20:56:49.487027+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:50.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:49 smithi064 conmon[107122]: audit 2023-06-10T20:56:49.487424+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T20:56:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: audit 2023-06-10T20:56:50.245715+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T20:56:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: cluster 2023-06-10T20:56:50.245826+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-10T20:56:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: audit 2023-06-10T20:56:50.246005+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: audit 2023-06-10T20:56:50.246490+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: audit 2023-06-10T20:56:50.246824+0000 mon.a (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[97224]: cluster 2023-06-10T20:56:50.507022+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: audit 2023-06-10T20:56:50.245715+0000 mon.a (mon.0) 394 : 2023-06-10T20:56:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T20:56:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: cluster 2023-06-10T20:56:50.245826+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-10T20:56:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: audit 2023-06-10T20:56:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: 2023-06-10T20:56:50.246005+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: audit 2023-06-10T20:56:50.246490+0000 mon.b 2023-06-10T20:56:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: -06-10T20:56:50.246824+0000 mon.a (mon.0 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: ) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: -06-10T20:56:50.507022+0000 mgr.y (mgr. 2023-06-10T20:56:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:51 smithi064 conmon[107122]: 14148) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:51.695 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[103728]: debug 2023-06-10T20:56:51.249+0000 7f1f44100700 -1 osd.4 0 waiting for initial osdmap 2023-06-10T20:56:51.695 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[103728]: debug 2023-06-10T20:56:51.255+0000 7f1f3d076700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:56:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: audit 2023-06-10T20:56:50.245715+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T20:56:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: cluster 2023-06-10T20:56:50.245826+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-10T20:56:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: audit 2023-06-10T20:56:50.246005+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: audit 2023-06-10T20:56:50.246490+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: audit 2023-06-10T20:56:50.246824+0000 mon.a (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:56:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:51 smithi152 conmon[97365]: cluster 2023-06-10T20:56:50.507022+0000 mgr.y (mgr.14148) 119 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-10T20:56:51.845 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-06-10T20:56:51.846 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-06-10T20:56:51.846 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2023-06-10T20:56:51.846 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2023-06-10T20:56:51.846 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124941 s, 839 MB/s 2023-06-10T20:56:51.846 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2023-06-10T20:56:52.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:52 smithi152 conmon[97365]: audit 2023-06-10T20:56:51.247113+0000 mon.a (mon.0) 398 : audit [INF] 2023-06-10T20:56:52.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:52 smithi152 conmon[97365]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:56:52.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:52 smithi152 conmon[97365]: cluster 2023-06-10T20:56:51.247264+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-10T20:56:52.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:52 smithi152 conmon[97365]: audit 2023-06-10T20:56:51.247902+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:52 smithi152 conmon[97365]: audit 2023-06-10T20:56:51.253376+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: audit 2023-06-10T20:56:51.247113+0000 mon.a ( 2023-06-10T20:56:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:56:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: cluster 2023-06-10T20:56:51.247264+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-10T20:56:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: audit 2023-06-10T20:56:51.247902+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[97224]: 56:51.253376+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: audit 2023-06-10T20:56:51.247113+0000 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: cluster 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: 2023-06-10T20:56:51.247264+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-10T20:56:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: -06-10T20:56:51.247902+0000 mon.a (mon.0) 400 : 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: audit 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: 2023-06-10T20:56:51 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: .253376+0000 mon.a (mon.0) 401 : audit [DBG] 2023-06-10T20:56:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:52 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:52.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi152:vg_nvme/lv_3 2023-06-10T20:56:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:50.531448+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:50.531495+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:52.250514+0000 mon.a (mon.0) 402 : cluster [INF] osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563] boot 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:52.250591+0000 mon.a 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: :56:52.250887+0000 mon.a (mon.0) 404 : audit [DBG] 2023-06-10T20:56:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:52.507504 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 120 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cephadm 2023 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: -06-10T20:56:52.870771+0000 mgr.y (mgr.14148 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: ) 121 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:56:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: -10T20:56:52.875619+0000 mon.a ( 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: mon.0) 405 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T20:56:52 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: .876734+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cephadm 2023-06- 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 10T20:56:52.877489+0000 mgr.y (mgr.14148) 122 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: : cephadm [INF] Adjusting osd_memory_target on smithi152 to 17168M 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:56:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 56:52.882142+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 2023-06-10T20:56:52.883369+0000 mon.a (mon. 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 0) 408 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 20:56:52.884251+0000 mon.a (mon.0) 409 : 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: audit 2023-06-10T20:56:52 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: .889330+0000 mon.a (mon.0) 410 : audit [INF] 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: cluster 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[97224]: 2023-06-10T20:56:53.250033+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023-06-10T20:56 2023-06-10T20:56:53.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: :50.531448+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:50 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: .531495+0000 osd.4 (osd. 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:56:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: :56:52.250514+0000 mon.a (mon. 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 0) 402 : cluster [INF] 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563] boot 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:52 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: .250591+0000 mon.a (mon.0) 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-10T20:56:53.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: -06-10T20:56:52.250887+0000 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: mon.a (mon.0) 404 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: -06-10T20:56:52.507504+0000 mgr.y ( 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: mgr.14148) 120 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T20:56:53.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 10T20:56:52.870771+0000 mgr.y ( 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: mgr.14148) 121 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: :52.875619+0000 mon.a (mon. 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 0) 405 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06-10T20:56:53.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:52. 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 876734+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cephadm 2023-06-10T20:56 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: :52.877489+0000 mgr.y (mgr.14148) 122 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 17168M 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06-10T20:56:52.882142 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: +0000 mon.a (mon.0) 407 : audit [INF] 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: -10T20:56:52.883369+0000 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: mon.a (mon.0) 408 : 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 10T20:56:52.884251+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:53.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: audit 2023-06-10T20:56:53.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: 2023-06-10T20:56:52.889330+0000 mon.a (mon.0) 410 2023-06-10T20:56:53.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:53 smithi064 conmon[107122]: cluster 2023-06-10T20:56:53.250033+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-10T20:56:53.676 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023-06-10T20:56:50.531448+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:56:53.676 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023-06-10T20:56:50.531495+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:56:53.676 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023-06-10T20:56:52.250514+0000 mon.a (mon.0) 402 : cluster [INF] osd.4 [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563] boot 2023-06-10T20:56:53.676 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023-06-10T20:56:52.250591+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52.250887+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023-06-10T20:56:52.507504+0000 mgr.y (mgr.14148) 120 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:52.870771 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: +0000 mgr.y (mgr.14148) 121 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52.875619+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52.876734+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:56:53.677 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cephadm 2023-06-10T20:56:52.877489+0000 mgr.y (mgr.14148) 122 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 17168M 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52.882142+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52.883369+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: 10T20:56:52.884251+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: audit 2023-06-10T20:56:52 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: .889330+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:56:53.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: cluster 2023- 2023-06-10T20:56:53.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: 06-10T20:56:53.250033+0000 mon.a ( 2023-06-10T20:56:53.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:53 smithi152 conmon[97365]: mon.0) 411 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-10T20:56:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: cluster 2023-06-10T20:56:54.251171+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-06-10T20:56:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: cluster 2023-06-10T20:56:54.508056+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: audit 2023-06-10T20:56:54.638919+0000 mgr.y (mgr.14148) 124 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: audit 2023-06-10T20:56:54.641952+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: audit 2023-06-10T20:56:54.644783+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:55 smithi152 conmon[97365]: audit 2023-06-10T20:56:54.645655+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: cluster 2023-06-10T20:56:54.251171+0000 mon.a (mon.0) 412 : cluster [DBG] 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: osdmap e32: 5 total, 5 up, 5 in 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: cluster 2023-06-10T20:56:54.508056+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: audit 2023-06-10T20:56:54.638919+0000 mgr.y (mgr.14148) 124 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: audit 2023-06-10T20:56:54.641952+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: audit 2023-06-10T20:56:54.644783+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[97224]: audit 2023-06-10T20:56:54.645655+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: cluster 2023-06-10T20:56:54.251171+0000 2023-06-10T20:56:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: mon.a (mon.0) 412 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-06-10T20:56:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: cluster 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: 2023-06-10T20:56:54.508056+0000 mgr.y (mgr.14148) 123 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: T20:56:54.638919+0000 mgr.y (mgr.14148) 124 : audit [DBG] from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: audit 2023-06-10T20:56 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: :54.641952+0000 mon.a (mon.0) 413 : audit 2023-06-10T20:56:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:56:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: audit 2023-06-10T20:56:54 2023-06-10T20:56:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: .644783+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:56:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: audit 2023 2023-06-10T20:56:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: -06-10T20:56:54.645655+0000 mon.a (mon.0) 415 : audit 2023-06-10T20:56:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:55 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:56:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: audit 2023-06-10T20:56:56.758985 2023-06-10T20:56:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: +0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.152:0/3774390232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: audit 2023-06-10T20:56:56.759519+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: audit 2023-06-10T20:56:56.764870+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]': finished 2023-06-10T20:56:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: cluster 2023-06-10T20:56:56.764976+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-10T20:56:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[97224]: audit 2023-06-10T20:56:56.765156+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:56:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: audit 2023-06-10T20:56:56.758985 2023-06-10T20:56:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: +0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.152:0/3774390232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: 56:56.759519+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: 56.764870+0000 mon.a (mon.0) 417 : audit [INF] 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]': finished 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: cluster 2023-06-10T20:56 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: :56.764976+0000 mon.a (mon.0) 418 : 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-10T20:56:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: audit 2023-06-10T20:56: 2023-06-10T20:56:57.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: 56.765156+0000 mon.a (mon.0) 419 : audit [DBG] 2023-06-10T20:56:57.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:56 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:56:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: audit 2023-06-10T20:56:56.758985+0000 mon.b (mon.1 2023-06-10T20:56:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: ) 11 : audit [INF] from='client.? 172.21.15.152:0/3774390232' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: audit 2023-06-10T20:56:56.759519+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]: dispatch 2023-06-10T20:56:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: audit 2023-06-10T20:56:56.764870+0000 mon.a (mon.0) 417 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb69283f-ccc4-4166-8c9a-05522cd8fd5c"}]': finished 2023-06-10T20:56:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: cluster 2023-06-10T20:56:56.764976+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-10T20:56:57.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:56 smithi152 conmon[97365]: audit 2023-06-10T20:56:56.765156+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:56:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[107122]: cluster 2023-06-10T20:56:56.508531+0000 mgr.y ( 2023-06-10T20:56:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[107122]: mgr.14148) 125 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[107122]: audit 2023-06-10T20:56:57.329216+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.152:0/3722245990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[107122]: 2023-06-10T20:56:58.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[97224]: cluster 2023-06-10T20:56:56.508531+0000 mgr.y 2023-06-10T20:56:58.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[97224]: (mgr.14148) 125 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:58.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[97224]: audit 2023-06-10T20:56:58.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[97224]: 2023-06-10T20:56:57.329216+0000 mon.b (mon.1 2023-06-10T20:56:58.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:57 smithi064 conmon[97224]: ) 12 : audit [DBG] from='client.? 172.21.15.152:0/3722245990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:56:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:57 smithi152 conmon[97365]: cluster 2023-06-10T20:56:56.508531+0000 mgr.y (mgr.14148) 125 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:56:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:57 smithi152 conmon[97365]: audit 2023-06-10T20:56:57.329216+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.152:0/3722245990' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:56:59 smithi064 conmon[97224]: cluster 2023-06-10T20:56:58.508884+0000 mgr.y (mgr.14148) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:59 smithi064 conmon[107122]: cluster 2023-06-10T20:56:58.508884+0000 mgr.y (mgr.14148) 126 : cluster 2023-06-10T20:57:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:56:59 smithi064 conmon[107122]: [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:56:59 smithi152 conmon[97365]: cluster 2023-06-10T20:56:58.508884+0000 mgr.y (mgr.14148) 126 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:02.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:01 smithi064 conmon[97224]: cluster 2023-06-10T20:57:00.509458+0000 mgr.y (mgr.14148) 127 2023-06-10T20:57:02.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:01 smithi064 conmon[97224]: : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:02.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:01 smithi064 conmon[107122]: cluster 2023-06-10T20:57:00.509458+0000 mgr.y (mgr.14148 2023-06-10T20:57:02.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:01 smithi064 conmon[107122]: ) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:01 smithi152 conmon[97365]: cluster 2023-06-10T20:57:00.509458+0000 mgr.y (mgr.14148) 127 : cluster [DBG] pgmap v109: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:03.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:02 smithi152 conmon[97365]: audit 2023-06-10T20:57:02.528487+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T20:57:03.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:02 smithi152 conmon[97365]: audit 2023-06-10T20:57:02.529397+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[97224]: audit 2023-06-10T20:57:02.528487+0000 mon.a (mon.0 2023-06-10T20:57:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[97224]: ) 420 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T20:57:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[97224]: audit 2023-06-10T20:57:02.529397+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:03.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[107122]: audit 2023-06-10T20:57:02.528487+0000 mon.a (mon 2023-06-10T20:57:03.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[107122]: .0) 420 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T20:57:03.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[107122]: audit 2023-06-10T20:57:02.529397+0000 mon.a ( 2023-06-10T20:57:03.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:02 smithi064 conmon[107122]: mon.0) 421 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:04.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:03 smithi152 conmon[97365]: cluster 2023-06-10T20:57:02.509863+0000 mgr.y (mgr.14148 2023-06-10T20:57:04.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:03 smithi152 conmon[97365]: ) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:04.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:03 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:02.530221+0000 mgr.y (mgr.14148) 129 : cephadm [INF] Deploying daemon osd.5 on smithi152 2023-06-10T20:57:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[97224]: cluster 2023-06-10T20:57:02.509863+0000 mgr.y (mgr.14148) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:02.530221+0000 mgr.y (mgr.14148) 129 : cephadm [INF] Deploying daemon osd.5 on smithi152 2023-06-10T20:57:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[107122]: cluster 2023-06-10T20:57:02.509863+0000 mgr.y ( 2023-06-10T20:57:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[107122]: mgr.14148) 128 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:02.530221+0000 2023-06-10T20:57:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[107122]: mgr.y (mgr.14148) 129 : cephadm [INF] 2023-06-10T20:57:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:03 smithi064 conmon[107122]: Deploying daemon osd.5 on smithi152 2023-06-10T20:57:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:05 smithi152 conmon[97365]: cluster 2023-06-10T20:57:04.510372+0000 mgr.y (mgr.14148) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:05 smithi064 conmon[97224]: cluster 2023-06-10T20:57:04.510372+0000 mgr.y (mgr.14148) 130 2023-06-10T20:57:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:05 smithi064 conmon[97224]: : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:05 smithi064 conmon[107122]: cluster 2023-06-10T20:57:04.510372+0000 mgr.y (mgr. 2023-06-10T20:57:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:05 smithi064 conmon[107122]: 14148) 130 : cluster [DBG] pgmap v111: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:08 smithi152 conmon[97365]: cluster 2023-06-10T20:57:06.510719+0000 mgr.y 2023-06-10T20:57:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:08 smithi152 conmon[97365]: (mgr.14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:08 smithi152 conmon[97365]: audit 2023-06-10T20:57:07.123140+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:08 smithi152 conmon[97365]: audit 2023-06-10T20:57:07.144955+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[97224]: cluster 2023-06-10T20:57:06.510719+0000 mgr.y (mgr.14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[97224]: audit 2023-06-10T20:57:07.123140+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[97224]: audit 2023-06-10T20:57: 2023-06-10T20:57:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[97224]: 07.144955+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: cluster 2023-06-10T20:57:06.510719+0000 mgr.y (mgr 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: .14148) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: audit 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: 2023-06-10T20:57:07.123140+0000 mon.a (mon.0) 422 : audit [INF] 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:08.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: 06-10T20:57:07.144955+0000 mon.a (mon. 2023-06-10T20:57:08.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:08 smithi064 conmon[107122]: 0) 423 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:09.767 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 5 on host 'smithi152' 2023-06-10T20:57:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: cluster 2023-06-10T20:57:08. 2023-06-10T20:57:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: 511178+0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:10.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06-10T20:57:08.812125+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06-10T20:57: 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: 08.813090+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06-10T20:57:08.814020+0000 mon.a (mon.0 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: ) 426 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06-10T20:57: 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: 08.818904+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06-10T20:57 2023-06-10T20:57:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: :09.763552+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[97224]: -10T20:57:09.787983+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: cluster 2023-06-10T20:57:08.511178+0000 mgr.y ( 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: mgr.14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023-06-10T20:57:08.812125+0000 mon.a (mon.0) 424 : 2023-06-10T20:57:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: 2023-06-10T20:57:08. 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: 813090+0000 mon.a (mon.0) 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: 425 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: 06-10T20:57:08.814020+0000 mon.a (mon.0 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: ) 426 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023-06-10T20:57:08.818904+0000 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: 06-10T20:57:09.763552+0000 mon.a (mon.0) 428 : audit [INF] 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: :57:09.787983+0000 mon.a (mon.0) 429 : audit 2023-06-10T20:57:10.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:09 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: cluster 2023-06-10T20:57:08.511178+0000 mgr.y (mgr.14148) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:08.812125+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:08.813090+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:08.814020+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:08.818904+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:09.763552+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:09 smithi152 conmon[97365]: audit 2023-06-10T20:57:09.787983+0000 mon.a (mon.0) 429 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:10.303 DEBUG:teuthology.orchestra.run.smithi152:osd.5> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5.service 2023-06-10T20:57:10.400 INFO:tasks.cephadm:Deploying osd.6 on smithi152 with /dev/vg_nvme/lv_2... 2023-06-10T20:57:10.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-06-10T20:57:10.432 INFO:journalctl@ceph.osd.5.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:57:10.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:10 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.343463+0000 2023-06-10T20:57:10.915 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:10 smithi152 conmon[97365]: mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:10.915 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:10 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.343784+0000 mon.a (mon.0) 430 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[97224]: audit 2023-06-10T20:57:10.343463+0000 mon.b 2023-06-10T20:57:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[97224]: (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:11.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[97224]: audit 2023-06-10T20:57:10.343784+0000 mon.a (mon.0) 430 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[107122]: audit 2023-06-10T20:57:10.343463+0000 mon.b (mon.1 2023-06-10T20:57:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[107122]: ) 13 : audit [INF] from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:11.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[107122]: audit 2023-06-10T20:57:10.343784+0000 mon.a (mon.0) 430 : 2023-06-10T20:57:11.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:10 smithi064 conmon[107122]: audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T20:57:12.112 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[109488]: debug 2023-06-10T20:57:11.829+0000 7f6897611700 -1 osd.5 0 waiting for initial osdmap 2023-06-10T20:57:12.112 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[109488]: debug 2023-06-10T20:57:11.834+0000 7f6890587700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:57:12.113 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: cluster 2023-06-10T20:57:10.511617+0000 mgr.y (mgr.14148) 2023-06-10T20:57:12.113 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:12.113 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.820922+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T20:57:12.113 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: cluster 2023-06-10T20:57:10.821009+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-10T20:57:12.113 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.821159+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:12.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.824156+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:11 smithi152 conmon[97365]: audit 2023-06-10T20:57:10.824700+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: cluster 2023-06-10T20:57:10.511617+0000 mgr.y (mgr.14148 2023-06-10T20:57:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: ) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: audit 2023-06-10T20:57:10.820922+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T20:57:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: cluster 2023-06-10T20:57:10.821009+0000 mon.a (mon.0) 432 : 2023-06-10T20:57:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-10T20:57:12.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: T20:57:10.821159+0000 mon.a (mon.0) 433 : audit [DBG] 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: audit 2023-06-10T20:57 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: :10.824156+0000 mon.b (mon.1) 14 : audit [INF] 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: 06-10T20:57:10.824700 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: +0000 mon.a (mon.0) 434 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[97224]: : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: cluster 2023-06-10T20:57:10.511617+0000 mgr.y ( 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: mgr.14148) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:12.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: audit 2023-06-10T20:57:10.820922+0000 mon.a (mon.0) 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: 10T20:57:10.821009+0000 mon.a (mon.0) 432 : cluster 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: audit 2023-06-10T20:57:10 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: .821159+0000 mon.a (mon.0) 433 : audit 2023-06-10T20:57:12.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: audit 2023-06-10T20:57: 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: 10.824156+0000 mon.b (mon.1) 14 : audit 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: [INF] from='osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: 20:57:10.824700+0000 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: mon.a (mon.0) 434 : audit 2023-06-10T20:57:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:11 smithi064 conmon[107122]: [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:12.868 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-06-10T20:57:12.868 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-06-10T20:57:12.869 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2023-06-10T20:57:12.869 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2023-06-10T20:57:12.869 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111922 s, 937 MB/s 2023-06-10T20:57:12.869 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2023-06-10T20:57:12.908 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: audit 2023-06-10T20:57:11.825148+0000 mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: cluster 2023-06-10T20:57:11.825258+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: audit 2023-06-10T20:57:11.825704+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: audit 2023-06-10T20:57:11.830863+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: cluster 2023-06-10T20:57:12.825003+0000 mon.a (mon.0) 439 : cluster [INF] osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943] boot 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: cluster 2023-06-10T20:57:12.825072+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-06-10T20:57:12.909 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:12 smithi152 conmon[97365]: audit 2023-06-10T20:57:12.825216+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: audit 2023-06-10T20:57:11.825148+0000 mon.a (mon. 2023-06-10T20:57:13.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: 0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: cluster 2023-06-10T20:57:11.825258+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: audit 2023-06-10T20:57:11.825704+0000 mon.a (mon.0) 437 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: -10T20:57:11.830863+0000 mon.a (mon.0) 2023-06-10T20:57:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: 438 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: -10T20:57:12.825003+0000 mon.a (mon.0) 439 : cluster [INF] osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943] boot 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: cluster 2023-06-10T20: 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: 57:12.825072+0000 mon.a (mon.0) 440 : cluster [DBG] 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: osdmap e36: 6 total, 6 up, 6 in 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: audit 2023-06-10T20:57 2023-06-10T20:57:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: :12.825216+0000 mon.a (mon.0 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[107122]: ) 441 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: audit 2023-06-10T20:57:11.825148+0000 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: mon.a (mon.0) 435 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: cluster 2023-06-10T20:57:11.825258+0000 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: mon.a (mon.0) 436 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-10T20:57:13.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: -06-10T20:57:11.825704+0000 mon.a 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: (mon.0) 437 : audit 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: -10T20:57:11.830863+0000 mon.a (mon.0 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: ) 438 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: cluster 2023-06 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: -10T20:57:12.825003+0000 mon.a (mon. 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: 0) 439 : cluster [INF] osd.5 [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943] boot 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: 06-10T20:57:12.825072+0000 mon.a (mon.0) 440 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:13.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: 06-10T20:57:12.825216+0000 mon.a (mon 2023-06-10T20:57:13.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: .0) 441 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:57:13.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:12 smithi064 conmon[97224]: 2023-06-10T20:57:13.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi152:vg_nvme/lv_2 2023-06-10T20:57:14.105 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:13 smithi152 conmon[97365]: cluster 2023-06-10T20:57:11.376532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:14.105 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:13 smithi152 conmon[97365]: cluster 2023-06-10T20:57:11.376594+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:14.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:13 smithi152 conmon[97365]: cluster 2023-06-10T20:57:12.512066+0000 mgr.y (mgr.14148) 134 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:14.106 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:13 smithi152 conmon[97365]: cluster 2023-06-10T20:57:13.823923+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-10T20:57:14.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: cluster 2023-06-10T20:57:11.376532+0000 osd.5 ( 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: cluster 2023-06-10T20:57:11 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: .376594+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: 10T20:57:12.512066+0000 mgr.y ( 2023-06-10T20:57:14.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: mgr.14148) 134 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:14.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: cluster 2023- 2023-06-10T20:57:14.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: 06-10T20:57:13.823923+0000 2023-06-10T20:57:14.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: mon.a (mon.0) 442 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-10T20:57:14.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[107122]: 2023-06-10T20:57:14.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: cluster 2023-06-10T20:57:11.376532+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: cluster 2023-06-10T20:57:11.376594+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: cluster 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: 2023-06-10T 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: 20:57:12.512066+0000 mgr.y ( 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: mgr.14148) 134 : cluster [DBG] 2023-06-10T20:57:14.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: pgmap v117: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-10T20:57:14.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: cluster 2023-06-10T20:57:14.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: 2023-06-10T20:57:13.823923 2023-06-10T20:57:14.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: +0000 mon.a (mon.0) 442 2023-06-10T20:57:14.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:13 smithi064 conmon[97224]: : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-10T20:57:15.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:13.846523+0000 mgr.y (mgr.14148) 2023-06-10T20:57:15.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: 135 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:15.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.853696+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.854934+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.855898+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:13.856607+0000 mgr.y (mgr.14148) 136 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 8584M 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.863067+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.864171+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.865045+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:15.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: audit 2023-06-10T20:57:13.872281+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.118 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:14 smithi152 conmon[97365]: cluster 2023-06-10T20:57:14.830734+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-10T20:57:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:13.846523 2023-06-10T20:57:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: +0000 mgr.y (mgr.14148) 135 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 2023-06-10T20:57:13.853696+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 20:57:13.854934+0000 mon.a (mon.0) 444 : audit 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:15.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: -10T20:57:13.855898+0000 mon.a (mon.0 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: ) 445 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 2023-06-10T20:57:13.856607+0000 mgr.y 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: (mgr.14148) 136 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 8584M 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:15.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: -10T20:57:13.863067+0000 mon.a (mon. 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: 0) 446 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: -10T20:57:13.864171+0000 mon.a (mon.0) 447 : audit [DBG] 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: -10T20:57:13.865045+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: audit 2023-06-10T20:57:13.872281+0000 mon.a (mon.0) 449 : audit [INF] 2023-06-10T20:57:15.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: cluster 2023-06-10T20:57 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[107122]: :14.830734+0000 mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: cephadm 2023-06- 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 10T20:57:13.846523+0000 mgr.y (mgr.14148) 135 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023-06-10T20:57:13.853696 2023-06-10T20:57:15.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: +0000 mon.a (mon.0) 443 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: :57:13.854934+0000 mon.a (mon.0) 444 : 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023-06-10T20:57 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: :13.855898+0000 mon.a (mon.0) 445 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: cephadm 2023-06-10T20: 2023-06-10T20:57:15.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 57:13.856607+0000 mgr.y (mgr.14148) 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 136 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 8584M 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: T20:57:13.863067+0000 mon.a ( 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: mon.0) 446 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 06-10T20:57:13.864171+0000 mon.a 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: (mon.0) 447 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:15.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: -06-10T20:57:13.865045+0000 mon.a ( 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: mon.0) 448 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: -06-10T20:57:13.872281+0000 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: mon.a (mon.0) 449 : audit [INF] 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: cluster 2023-06-10T20:57:15.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 2023-06-10T20:57:14.830734+0000 2023-06-10T20:57:15.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: mon.a (mon.0) 450 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-10T20:57:15.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:14 smithi064 conmon[97224]: 2023-06-10T20:57:16.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: cluster 2023-06-10T20:57:14.512456+0000 mgr.y (mgr 2023-06-10T20:57:16.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: .14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: audit 2023-06-10T20:57:15.492884+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: audit 2023-06-10T20:57:15.495763+0000 mon.a (mon.0) 452 2023-06-10T20:57:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:57:16.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: 20:57:15.496679+0000 mon.a (mon. 2023-06-10T20:57:16.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[97224]: 0) 453 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:16.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: cluster 2023-06-10T20:57:14.512456+0000 2023-06-10T20:57:16.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: 57:15.492884+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: -06-10T20:57:15.495763+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: 2023-06-10T20:57:16.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: audit 2023-06-10T20:57:16.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: 2023-06-10T20:57:15.496679+0000 mon.a (mon.0) 453 : audit 2023-06-10T20:57:16.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:15 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:16.162 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:15 smithi152 conmon[97365]: cluster 2023-06-10T20:57:14.512456+0000 mgr.y (mgr.14148) 137 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:16.162 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:15 smithi152 conmon[97365]: audit 2023-06-10T20:57:15.492884+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:16.163 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:15 smithi152 conmon[97365]: audit 2023-06-10T20:57:15.495763+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:16.163 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:15 smithi152 conmon[97365]: audit 2023-06-10T20:57:15.496679+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:16.997 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:16 smithi152 conmon[97365]: audit 2023-06-10T20:57:15.489869+0000 mgr.y (mgr.14148) 2023-06-10T20:57:16.997 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:16 smithi152 conmon[97365]: 138 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:16 smithi064 conmon[97224]: audit 2023-06-10T20:57:15.489869+0000 mgr.y 2023-06-10T20:57:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:16 smithi064 conmon[97224]: (mgr.14148) 138 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:16 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:57:17.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:16 smithi064 conmon[107122]: :57:15.489869+0000 mgr.y (mgr.14148) 138 : audit [DBG] from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: cluster 2023-06-10T20:57:16.512928+0000 mgr.y (mgr.14148) 139 2023-06-10T20:57:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: audit 2023-06-10T20:57:17.674584+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.152:0/3798480648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: audit 2023-06-10T20:57:17.674987 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: +0000 mon.a (mon.0) 454 : audit [INF] 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: audit 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: 2023-06-10T20:57:17.685256+0000 mon.a ( 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]': finished 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T20:57:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: -10T20:57:17.685412+0000 mon.a 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: (mon.0) 456 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: 06-10T20:57:17.685584+0000 mon.a 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[107122]: (mon.0) 457 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: cluster 2023-06-10T20:57:16.512928+0000 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: mgr.y (mgr.14148) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: audit 2023-06-10T20:57:17.674584+0000 mon.b (mon.1 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: ) 15 : audit [INF] from='client.? 172.21.15.152:0/3798480648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: :57:17.674987+0000 mon.a (mon 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: .0) 454 : audit [INF] 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:18.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: -10T20:57:17.685256+0000 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: mon.a (mon.0) 455 : audit 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]': finished 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: 06-10T20:57:17.685412 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: +0000 mon.a (mon.0) 456 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:18.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: -06-10T20:57:17.685584 2023-06-10T20:57:18.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: +0000 mon.a (mon.0) 2023-06-10T20:57:18.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:17 smithi064 conmon[97224]: 457 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: cluster 2023-06-10T20:57:16.512928+0000 mgr.y (mgr.14148) 139 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: audit 2023-06-10T20:57:17.674584+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.152:0/3798480648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: audit 2023-06-10T20:57:17.674987+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]: dispatch 2023-06-10T20:57:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: audit 2023-06-10T20:57:17.685256+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "834d3ada-5c5e-40f0-8520-ad53ff0648ce"}]': finished 2023-06-10T20:57:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: cluster 2023-06-10T20:57:17.685412+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-10T20:57:18.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:17 smithi152 conmon[97365]: audit 2023-06-10T20:57:17.685584+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:18 smithi064 conmon[107122]: audit 2023-06-10T20:57:18.263482+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.152:0/1634950488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:18 smithi064 conmon[97224]: audit 2023-06-10T20:57:18.263482+0000 mon.b 2023-06-10T20:57:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:18 smithi064 conmon[97224]: (mon.1) 16 : audit [DBG] from='client.? 172.21.15.152:0/1634950488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:18 smithi152 conmon[97365]: audit 2023-06-10T20:57:18.263482+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.152:0/1634950488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:19 smithi064 conmon[97224]: cluster 2023-06-10T20:57:18.513447+0000 mgr.y (mgr.14148) 140 : cluster 2023-06-10T20:57:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:19 smithi064 conmon[97224]: [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:19 smithi064 conmon[107122]: cluster 2023-06-10T20:57:18.513447+0000 mgr.y 2023-06-10T20:57:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:19 smithi064 conmon[107122]: (mgr.14148) 140 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:20.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:19 smithi152 conmon[97365]: cluster 2023-06-10T20:57:18.513447+0000 mgr.y (mgr.14148) 140 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:22.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:22 smithi152 conmon[97365]: cluster 2023-06-10T20:57:20.513881+0000 mgr.y (mgr.14148) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:22 smithi064 conmon[97224]: cluster 2023-06-10T20:57:20.513881+0000 mgr.y (mgr.14148) 141 : cluster 2023-06-10T20:57:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:22 smithi064 conmon[97224]: [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:22.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:22 smithi064 conmon[107122]: cluster 2023-06-10T20:57:20.513881+0000 mgr.y ( 2023-06-10T20:57:22.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:22 smithi064 conmon[107122]: mgr.14148) 141 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:24.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:24 smithi152 conmon[97365]: cluster 2023-06-10T20:57:22.514254+0000 mgr.y (mgr.14148) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:24.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:24 smithi152 conmon[97365]: audit 2023-06-10T20:57:23.451370+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T20:57:24.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:24 smithi152 conmon[97365]: audit 2023-06-10T20:57:23.452345+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:24.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[97224]: cluster 2023-06-10T20:57:22.514254+0000 mgr.y (mgr.14148) 142 2023-06-10T20:57:24.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[97224]: : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:24.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[97224]: audit 2023-06-10T20:57:23.451370+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T20:57:24.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:24.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[97224]: 06-10T20:57:23.452345+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:24.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: cluster 2023-06-10T20:57:22.514254+0000 mgr.y (mgr 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: .14148) 142 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: audit 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: 2023-06-10T20:57:23.451370 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: +0000 mon.a (mon.0) 458 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:57:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: 10T20:57:23.452345+0000 mon.a 2023-06-10T20:57:24.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:24 smithi064 conmon[107122]: (mon.0) 459 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:25.371 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:25 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:23.453159+0000 mgr.y (mgr.14148) 143 : cephadm [INF] Deploying daemon osd.6 on smithi152 2023-06-10T20:57:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:25 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:23.453159+0000 mgr.y (mgr.14148) 143 : cephadm 2023-06-10T20:57:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:25 smithi064 conmon[97224]: [INF] Deploying daemon osd.6 on smithi152 2023-06-10T20:57:25.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:25 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:23.453159+0000 2023-06-10T20:57:25.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:25 smithi064 conmon[107122]: mgr.y (mgr.14148) 143 : cephadm [INF] Deploying daemon osd.6 on smithi152 2023-06-10T20:57:26.367 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:26 smithi152 conmon[97365]: cluster 2023-06-10T20:57:24.514633+0000 mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:26 smithi064 conmon[107122]: cluster 2023-06-10T20:57:24.514633+0000 2023-06-10T20:57:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:26 smithi064 conmon[107122]: mgr.y (mgr.14148) 144 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:26.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:26 smithi064 conmon[97224]: cluster 2023-06-10T20:57:24.514633+0000 mgr.y (mgr.14148) 144 : cluster 2023-06-10T20:57:26.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:26 smithi064 conmon[97224]: [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:28.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:28 smithi064 conmon[97224]: cluster 2023-06-10T20:57:26.515122+0000 mgr.y (mgr.14148 2023-06-10T20:57:28.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:28 smithi064 conmon[97224]: ) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:28 smithi064 conmon[107122]: cluster 2023-06-10T20:57:26. 2023-06-10T20:57:28.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:28 smithi064 conmon[107122]: 515122+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:28 smithi152 conmon[97365]: cluster 2023-06-10T20:57:26.515122+0000 mgr.y (mgr.14148) 145 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:29.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:29 smithi152 conmon[97365]: audit 2023-06-10T20:57:28.021319+0000 mon.a ( 2023-06-10T20:57:29.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:29 smithi152 conmon[97365]: mon.0) 460 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:29.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:29 smithi152 conmon[97365]: audit 2023-06-10T20:57:28.026738+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[97224]: audit 2023-06-10T20:57:28.021319+0000 mon.a (mon.0) 460 : audit 2023-06-10T20:57:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[97224]: audit 2023-06-10T20:57:28.026738+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[107122]: audit 2023-06-10T20:57:28.021319+0000 mon.a (mon 2023-06-10T20:57:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[107122]: .0) 460 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:57:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[107122]: 57:28.026738+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:29 smithi064 conmon[107122]: 2023-06-10T20:57:30.403 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:30 smithi152 conmon[97365]: cluster 2023-06-10T20:57:28.515716+0000 mgr.y (mgr.14148) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:30.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:30 smithi152 conmon[97365]: audit 2023-06-10T20:57:29.824686+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:30 smithi152 conmon[97365]: audit 2023-06-10T20:57:29.825607+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:30.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:30 smithi152 conmon[97365]: audit 2023-06-10T20:57:29.826537+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:30.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:30 smithi152 conmon[97365]: audit 2023-06-10T20:57:29.832149+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: cluster 2023-06-10T20:57:28.515716+0000 mgr.y (mgr.14148) 146 : cluster [DBG] 2023-06-10T20:57:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: pgmap v129: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: audit 2023-06-10T20:57:29.824686+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: audit 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: 2023-06-10T20:57:29.825607+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: 06-10T20:57:29.826537+0000 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: mon.a (mon.0) 464 : audit 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:30.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: -10T20:57:29.832149+0000 mon.a (mon.0) 465 : audit 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: cluster 2023-06-10T20:57:28.515716+0000 mgr.y 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: (mgr.14148) 146 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: audit 2023-06-10T20:57:29. 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: 824686+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: -10T20:57:29.825607+0000 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: mon.a (mon.0) 463 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: 10T20:57:29.826537+0000 mon.a (mon. 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: 0) 464 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: 2023-06-10T20:57:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: audit 2023-06-10T20:57:30.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: 2023-06-10T20:57 2023-06-10T20:57:30.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: :29.832149+0000 mon.a (mon 2023-06-10T20:57:30.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:30 smithi064 conmon[97224]: .0) 465 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:30.823 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 6 on host 'smithi152' 2023-06-10T20:57:31.383 DEBUG:teuthology.orchestra.run.smithi152:osd.6> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.6.service 2023-06-10T20:57:31.386 INFO:tasks.cephadm:Deploying osd.7 on smithi152 with /dev/vg_nvme/lv_1... 2023-06-10T20:57:31.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-06-10T20:57:31.417 INFO:journalctl@ceph.osd.6.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:57:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:31 smithi152 conmon[97365]: cluster 2023-06-10T20:57:30.516217+0000 mgr.y (mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:31 smithi152 conmon[97365]: audit 2023-06-10T20:57:30.819079+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:31 smithi152 conmon[97365]: audit 2023-06-10T20:57:30.851184+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:31.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:31 smithi152 conmon[97365]: audit 2023-06-10T20:57:31.236162+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:31.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:31 smithi152 conmon[97365]: audit 2023-06-10T20:57:31.236477+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:32.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: cluster 2023-06-10T20:57:30.516217+0000 2023-06-10T20:57:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: mgr.y (mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: audit 2023-06-10T20:57:30.819079+0000 2023-06-10T20:57:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: audit 2023-06-10T20:57:30. 2023-06-10T20:57:32.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: 851184+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:32.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: audit 2023-06-10T20:57:32.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: 2023-06-10T20:57:31.236162+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:32.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: audit 2023-06-10T20:57: 2023-06-10T20:57:32.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[97224]: 31.236477+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:32.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: cluster 2023-06-10T20:57:30.516217+0000 mgr.y ( 2023-06-10T20:57:32.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: mgr.14148) 147 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:32.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: audit 2023-06-10T20:57 2023-06-10T20:57:32.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: :30.819079+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:32.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:32.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: 06-10T20:57:30.851184+0000 2023-06-10T20:57:32.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: mon.a (mon.0) 467 : audit [DBG] 2023-06-10T20:57:32.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:32.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: audit 2023-06-10T20:57:32.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: 2023-06-10T20:57:31.236162+0000 mon.b 2023-06-10T20:57:32.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:32.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: 2023-06-10T20:57:32.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:57:32.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:31 smithi064 conmon[107122]: 10T20:57:31.236477+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-10T20:57:33.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: audit 2023-06-10T20:57:32.032732+0000 mon.a (mon.0) 2023-06-10T20:57:33.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-10T20:57:33.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: cluster 2023-06-10T20:57:32.032818+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-10T20:57:33.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: audit 2023-06-10T20:57:32.032985+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: audit 2023-06-10T20:57:32.033378+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: audit 2023-06-10T20:57:32.033700+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[97365]: cluster 2023-06-10T20:57:32.516682+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:33.194 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[115220]: debug 2023-06-10T20:57:33.040+0000 7f213bc75700 -1 osd.6 0 waiting for initial osdmap 2023-06-10T20:57:33.194 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 20:57:33 smithi152 conmon[115220]: debug 2023-06-10T20:57:33.046+0000 7f21383f2700 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:57:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: audit 2023-06-10T20:57:32.032732+0000 mon.a (mon 2023-06-10T20:57:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: .0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-10T20:57:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: cluster 2023-06-10T20:57:32.032818+0000 mon.a (mon.0) 470 : cluster 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: 20:57:32.032985+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: audit 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: 2023-06-10T20:57:32.033378+0000 mon.b 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: audit 2023-06-10T20:57:32 2023-06-10T20:57:33.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: .033700+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[97224]: 10T20:57:32.516682+0000 mgr.y (mgr.14148) 148 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: audit 2023-06-10T20:57:32.032732 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: +0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: cluster 2023-06-10T20:57 2023-06-10T20:57:33.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: :32.032818+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: audit 2023-06-10T20:57:32 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: .032985+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: -10T20:57:32.033378+0000 mon.b (mon.1) 18 : 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: audit [INF] from='osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: audit 2023-06-10T20:57:33.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: 2023-06-10T20:57:32.033700+0000 mon.a 2023-06-10T20:57:33.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:33.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T20:57:33.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: 20:57:32.516682+0000 mgr.y (mgr.14148) 148 : cluster [DBG] 2023-06-10T20:57:33.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:33 smithi064 conmon[107122]: pgmap v132: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-10T20:57:33.715 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-06-10T20:57:33.715 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-06-10T20:57:33.715 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2023-06-10T20:57:33.715 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2023-06-10T20:57:33.715 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130419 s, 804 MB/s 2023-06-10T20:57:33.716 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2023-06-10T20:57:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[97224]: audit 2023-06-10T20:57:33.037595+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[97224]: cluster 2023-06-10T20:57:33.037767+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-10T20:57:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[97224]: audit 2023-06-10T20:57:33.038247+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[97224]: audit 2023-06-10T20:57:33.041113+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: audit 2023-06-10T20:57:33.037595+0000 mon.a (mon.0 2023-06-10T20:57:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: ) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: cluster 2023-06-10T20:57:33.037767+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-10T20:57:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: audit 2023-06-10T20:57:33.038247+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: 2023-06-10T20:57:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: audit 2023-06-10T20:57:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: 2023-06-10T20:57:33.041113 2023-06-10T20:57:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: +0000 mon.a (mon.0) 476 : audit 2023-06-10T20:57:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:34 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:34 smithi152 conmon[97365]: audit 2023-06-10T20:57:33.037595+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:34 smithi152 conmon[97365]: cluster 2023-06-10T20:57:33.037767+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-10T20:57:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:34 smithi152 conmon[97365]: audit 2023-06-10T20:57:33.038247+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:34 smithi152 conmon[97365]: audit 2023-06-10T20:57:33.041113+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:34.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch daemon add osd smithi152:vg_nvme/lv_1 2023-06-10T20:57:35.061 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:32.266827+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:35.061 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:32.266878+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:35.061 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:34.041222+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000] boot 2023-06-10T20:57:35.061 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:34.041281+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-10T20:57:35.061 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: audit 2023-06-10T20:57:34.041746+0000 mon.a (mon.0) 479 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:35.062 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:34.517196+0000 mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:35.062 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:35 smithi152 conmon[97365]: cluster 2023-06-10T20:57:35.042224+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-10T20:57:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06-10T20:57:32.266827+0000 osd.6 (osd.6) 1 2023-06-10T20:57:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06-10T20:57:32.266878+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06-10T 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: 20:57:34.041222+0000 mon.a (mon.0) 477 : cluster [INF] osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000] boot 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06-10T20:57:34.041281+0000 mon.a 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: (mon.0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-10T20:57:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: audit 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: 2023-06-10T20:57:34.041746+0000 mon.a ( 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: mon.0) 479 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: 2023-06-10T20 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: :57:34.517196+0000 mgr.y (mgr.14148) 149 2023-06-10T20:57:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: cluster 2023-06 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: -10T20:57:35.042224+0000 mon.a (mon.0 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[97224]: ) 480 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06-10T20:57:32.266827+0000 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 2023-06-10T20:57:32.266878+0000 osd.6 (osd.6) 2 : cluster [DBG] 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: purged_snaps scrub ok 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 10T20:57:34.041222+0000 mon.a (mon.0 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: ) 477 : cluster [INF] osd.6 [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000] boot 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:57:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 10T20:57:34.041281+0000 mon.a (mon. 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 0) 478 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: T20:57:34.041746+0000 mon.a (mon.0 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: ) 479 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 2023-06-10T20:57:34.517196+0000 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: mgr.y (mgr.14148) 149 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T20:57:35.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: 10T20:57:35.042224+0000 mon.a (mon.0) 480 : cluster 2023-06-10T20:57:35.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:35 smithi064 conmon[107122]: [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-10T20:57:36.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:35.207083+0000 mgr.y (mgr. 2023-06-10T20:57:36.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: 14148) 150 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:36.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.211945+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.213125+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.214112+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.215051+0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:35.215797+0000 mgr.y (mgr.14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 5722M 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.219923+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.221043+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.222021+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:36.253 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: audit 2023-06-10T20:57:35.227061+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.254 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:36 smithi152 conmon[97365]: cluster 2023-06-10T20:57:36.041220+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-10T20:57:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:35.207083+0000 mgr.y (mgr. 2023-06-10T20:57:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 14148) 150 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10T20:57:35.211945 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: +0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 57:35.213125+0000 mon.a (mon.0) 482 : 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 2023-06-10T20:57: 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 35.214112+0000 mon.a (mon.0) 483 : audit 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: T20:57:35.215051+0000 mon.a (mon.0) 484 2023-06-10T20:57:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:36.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 2023-06-10T20:57:35.215797+0000 mgr.y 2023-06-10T20:57:36.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: (mgr.14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 5722M 2023-06-10T20:57:36.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 10T20:57:35.219923+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 2023-06-10T20:57:35.221043+0000 mon.a ( 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: mon.0) 486 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:57:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: T20:57:35.222021+0000 mon.a (mon.0) 487 : audit [INF] 2023-06-10T20:57:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:57:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: :57:35.227061+0000 mon.a (mon.0) 488 2023-06-10T20:57:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:57:36.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 06-10T20:57:36.041220+0000 mon.a (mon. 2023-06-10T20:57:36.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[97224]: 0) 489 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-10T20:57:36.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:35.207083+0000 mgr.y (mgr. 2023-06-10T20:57:36.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 14148) 150 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:36.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20:57:35.211945+0000 mon.a 2023-06-10T20:57:36.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: (mon.0) 481 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 2023-06-10T20:57:35.213125+0000 mon.a (mon.0) 482 : audit [INF] 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20:57:35 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: .214112+0000 mon.a (mon.0) 483 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20:57:35.215051 2023-06-10T20:57:36.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: +0000 mon.a (mon.0) 484 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 2023-06-10T20:57:35.215797+0000 mgr.y (mgr. 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 14148) 151 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 5722M 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 06-10T20:57:35.219923+0000 mon.a (mon. 2023-06-10T20:57:36.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 0) 485 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: T20:57:35.221043+0000 mon.a (mon.0) 486 : audit 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: -06-10T20:57:35.222021 2023-06-10T20:57:36.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: +0000 mon.a (mon.0) 487 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: :57:35.227061+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: cluster 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: 2023-06-10T20:57:36.041220+0000 2023-06-10T20:57:36.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:36 smithi064 conmon[107122]: mon.a (mon.0) 489 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-10T20:57:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:37 smithi152 conmon[97365]: cluster 2023-06-10T20:57:36.517634+0000 mgr.y (mgr.14148) 152 : cluster [DBG] pgmap v138: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:37 smithi152 conmon[97365]: audit 2023-06-10T20:57:36.613961+0000 mgr.y (mgr.14148) 153 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:37 smithi152 conmon[97365]: audit 2023-06-10T20:57:36.616998+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:37 smithi152 conmon[97365]: audit 2023-06-10T20:57:36.620123+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:37 smithi152 conmon[97365]: audit 2023-06-10T20:57:36.621088+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:37.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: cluster 2023-06-10T20:57:36.517634+0000 mgr.y 2023-06-10T20:57:37.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: (mgr.14148) 152 : cluster [DBG] pgmap v138: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:37.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.613961+0000 mgr.y (mgr.14148) 153 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.616998+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.620123+0000 mon.a (mon. 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: 0) 491 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: audit 2023-06-10T20:57:36.621088+0000 mon.a (mon.0) 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[97224]: 492 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:37.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: cluster 2023-06-10T20:57:36 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: .517634+0000 mgr.y (mgr.14148) 152 : cluster [DBG] pgmap v138: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: 57:36.613961+0000 mgr.y (mgr.14148) 153 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: 20:57:36.616998+0000 mon.a (mon.0) 490 2023-06-10T20:57:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: -06-10T20:57:36.620123+0000 mon.a (mon 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: .0) 491 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: -10T20:57:36.621088+0000 mon.a (mon.0) 2023-06-10T20:57:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:37 smithi064 conmon[107122]: 492 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:38 smithi152 conmon[97365]: cluster 2023-06-10T20:57:37.211737+0000 mon.a ( 2023-06-10T20:57:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:38 smithi152 conmon[97365]: mon.0) 493 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:38 smithi064 conmon[97224]: cluster 2023-06-10T20:57:37.211737+0000 mon.a (mon.0) 493 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:38 smithi064 conmon[107122]: cluster 2023-06-10T20:57:37.211737+0000 mon.a ( 2023-06-10T20:57:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:38 smithi064 conmon[107122]: mon.0) 493 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: cluster 2023-06-10T20:57:38.518026+0000 mgr.y (mgr.14148) 154 : 2023-06-10T20:57:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: cluster [DBG] pgmap v139: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: audit 2023-06-10T20:57:38.830536+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.152:0/661073653' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: audit 2023-06-10T20:57:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: 2023-06-10T20:57:38.831159+0000 mon.a (mon.0) 494 2023-06-10T20:57:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: -10T20:57:38.837055+0000 mon.a (mon.0) 495 : audit 2023-06-10T20:57:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]': finished 2023-06-10T20:57:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T20:57:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: 10T20:57:38.837138+0000 mon.a (mon.0) 496 : cluster 2023-06-10T20:57:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-10T20:57:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:39.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: 06-10T20:57:38.837309+0000 mon.a (mon 2023-06-10T20:57:39.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: .0) 497 : audit [DBG] 2023-06-10T20:57:39.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:39.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: cluster 2023-06-10T20:57:38 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: .518026+0000 mgr.y (mgr.14148) 154 : cluster [DBG] pgmap v139: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: audit 2023-06-10T20:57:38.830536+0000 mon.b 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: (mon.1) 19 : audit [INF] from='client.? 172.21.15.152:0/661073653' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 06-10T20:57:38.831159 2023-06-10T20:57:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: +0000 mon.a (mon.0) 494 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: -06-10T20:57:38 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: .837055+0000 mon.a (mon. 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]': finished 2023-06-10T20:57:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: cluster 2023-06-10T20:57:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 2023-06 2023-06-10T20:57:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: -10T20:57:38. 2023-06-10T20:57:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 837138+0000 mon.a (mon.0) 2023-06-10T20:57:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 496 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-10T20:57:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: audit 2023-06-10T20:57:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 2023- 2023-06-10T20:57:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 06-10T20:57:38 2023-06-10T20:57:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: .837309+0000 mon.a (mon. 2023-06-10T20:57:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 0) 497 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:39 smithi064 conmon[107122]: 2023-06-10T20:57:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: cluster 2023-06-10T20:57:38.518026+0000 mgr.y (mgr.14148 2023-06-10T20:57:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: ) 154 : cluster [DBG] pgmap v139: 1 pgs: 1 remapped+peering; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: audit 2023-06-10T20:57:38.830536+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.152:0/661073653' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: audit 2023-06-10T20:57:38.831159+0000 mon.a (mon.0) 494 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]: dispatch 2023-06-10T20:57:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: audit 2023-06-10T20:57:38.837055+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91"}]': finished 2023-06-10T20:57:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: cluster 2023-06-10T20:57:38.837138+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-10T20:57:39.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:39 smithi152 conmon[97365]: audit 2023-06-10T20:57:38.837309+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:40.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:40 smithi064 conmon[97224]: audit 2023-06-10T20:57:39.481668+0000 mon.b ( 2023-06-10T20:57:40.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:40 smithi064 conmon[97224]: mon.1) 20 : audit [DBG] from='client.? 172.21.15.152:0/795304824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:40 smithi064 conmon[107122]: audit 2023-06-10T20:57:39. 2023-06-10T20:57:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:40 smithi064 conmon[107122]: 481668+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.152:0/795304824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:40 smithi152 conmon[97365]: audit 2023-06-10T20:57:39.481668+0000 mon.b (mon.1) 20 : 2023-06-10T20:57:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:40 smithi152 conmon[97365]: audit [DBG] from='client.? 172.21.15.152:0/795304824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-10T20:57:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:41 smithi152 conmon[97365]: cluster 2023-06-10T20:57:40.518596+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:41 smithi152 conmon[97365]: cluster 2023-06-10T20:57:41.226856+0000 mon.a (mon.0) 498 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-06-10T20:57:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:41 smithi152 conmon[97365]: cluster 2023-06-10T20:57:41.226892+0000 mon.a (mon.0) 499 : cluster [INF] Cluster is now healthy 2023-06-10T20:57:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: cluster 2023-06-10T20:57:40.518596 2023-06-10T20:57:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: +0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: cluster 2023-06-10T20:57 2023-06-10T20:57:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: :41.226856+0000 mon.a (mon.0) 498 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-06-10T20:57:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: :57:41.226892+0000 mon.a (mon.0) 499 : cluster [INF] Cluster is now healthy 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[97224]: 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: :57:40.518596+0000 mgr.y (mgr.14148) 155 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: cluster 2023-06-10T20:57:41.226856+0000 mon.a (mon 2023-06-10T20:57:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: .0) 498 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-06-10T20:57:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: cluster 2023-06-10T20:57:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: 2023-06-10T20:57:41. 2023-06-10T20:57:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: 226892+0000 mon.a (mon.0) 499 2023-06-10T20:57:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:41 smithi064 conmon[107122]: : cluster [INF] Cluster is now healthy 2023-06-10T20:57:44.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:43 smithi152 conmon[97365]: cluster 2023-06-10T20:57:42.519055+0000 mgr.y (mgr.14148) 156 : cluster [DBG] 2023-06-10T20:57:44.321 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:43 smithi152 conmon[97365]: pgmap v142: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:43 smithi064 conmon[97224]: cluster 2023-06-10T20:57:42.519055+0000 mgr.y (mgr.14148) 2023-06-10T20:57:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:43 smithi064 conmon[97224]: 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:43 smithi064 conmon[107122]: cluster 2023-06-10T20:57:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:43 smithi064 conmon[107122]: 2023-06-10T20:57:42.519055+0000 mgr.y (mgr.14148) 156 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:44 smithi152 conmon[97365]: audit 2023-06-10T20:57:44.881346+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T20:57:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:44 smithi152 conmon[97365]: audit 2023-06-10T20:57:44.882259+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[97224]: audit 2023-06-10T20:57:44.881346+0000 mon.a ( 2023-06-10T20:57:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[97224]: mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T20:57:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[97224]: audit 2023-06-10T20:57:44.882259+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[107122]: audit 2023-06-10T20:57:44 2023-06-10T20:57:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[107122]: .881346+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T20:57:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[107122]: audit 2023-06-10T20:57: 2023-06-10T20:57:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[107122]: 44.882259+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:44 smithi064 conmon[107122]: 2023-06-10T20:57:46.097 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:45 smithi152 conmon[97365]: cluster 2023-06-10T20:57:44.519426+0000 mgr.y (mgr.14148) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:46.098 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:45 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:44.883016+0000 mgr.y (mgr.14148) 158 : cephadm [INF] Deploying daemon osd.7 on smithi152 2023-06-10T20:57:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[97224]: cluster 2023-06-10T20:57:44.519426+0000 mgr.y 2023-06-10T20:57:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[97224]: (mgr.14148) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:44.883016+0000 mgr.y (mgr.14148) 158 : 2023-06-10T20:57:46.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[97224]: cephadm [INF] Deploying daemon osd.7 on smithi152 2023-06-10T20:57:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[107122]: cluster 2023-06-10T20:57:44.519426+0000 mgr.y (mgr. 2023-06-10T20:57:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[107122]: 14148) 157 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:44.883016+0000 mgr.y (mgr.14148) 158 : cephadm [INF] Deploying daemon osd.7 on smithi152 2023-06-10T20:57:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:45 smithi064 conmon[107122]: 2023-06-10T20:57:48.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:47 smithi152 conmon[97365]: cluster 2023-06-10T20:57:46.519886+0000 mgr.y (mgr.14148) 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:48.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:47 smithi064 conmon[97224]: cluster 2023-06-10T20:57:46.519886+0000 mgr.y ( 2023-06-10T20:57:48.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:47 smithi064 conmon[97224]: mgr.14148) 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:48.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:47 smithi064 conmon[107122]: cluster 2023-06-10T20:57:46.519886+0000 mgr.y (mgr 2023-06-10T20:57:48.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:47 smithi064 conmon[107122]: .14148) 159 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:50.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:49 smithi152 conmon[97365]: cluster 2023-06-10T20:57:48.520459+0000 mgr.y (mgr.14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:50.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:49 smithi152 conmon[97365]: audit 2023-06-10T20:57:49.498955+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:50.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:49 smithi152 conmon[97365]: audit 2023-06-10T20:57:49.531000+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: cluster 2023-06-10T20:57:48.520459+0000 2023-06-10T20:57:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: mgr.y (mgr.14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: audit 2023-06-10T20:57:49.498955+0000 mon.a (mon.0) 502 : audit [INF] 2023-06-10T20:57:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: audit 2023-06-10T20:57:49.531000+0000 mon.a (mon. 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:49 smithi064 conmon[97224]: 0) 503 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: cluster 2023-06-10T20:57:48.520459+0000 mgr.y ( 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: mgr.14148) 160 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: audit 2023-06-10T20:57:49.498955+0000 mon.a (mon.0) 502 2023-06-10T20:57:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:50.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:57:50.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: :57:49.531000+0000 mon.a (mon.0) 503 : audit [DBG] 2023-06-10T20:57:50.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:50 smithi064 conmon[107122]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:52.182 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 7 on host 'smithi152' 2023-06-10T20:57:52.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:52 smithi152 conmon[97365]: cluster 2023-06-10T20:57:50.520848+0000 mgr.y (mgr.14148) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:52.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:52 smithi152 conmon[97365]: audit 2023-06-10T20:57:51.403773+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.376 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:52 smithi152 conmon[97365]: audit 2023-06-10T20:57:51.734039+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:52.376 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:52 smithi152 conmon[97365]: audit 2023-06-10T20:57:51.734979+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:52.376 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:52 smithi152 conmon[97365]: audit 2023-06-10T20:57:51.743231+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: cluster 2023-06-10T20:57:50.520848+0000 mgr.y ( 2023-06-10T20:57:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: mgr.14148) 161 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: audit 2023-06-10T20:57:51.403773+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: -10T20:57:51.734039+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: audit 2023-06-10T20:57:51 2023-06-10T20:57:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: .734979+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[97224]: 06-10T20:57:51.743231+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: cluster 2023-06-10T20:57:50.520848+0000 mgr.y (mgr.14148) 161 : cluster [DBG] 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: pgmap v146: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: audit 2023-06-10T20:57:51.403773+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: audit 2023-06-10T20:57:51 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: .734039+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: audit 2023-06-10T20:57:51.734979+0000 mon.a (mon.0 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: ) 506 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: audit 2023-06-10T20:57:51.743231+0000 2023-06-10T20:57:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:52 smithi064 conmon[107122]: mon.a (mon.0) 507 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:52.806 DEBUG:teuthology.orchestra.run.smithi152:osd.7> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.7.service 2023-06-10T20:57:52.808 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-06-10T20:57:52.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd stat -f json 2023-06-10T20:57:52.840 INFO:journalctl@ceph.osd.7.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:57:53.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:53 smithi152 conmon[97365]: audit 2023-06-10T20:57:52.178019+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:53.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:53 smithi152 conmon[97365]: audit 2023-06-10T20:57:52.211134+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:53.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:53 smithi152 conmon[97365]: cluster 2023-06-10T20:57:52.521423+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:53.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:53 smithi152 conmon[97365]: audit 2023-06-10T20:57:52.715543+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:53.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:53 smithi152 conmon[97365]: audit 2023-06-10T20:57:52.715917+0000 mon.a (mon.0) 510 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:53.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: audit 2023-06-10T20:57:52.178019+0000 2023-06-10T20:57:53.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: mon.a (mon.0) 508 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:53.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: audit 2023-06-10T20:57:52.211134+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:53.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: cluster 2023-06-10T20:57:52.521423+0000 mgr.y (mgr.14148) 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:53.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: audit 2023-06-10T20:57:52.715543+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[97224]: audit 2023-06-10T20:57:52.715917+0000 mon.a (mon.0) 510 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: audit 2023-06-10T20:57:52. 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: 178019+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: audit 2023-06-10T20:57 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: :52.211134+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:57:53.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: -06-10T20:57:52.521423+0000 mgr.y (mgr.14148) 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: 162 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: 20:57:52.715543+0000 mon.b (mon.1) 21 : audit 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: audit 2023-06-10T20:57 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: :52.715917+0000 mon.a (mon.0) 510 : audit [INF] 2023-06-10T20:57:53.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:53 smithi064 conmon[107122]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-10T20:57:54.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[97224]: audit 2023-06-10T20:57:53.182474+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-10T20:57:54.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[97224]: cluster 2023-06-10T20:57:53.182552+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-10T20:57:54.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[97224]: audit 2023-06-10T20:57:53.182806+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:54.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[97224]: audit 2023-06-10T20:57:53.183450+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[97224]: audit 2023-06-10T20:57:53.183723+0000 mon.a (mon.0) 514 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: audit 2023-06-10T20:57:53.182474+0000 mon.a (mon 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: .0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: cluster 2023-06-10T20:57:53.182552+0000 mon.a (mon.0) 512 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: T20:57:53.182806+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:54.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: audit 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: 2023-06-10T20:57:53.183450+0000 mon.b (mon. 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: 1) 22 : audit [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: 06-10T20:57:53.183723+0000 mon.a ( 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: mon.0) 514 : audit [INF] 2023-06-10T20:57:54.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:54 smithi064 conmon[107122]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: audit 2023-06-10T20:57:53.182474 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: +0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: cluster 2023-06-10T20:57:53.182552+0000 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: mon.a (mon.0) 512 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: 10T20:57:53.182806+0000 mon.a (mon. 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: 0) 513 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:54.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: audit 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: 2023-06-10T20:57:53.183450+0000 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: :57:53.183723+0000 mon.a (mon.0) 2023-06-10T20:57:54.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[97365]: 514 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T20:57:54.448 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[121150]: debug 2023-06-10T20:57:54.186+0000 7f947ac66700 -1 osd.7 0 waiting for initial osdmap 2023-06-10T20:57:54.448 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 20:57:54 smithi152 conmon[121150]: debug 2023-06-10T20:57:54.194+0000 7f9473bdc700 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T20:57:54.664 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:57:55.358 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":47,"num_osds":8,"num_up_osds":7,"osd_up_since":1686430654,"num_in_osds":8,"osd_in_since":1686430658,"num_remapped_pgs":0} 2023-06-10T20:57:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: audit 2023-06-10T20:57:54.184121+0000 mon.a (mon.0) 2023-06-10T20:57:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: cluster 2023-06-10T20:57:54.184208+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-10T20:57:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: audit 2023-06-10T20:57:54.184932+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: audit 2023-06-10T20:57:54.187341+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: cluster 2023-06-10T20:57:54.521703+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[107122]: audit 2023-06-10T20:57:54.661502+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.64:0/1638731113' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: audit 2023-06-10T20:57:54.184121+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: cluster 2023-06-10T20:57:54.184208+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-10T20:57:55.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: audit 2023-06-10T20:57:54.184932+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: audit 2023-06-10T20:57:54.187341+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: cluster 2023-06-10T20:57:54.521703+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:55 smithi064 conmon[97224]: audit 2023-06-10T20:57:54.661502+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.64:0/1638731113' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:57:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: audit 2023-06-10T20:57:54.184121+0000 mon.a (mon.0) 515 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2023-06-10T20:57:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: cluster 2023-06-10T20:57:54.184208+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-10T20:57:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: audit 2023-06-10T20:57:54.184932+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: audit 2023-06-10T20:57:54.187341+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: cluster 2023-06-10T20:57:54.521703+0000 mgr.y (mgr.14148) 163 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-10T20:57:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:55 smithi152 conmon[97365]: audit 2023-06-10T20:57:54.661502+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.64:0/1638731113' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:57:56.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd stat -f json 2023-06-10T20:57:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cluster 2023-06-10T20:57:53.711391+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cluster 2023-06-10T20:57:53.711437+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cluster 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20:57:55.191002+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648] boot 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cluster 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20:57:55.191081+0000 mon.a 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06-10T20:57:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20:57:55.191698+0000 mon.a (mon.0 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 521 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cephadm 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20:57:55.462405+0000 mgr.y ( 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: mgr.14148) 164 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -10T20:57:55.467699+0000 mon.a (mon.0 2023-06-10T20:57:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 522 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 06-10T20:57:55.468807+0000 mon.a (mon.0 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 523 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -10T20:57:55.469601+0000 mon.a (mon.0 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 524 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -10T20:57:55.470321+0000 mon.a (mon 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: .0) 525 : audit [INF] 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -06-10T20:57:55.471023+0000 mon.a ( 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: mon.0) 526 : audit [INF] 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T20:57:56.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 06-10T20:57:55.471663+0000 mgr.y ( 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: mgr.14148) 165 : cephadm [INF] 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: Adjusting osd_memory_target on smithi152 to 4292M 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -06-10T20:57:55.475896+0000 mon.a 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: (mon.0) 527 : audit [INF] 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023 2023-06-10T20:57:56.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -06-10T20:57:55.476796+0000 mon.a (mon. 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 0) 528 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20: 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 57:55.477542+0000 mon.a (mon.0 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 529 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 2023-06-10T20:57:56.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:57:56.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -10T20:57:55.482377+0000 mon.a (mon.0 2023-06-10T20:57:56.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: ) 530 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: cluster 2023-06 2023-06-10T20:57:56.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: -10T20:57:55.984800+0000 mon.a (mon. 2023-06-10T20:57:56.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[97224]: 0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cluster 2023-06-10T20:57 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: :53.711391+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cluster 2023- 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 06-10T20:57:53 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: .711437+0000 osd.7 (osd. 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:56.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cluster 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20:57: 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 55.191002+0000 mon.a (mon 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: .0) 519 : cluster [INF] osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648] boot 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: :55.191081+0000 mon.a 2023-06-10T20:57:56.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: (mon.0) 520 : cluster [DBG] 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: osdmap e48: 8 total, 8 up, 8 in 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -10T20:57:55.191698 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: +0000 mon.a (mon.0) 521 : 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cephadm 2023 2023-06-10T20:57:56.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57: 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 55.462405+0000 mgr.y (mgr.14148) 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 164 : cephadm [INF] Detected new or changed devices on smithi152 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06- 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 10T20:57:55.467699 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: +0000 mon.a (mon.0) 2023-06-10T20:57:56.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 522 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20:57 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: :55.468807+0000 mon.a (mon.0) 523 : 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:57:56.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: T20:57:55.469601+0000 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: mon.a (mon.0) 524 : audit 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57:55 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: .470321+0000 mon.a (mon.0 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: ) 525 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:56.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57:55.471023+0000 mon.a (mon 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: .0) 526 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cephadm 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20: 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 57:55.471663+0000 mgr.y ( 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: mgr.14148) 165 : cephadm [INF] 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: Adjusting osd_memory_target on smithi152 to 4292M 2023-06-10T20:57:56.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57:55.475896+0000 mon.a (mon 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: .0) 527 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:55.476796+0000 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 2023-06-10T20:57:55.477542+0000 mon.a 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: (mon.0) 529 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:56.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:57:56.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -10T20:57:55.482377+0000 mon.a (mon.0) 2023-06-10T20:57:56.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 530 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:57:56.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: -06-10T20:57: 2023-06-10T20:57:56.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:56 smithi064 conmon[107122]: 55.984800+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-10T20:57:56.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cluster 2023-06-10T20:57:53.711391+0000 osd.7 (osd.7) 1 : cluster 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: [DBG] purged_snaps scrub starts 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cluster 2023-06-10T20:57:53.711437+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cluster 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 2023-06-10T20:57:55.191002+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648] boot 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cluster 2023- 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 06-10T20:57:55.191081+0000 mon.a (mon.0) 2023-06-10T20:57:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 06-10T20:57:55.191698+0000 mon.a (mon. 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 0) 521 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 2023-06-10T20:57:55. 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 462405+0000 mgr.y (mgr.14148) 164 : cephadm [INF] 2023-06-10T20:57:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: Detected new or changed devices on smithi152 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: :57:55.467699+0000 mon.a (mon.0) 522 : audit 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 2023-06-10T20:57:55.468807+0000 mon.a ( 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: mon.0) 523 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: 2023-06-10T20:57:55.469601+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.470321+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.471023+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cephadm 2023-06-10T20:57:55.471663+0000 mgr.y (mgr.14148) 165 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 4292M 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.475896+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.476796+0000 mon.a (mon.0) 528 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.477542+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: audit 2023-06-10T20:57:55.482377+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:57:56.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:56 smithi152 conmon[97365]: cluster 2023-06-10T20:57:55.984800+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-10T20:57:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:57:58.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[107122]: cluster 2023-06-10T20:57:56.522119+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v153: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:57:58.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[107122]: cluster 2023-06-10T20:57:56.986332+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-10T20:57:58.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[107122]: cluster 2023-06-10T20:57:57.200119+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:58.293 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[97224]: cluster 2023-06-10T20:57:56.522119+0000 mgr.y (mgr.14148) 166 : cluster [DBG] pgmap v153: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:57:58.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[97224]: cluster 2023-06-10T20:57:56.986332+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-10T20:57:58.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:57 smithi064 conmon[97224]: cluster 2023-06-10T20:57:57.200119+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:57 smithi152 conmon[97365]: cluster 2023-06-10T20:57:56.522119+0000 mgr.y (mgr.14148) 166 : cluster [DBG] 2023-06-10T20:57:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:57 smithi152 conmon[97365]: pgmap v153: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:57:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:57 smithi152 conmon[97365]: cluster 2023-06-10T20:57:56.986332+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-10T20:57:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:57 smithi152 conmon[97365]: cluster 2023-06-10T20:57:57.200119+0000 2023-06-10T20:57:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:57 smithi152 conmon[97365]: mon.a (mon.0) 533 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T20:57:58.887 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1686430675,"num_in_osds":8,"osd_in_since":1686430658,"num_remapped_pgs":0} 2023-06-10T20:57:58.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T20:57:59.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:58 smithi064 conmon[97224]: audit 2023-06-10T20:57:58.267861+0000 mon.a (mon 2023-06-10T20:57:59.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:58 smithi064 conmon[97224]: .0) 534 : audit [DBG] from='client.? 172.21.15.64:0/53249254' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:57:59.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:58 smithi064 conmon[107122]: audit 2023-06-10T20:57 2023-06-10T20:57:59.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:58 smithi064 conmon[107122]: :58.267861+0000 mon.a (mon.0) 534 : audit [DBG] from='client.? 172.21.15.64:0/53249254' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:57:59.246 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:57:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:58 smithi152 conmon[97365]: audit 2023-06-10T20:57:58.267861+0000 mon.a (mon.0) 534 : audit [DBG] 2023-06-10T20:57:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:58 smithi152 conmon[97365]: from='client.? 172.21.15.64:0/53249254' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-10T20:58:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:57:59 smithi064 conmon[97224]: cluster 2023-06-10T20:57:58.522639+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v155: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:00.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:57:59 smithi064 conmon[107122]: cluster 2023-06-10T20:57:58.522639+0000 mgr.y (mgr.14148) 167 : cluster [DBG] pgmap v155: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:59 smithi152 conmon[97365]: cluster 2023-06-10T20:57:58.522639+0000 mgr.y 2023-06-10T20:58:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:57:59 smithi152 conmon[97365]: (mgr.14148) 167 : cluster [DBG] pgmap v155: 1 pgs: 1 peering; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:00.855 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:58:00.856 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":50,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:57:56.981435+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:58:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:01 smithi064 conmon[107122]: audit 2023-06-10T20:58:00.850625 2023-06-10T20:58:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:01 smithi064 conmon[107122]: +0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.64:0/2511499266' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:01.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:01 smithi064 conmon[97224]: audit 2023-06-10T20:58:00 2023-06-10T20:58:01.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:01 smithi064 conmon[97224]: .850625+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.64:0/2511499266' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:01.421 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-06-10T20:56:09.705420+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-06-10T20:58:01.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool get device_health_metrics pg_num 2023-06-10T20:58:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:01 smithi152 conmon[97365]: audit 2023-06-10T20:58:00.850625+0000 mon.c (mon.2) 10 : audit 2023-06-10T20:58:01.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:01 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/2511499266' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:01.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:02.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[107122]: cluster 2023-06-10T20:58:00.523024+0000 mgr.y (mgr.14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:02.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[107122]: cluster 2023-06-10T20:58:00.993311+0000 mon.a (mon.0) 535 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T20:58:02.349 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[107122]: cluster 2023-06-10T20:58:00.993389+0000 mon.a (mon.0) 536 : cluster [INF] Cluster is now healthy 2023-06-10T20:58:02.349 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[97224]: cluster 2023-06-10T20:58:00.523024+0000 mgr.y (mgr.14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:02.349 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[97224]: cluster 2023-06-10T20:58:00.993311+0000 mon.a (mon.0) 535 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T20:58:02.349 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:02 smithi064 conmon[97224]: cluster 2023-06-10T20:58:00.993389+0000 mon.a (mon.0) 536 : cluster [INF] Cluster is now healthy 2023-06-10T20:58:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:02 smithi152 conmon[97365]: cluster 2023-06-10T20:58:00.523024+0000 mgr.y (mgr. 2023-06-10T20:58:02.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:02 smithi152 conmon[97365]: 14148) 168 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:02.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:02 smithi152 conmon[97365]: cluster 2023-06-10T20:58:00.993311+0000 mon.a (mon.0) 535 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T20:58:02.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:02 smithi152 conmon[97365]: cluster 2023-06- 2023-06-10T20:58:02.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:02 smithi152 conmon[97365]: 10T20:58:00.993389+0000 mon.a (mon.0) 536 : cluster [INF] Cluster is now healthy 2023-06-10T20:58:03.384 INFO:teuthology.orchestra.run.smithi064.stdout:pg_num: 1 2023-06-10T20:58:03.963 INFO:tasks.cephadm:Adding prometheus.a on smithi152 2023-06-10T20:58:03.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply prometheus '1;smithi152=a' 2023-06-10T20:58:04.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:04 smithi152 conmon[97365]: cluster 2023-06-10T20:58:02.523444+0000 mgr.y (mgr.14148) 169 : cluster 2023-06-10T20:58:04.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:04 smithi152 conmon[97365]: [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:04.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:04 smithi152 conmon[97365]: audit 2023-06-10T20:58:03.380720+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.64:0/1979861691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-10T20:58:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[97224]: cluster 2023-06-10T20:58:02.523444+0000 mgr.y 2023-06-10T20:58:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[97224]: (mgr.14148) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[97224]: audit 2023-06-10T20:58:03.380720+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.64:0/1979861691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-10T20:58:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[107122]: cluster 2023-06-10T20:58:02.523444+0000 mgr.y 2023-06-10T20:58:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[107122]: (mgr.14148) 169 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:04 smithi064 conmon[107122]: audit 2023-06-10T20:58:03.380720+0000 mon.a (mon.0) 537 : audit [DBG] from='client.? 172.21.15.64:0/1979861691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-10T20:58:05.784 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled prometheus update... 2023-06-10T20:58:06.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: cluster 2023-06-10T20:58:04.523787+0000 mgr.y (mgr.14148) 170 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.780100+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.819089+0000 mon.a (mon.0) 539 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.820938+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.821933+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.827334+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.834833+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-10T20:58:06.405 DEBUG:teuthology.orchestra.run.smithi152:prometheus.a> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a.service 2023-06-10T20:58:06.407 INFO:tasks.cephadm:Adding node-exporter.a on smithi064 2023-06-10T20:58:06.408 INFO:tasks.cephadm:Adding node-exporter.b on smithi152 2023-06-10T20:58:06.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply node-exporter '2;smithi064=a;smithi152=b' 2023-06-10T20:58:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: cluster 2023-06-10T20:58:04.523787+0000 mgr.y (mgr.14148) 170 : cluster [DBG] 2023-06-10T20:58:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06-10T20:58:05.780100+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06-10T20:58:05.819089+0000 mon.a (mon.0) 539 : audit 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: 10T20:58:05.820938+0000 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: mon.a (mon.0) 540 : audit 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:58:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: 10T20:58:05.821933+0000 mon.a (mon. 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: 0) 541 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: 2023-06-10T20:58:05.827334+0000 mon.a ( 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: mon.0) 542 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: -10T20:58:05.834833+0000 mon.a (mon.0) 543 : audit 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[107122]: [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: cluster 2023-06-10T20:58:04.523787+0000 mgr.y (mgr.14148) 170 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023-06-10T20:58:05.780100+0000 mon.a (mon. 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 0) 538 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 2023- 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 06-10T20:58:05. 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 819089+0000 mon.a (mon.0) 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 539 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 06-10T20:58:05.820938+0000 mon.a (mon. 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 0) 540 : audit [DBG] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:06.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: T20:58:05.821933+0000 mon.a (mon.0) 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 541 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 06-10T20:58:05.827334+0000 mon.a (mon. 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 0) 542 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 06-10T20:58:05.834833+0000 mon.a (mon. 2023-06-10T20:58:06.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97224]: 0) 543 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-10T20:58:06.436 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:58:07.129 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97677]: ignoring --setuser ceph since I am not root 2023-06-10T20:58:07.129 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97677]: ignoring --setgroup ceph since I am not root 2023-06-10T20:58:07.130 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:06 smithi064 conmon[97677]: debug 2023-06-10T20:58:06.877+0000 7f4d9f613700 1 -- 172.21.15.64:0/2704804798 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ad7d9b64e0 con 0x55ad7d98f000 2023-06-10T20:58:07.130 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97677]: debug 2023-06-10T20:58:07.026+0000 7f4daf7d8680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:58:07.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: audit 2023-06-10T20:58:05.774422+0000 2023-06-10T20:58:07.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: mgr.y (mgr.14148) 171 : audit [DBG] from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:07.130 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: cephadm 2023-06 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: -10T20:58:05.776098+0000 mgr.y (mgr.14148) 172 : cephadm [INF] Saving service prometheus spec with placement smithi152=a;count:1 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: audit 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: 2023-06-10T20:58:06.835969+0000 mon.a 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: (mon.0) 544 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: cluster 2023- 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: 06-10T20:58:06.836044+0000 mon.a (mon. 2023-06-10T20:58:07.131 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[107122]: 0) 545 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-06-10T20:58:07.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: audit 2023-06-10T20:58:05.774422+0000 2023-06-10T20:58:07.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: mgr.y (mgr.14148) 171 : audit [DBG] from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:07.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:05.776098+0000 mgr.y ( 2023-06-10T20:58:07.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: mgr.14148) 172 : cephadm [INF] Saving service prometheus spec with placement smithi152=a;count:1 2023-06-10T20:58:07.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: audit 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: 2023-06-10T20:58: 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: 06.835969+0000 mon.a (mon. 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: 0) 544 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: cluster 2023-06-10T20:58 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: :06.836044+0000 mon.a (mon.0) 545 : cluster [DBG] 2023-06-10T20:58:07.133 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97224]: mgrmap e15: y(active, since 3m), standbys: x 2023-06-10T20:58:07.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[99677]: ignoring --setuser ceph since I am not root 2023-06-10T20:58:07.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[99677]: ignoring --setgroup ceph since I am not root 2023-06-10T20:58:07.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:06 smithi152 conmon[99677]: debug 2023-06-10T20:58:06.996+0000 7f86751fe680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T20:58:07.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[99677]: debug 2023-06-10T20:58:07.083+0000 7f86751fe680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:58:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: audit 2023-06-10T20:58:05.774422+0000 mgr.y (mgr.14148) 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: 171 : audit [DBG] from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:05.776098+0000 mgr.y (mgr.14148) 172 : cephadm [INF] Saving service prometheus spec with placement smithi152=a;count:1 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: 58:06.835969+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14148 172.21.15.64:0/3297414636' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: cluster 2023-06-10T20:58:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: 2023-06-10T20:58:06.836044+0000 mon.a 2023-06-10T20:58:07.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: (mon.0) 545 : cluster [DBG] mgrmap e15: y(active, since 3m), standbys: x 2023-06-10T20:58:07.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[97365]: 2023-06-10T20:58:07.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:07 smithi152 conmon[99677]: debug 2023-06-10T20:58:07.473+0000 7f86751fe680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:58:07.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97677]: debug 2023-06-10T20:58:07.124+0000 7f4daf7d8680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T20:58:07.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:07 smithi064 conmon[97677]: debug 2023-06-10T20:58:07.572+0000 7f4daf7d8680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T20:58:08.571 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:08 smithi152 conmon[99677]: debug 2023-06-10T20:58:08.273+0000 7f86751fe680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:58:08.572 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:08 smithi152 conmon[99677]: debug 2023-06-10T20:58:08.357+0000 7f86751fe680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:58:08.851 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:08 smithi064 conmon[97677]: debug 2023-06-10T20:58:08.501+0000 7f4daf7d8680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T20:58:08.851 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:08 smithi064 conmon[97677]: debug 2023-06-10T20:58:08.597+0000 7f4daf7d8680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T20:58:08.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:08 smithi152 conmon[99677]: debug 2023-06-10T20:58:08.568+0000 7f86751fe680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:58:09.796 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:09 smithi152 conmon[99677]: debug 2023-06-10T20:58:09.500+0000 7f86751fe680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:58:09.796 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:09 smithi152 conmon[99677]: debug 2023-06-10T20:58:09.626+0000 7f86751fe680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:58:09.797 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:09 smithi152 conmon[99677]: debug 2023-06-10T20:58:09.704+0000 7f86751fe680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:58:10.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:09 smithi152 conmon[99677]: debug 2023-06-10T20:58:09.792+0000 7f86751fe680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:58:10.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:09 smithi152 conmon[99677]: debug 2023-06-10T20:58:09.989+0000 7f86751fe680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:58:10.246 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:08 smithi064 conmon[97677]: debug 2023-06-10T20:58:08.846+0000 7f4daf7d8680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T20:58:10.247 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:09 smithi064 conmon[97677]: debug 2023-06-10T20:58:09.983+0000 7f4daf7d8680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T20:58:10.247 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:10 smithi064 conmon[97677]: debug 2023-06-10T20:58:10.146+0000 7f4daf7d8680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:58:10.577 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:10 smithi064 conmon[97677]: debug 2023-06-10T20:58:10.242+0000 7f4daf7d8680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T20:58:10.578 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:10 smithi064 conmon[97677]: debug 2023-06-10T20:58:10.352+0000 7f4daf7d8680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T20:58:10.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:10 smithi152 conmon[99677]: debug 2023-06-10T20:58:10.263+0000 7f86751fe680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:58:10.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:10 smithi152 conmon[99677]: debug 2023-06-10T20:58:10.360+0000 7f86751fe680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:58:10.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:10 smithi064 conmon[97677]: debug 2023-06-10T20:58:10.573+0000 7f4daf7d8680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T20:58:11.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:10 smithi064 conmon[97677]: debug 2023-06-10T20:58:10.940+0000 7f4daf7d8680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T20:58:11.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:11 smithi064 conmon[97677]: debug 2023-06-10T20:58:11.060+0000 7f4daf7d8680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T20:58:11.430 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.082+0000 7f86751fe680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:58:11.431 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.163+0000 7f86751fe680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:58:11.431 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.254+0000 7f86751fe680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:58:11.945 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.426+0000 7f86751fe680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:58:11.945 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.687+0000 7f86751fe680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:58:11.945 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.816+0000 7f86751fe680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:58:12.204 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:11 smithi152 conmon[99677]: debug 2023-06-10T20:58:11.947+0000 7f86751fe680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:58:12.204 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: debug 2023-06-10T20:58:12.117+0000 7f86751fe680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:58:12.278 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:11 smithi064 conmon[97677]: debug 2023-06-10T20:58:11.936+0000 7f4daf7d8680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T20:58:12.278 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.036+0000 7f4daf7d8680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T20:58:12.278 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.148+0000 7f4daf7d8680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: cluster 2023-06-10T20:58:12.214556+0000 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: cluster 2023-06-10T20:58:12.214699+0000 mon.a (mon.0 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: ) 547 : cluster [DBG] Standby manager daemon x started 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: audit 2023-06-10T20:58:12. 2023-06-10T20:58:12.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: 216513+0000 mon.c (mon.2) 11 : audit [DBG] 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: 58:12.217167+0000 mon.c (mon.2) 12 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: audit 2023-06-10T20:58:12 2023-06-10T20:58:12.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: .219786+0000 mon.c (mon.2) 13 : audit [DBG] 2023-06-10T20:58:12.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:58:12.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:58:12.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: 06-10T20:58:12.220116+0000 mon.c (mon.2) 14 2023-06-10T20:58:12.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97224]: : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:58:12.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.365+0000 7f4daf7d8680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: cluster 2023-06-10T20:58: 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 12.214556+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: cluster 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2023-06-10T20:58 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: :12.214699+0000 mon.a ( 2023-06-10T20:58:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: mon.0) 547 : cluster 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: [DBG] Standby manager daemon x started 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: audit 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: T20:58:12.216513+0000 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: mon.c (mon.2) 11 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:58:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: audit 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2023-06-10T20:58 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: :12.217167+0000 mon.c ( 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: mon.2) 12 : audit 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: audit 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2023-06-10T20: 2023-06-10T20:58:12.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 58:12.219786+0000 mon.c 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: (mon.2) 13 : audit 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: audit 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2023-06 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: -10T20:58:12 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: .220116+0000 mon.c (mon. 2023-06-10T20:58:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: 2) 14 : audit [DBG] 2023-06-10T20:58:12.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[107122]: from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:58:12.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: debug 2023-06-10T20:58:12.199+0000 7f86751fe680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:58:12.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: [10/Jun/2023:20:58:12] ENGINE Bus STARTING 2023-06-10T20:58:12.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: CherryPy Checker: 2023-06-10T20:58:12.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: The Application mounted at '' has an empty config. 2023-06-10T20:58:12.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: 2023-06-10T20:58:12.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: [10/Jun/2023:20:58:12] ENGINE Serving on http://:::9283 2023-06-10T20:58:12.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[99677]: [10/Jun/2023:20:58:12] ENGINE Bus STARTED 2023-06-10T20:58:12.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: cluster 2023-06-10T20:58:12.214556+0000 mon.a (mon.0) 546 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T20:58:12.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: cluster 2023-06-10T20:58:12.214699+0000 mon.a (mon.0) 547 : cluster [DBG] Standby manager daemon x started 2023-06-10T20:58:12.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: audit 2023-06-10T20:58:12.216513+0000 2023-06-10T20:58:12.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: mon.c (mon.2) 11 : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: audit 2023-06-10T20: 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: 58:12.217167+0000 mon.c (mon.2) 12 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: T20:58:12.219786+0000 mon.c ( 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: mon.2) 13 : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T20:58:12.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:58:12.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: 06-10T20:58:12.220116 2023-06-10T20:58:12.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: +0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.? 172.21.15.152:0/2665153733' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T20:58:12.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:12 smithi152 conmon[97365]: 2023-06-10T20:58:12.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.653+0000 7f4daf7d8680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T20:58:12.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.817+0000 7f4daf7d8680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T20:58:13.277 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:12 smithi064 conmon[97677]: debug 2023-06-10T20:58:12.986+0000 7f4daf7d8680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T20:58:13.278 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[97677]: debug 2023-06-10T20:58:13.202+0000 7f4daf7d8680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T20:58:13.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[97677]: debug 2023-06-10T20:58:13.305+0000 7f4daf7d8680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T20:58:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[97224]: cluster 2023-06-10T20:58:12. 2023-06-10T20:58:13.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[97224]: 271552+0000 mon.a (mon.0) 548 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-06-10T20:58:13.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[107122]: cluster 2023-06-10T20:58:12.271552+0000 mon.a (mon 2023-06-10T20:58:13.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:13 smithi064 conmon[107122]: .0) 548 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-06-10T20:58:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:13 smithi152 conmon[97365]: cluster 2023-06-10T20:58:12.271552+0000 mon.a (mon.0) 548 : cluster [DBG] mgrmap e16: y(active, since 4m), standbys: x 2023-06-10T20:58:14.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:14 smithi152 conmon[97365]: cluster 2023-06-10T20:58:13.320226+0000 mon.a (mon.0) 549 : cluster [INF] 2023-06-10T20:58:14.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:14 smithi152 conmon[97365]: Active manager daemon y restarted 2023-06-10T20:58:14.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:14 smithi152 conmon[97365]: cluster 2023-06-10T20:58:13.320732+0000 mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-06-10T20:58:14.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:14 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T20:58:14.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:14 smithi152 conmon[97365]: 20:58:13.324264+0000 mon.a (mon.0) 551 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-10T20:58:14.567 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: [10/Jun/2023:20:58:14] ENGINE Bus STARTING 2023-06-10T20:58:14.567 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: [80B blob data] 2023-06-10T20:58:14.567 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: [79B blob data] 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: cluster 2023-06-10T20:58:13.320226+0000 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: mon.a (mon.0) 549 : cluster [INF] Active manager daemon y restarted 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: :58:13.320732+0000 mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: 2023-06-10T20:58:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: cluster 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: 2023-06-10T20:58:13.324264+0000 mon.a ( 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[107122]: mon.0) 551 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: cluster 2023-06-10T20:58:13.320226 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: +0000 mon.a (mon.0) 549 : cluster [INF] Active manager daemon y restarted 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: cluster 2023-06-10T20:58:13.320732+0000 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: mon.a (mon.0) 550 : cluster [INF] Activating manager daemon y 2023-06-10T20:58:14.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: cluster 2023-06 2023-06-10T20:58:14.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: -10T20:58:13.324264+0000 mon.a (mon.0) 551 : 2023-06-10T20:58:14.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97224]: cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-10T20:58:14.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: CherryPy Checker: 2023-06-10T20:58:14.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: The Application mounted at '' has an empty config. 2023-06-10T20:58:14.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: 2023-06-10T20:58:14.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: [10/Jun/2023:20:58:14] ENGINE Serving on http://:::9283 2023-06-10T20:58:14.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:14 smithi064 conmon[97677]: [10/Jun/2023:20:58:14] ENGINE Bus STARTED 2023-06-10T20:58:15.305 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled node-exporter update... 2023-06-10T20:58:15.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: cluster 2023-06-10T20:58:14.286270+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.965622s), standbys: x 2023-06-10T20:58:15.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.287692+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:58:15.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.287944+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.288146+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.289082+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.289280+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.289550+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.289786+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.290035+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:58:15.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.290330+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.290614+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.290898+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.291182+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.291469+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.291774+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:58:15.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: -10T20:58:14.292020+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:14.292552+0000 mon.a (mon.0) 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: 568 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: cluster 2023-06-10T20:58 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: :14.307449+0000 mon.a (mon.0) 569 : cluster [INF] Manager daemon y is now available 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: 2023-06-10T20:58:14.441778+0000 mon.a (mon.0) 570 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: 2023-06-10T20:58:14.443682+0000 mon.a 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: (mon.0) 571 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023-06-10T20:58 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: :14.481129+0000 mon.a (mon.0) 572 : audit [INF] 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023- 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: 06-10T20:58:14.491493+0000 mon.a (mon.0) 573 : audit [INF] 2023-06-10T20:58:15.559 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:58:15.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: audit 2023 2023-06-10T20:58:15.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: -06-10T20:58:14.497337+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:58:15.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:15 smithi152 conmon[97365]: 2023-06-10T20:58:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: cluster 2023-06-10T20:58:14.286270+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.965622s), standbys: x 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.287692+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.287944+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.288146+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.289082+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.289280+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.289550+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:58:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.289786+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:58:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.290035+0000 mon.a 2023-06-10T20:58:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:58:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.290330+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:58:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.290614+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:58:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.290898+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:58:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.291182+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:58:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.291469+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.291774+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.292020+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.292552+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: cluster 2023-06-10T20:58:14.307449+0000 mon.a (mon.0) 569 : cluster [INF] Manager daemon y is now available 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.441778+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.443682+0000 mon.a (mon.0) 571 : audit 2023-06-10T20:58:15.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.481129+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:15.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20:58:14.491493+0000 mon.a (mon 2023-06-10T20:58:15.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: .0) 573 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:58:15.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: audit 2023-06-10T20 2023-06-10T20:58:15.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: :58:14.497337+0000 mon.a (mon.0) 574 : audit [INF] 2023-06-10T20:58:15.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[97224]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:58:15.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: cluster 2023-06-10T20:58:14.286270+0000 mon.a (mon.0) 552 : cluster [DBG] mgrmap e17: y(active, starting, since 0.965622s), standbys: x 2023-06-10T20:58:15.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023 2023-06-10T20:58:15.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: -06-10T20:58:14.287692+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T20:58:15.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.287944+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T20:58:15.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.288146+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T20:58:15.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.289082+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T20:58:15.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.289280+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T20:58:15.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.289550+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T20:58:15.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:14.289786+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T20:58:15.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:58:15.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 58:14.290035+0000 mon.a (mon 2023-06-10T20:58:15.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: .0) 560 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T20:58:15.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:58:15.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 06-10T20:58:14. 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 290330+0000 mon.a (mon.0) 561 : audit 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: T20:58:14.290614+0000 mon.a (mon. 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 0) 562 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:15.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 2023-06-10T20 2023-06-10T20:58:15.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: :58:14.290898+0000 mon.a (mon. 2023-06-10T20:58:15.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 0) 563 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T20:58:15.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:15.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: -10T20:58:14.291182+0000 mon.a (mon 2023-06-10T20:58:15.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: .0) 564 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T20:58:15.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:15.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: -10T20:58:14.291469+0000 2023-06-10T20:58:15.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: mon.a (mon.0) 565 : audit 2023-06-10T20:58:15.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T20:58:15.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:58:15.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 06-10T20:58:14.291774+0000 2023-06-10T20:58:15.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: mon.a (mon.0) 566 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T20:58:15.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 2023-06-10T20:58:15.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:58:15.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 06-10T20:58:14.292020+0000 mon.a ( 2023-06-10T20:58:15.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T20:58:15.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:15.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: -10T20:58:14.292552+0000 mon.a (mon.0 2023-06-10T20:58:15.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: ) 568 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T20:58:15.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:58:15.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: :58:14.307449+0000 mon.a (mon.0) 2023-06-10T20:58:15.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 569 : cluster [INF] Manager daemon y is now available 2023-06-10T20:58:15.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20: 2023-06-10T20:58:15.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 58:14.441778+0000 mon.a (mon.0) 570 2023-06-10T20:58:15.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:58:15.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 20:58:14.443682+0000 mon.a (mon 2023-06-10T20:58:15.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: .0) 571 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:15.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:15.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:58:15.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: T20:58:14.481129+0000 mon.a (mon 2023-06-10T20:58:15.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: .0) 572 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:15.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06-10T20:58:15.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 2023-06-10T20:58:14. 2023-06-10T20:58:15.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 491493+0000 mon.a (mon.0) 573 : 2023-06-10T20:58:15.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T20:58:15.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:58:15.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: 10T20:58:14.497337+0000 mon.a ( 2023-06-10T20:58:15.681 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:15 smithi064 conmon[107122]: mon.0) 574 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T20:58:16.160 DEBUG:teuthology.orchestra.run.smithi064:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.a.service 2023-06-10T20:58:16.163 DEBUG:teuthology.orchestra.run.smithi152:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.b.service 2023-06-10T20:58:16.165 INFO:tasks.cephadm:Adding alertmanager.a on smithi064 2023-06-10T20:58:16.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply alertmanager '1;smithi064=a' 2023-06-10T20:58:16.196 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:58:16.197 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:58:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:16 smithi152 conmon[97365]: cluster 2023-06-10T20:58:15.289217+0000 mon.a (mon 2023-06-10T20:58:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:16 smithi152 conmon[97365]: .0) 575 : cluster [DBG] mgrmap e18: y(active, since 1.96858s), standbys: x 2023-06-10T20:58:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:16 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:15.298388+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi064=a;smithi152=b;count:2 2023-06-10T20:58:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:16 smithi152 conmon[97365]: audit 2023-06-10T20:58:15.301316+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:16 smithi152 conmon[97365]: audit 2023-06-10T20:58:15.985208+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[97224]: cluster 2023-06-10T20:58:15. 2023-06-10T20:58:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[97224]: 289217+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e18: y(active, since 1.96858s), standbys: x 2023-06-10T20:58:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:15.298388+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi064=a;smithi152=b;count:2 2023-06-10T20:58:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[97224]: audit 2023-06-10T20:58:15.301316+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[97224]: audit 2023-06-10T20:58:15.985208+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: cluster 2023-06-10T20:58:15.289217+0000 mon.a (mon.0 2023-06-10T20:58:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: ) 575 : cluster [DBG] mgrmap e18: y(active, since 1.96858s), standbys: x 2023-06-10T20:58:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:15.298388+0000 mgr.y (mgr.14427) 3 : cephadm [INF] 2023-06-10T20:58:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: Saving service node-exporter spec with placement smithi064=a;smithi152=b;count:2 2023-06-10T20:58:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: audit 2023-06-10T20:58:15.301316+0000 mon.a ( 2023-06-10T20:58:16.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:16.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: audit 2023-06-10T20:58:16.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: 2023-06-10T20:58:15.985208+0000 mon.a 2023-06-10T20:58:16.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:16 smithi064 conmon[107122]: (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cluster 2023-06-10T20:58:16.290231+0000 mgr.y (mgr. 2023-06-10T20:58:17.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: 14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:17.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cluster 2023-06-10T20:58:16.305184+0000 mon.a (mon.0) 578 : cluster 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10T20:58:16.434102+0000 mon.a ( 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10T20:58 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: :16.796197+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10T20:58:16.797149+0000 mon.a (mon.0) 2023-06-10T20:58:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: 581 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: 10T20:58:16.798522+0000 mgr.y (mgr.14427) 5 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: T20:58:16.798791+0000 mgr.y (mgr.14427) 6 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: 2023-06-10T20:58:16.824594+0000 mgr.y (mgr.14427) 7 : cephadm [INF] 2023-06-10T20:58:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: T20:58:16.832502+0000 mgr.y (mgr.14427) 8 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10T20:58:16.858182+0000 mon.a (mon.0) 582 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: 2023-06-10T20:58:16.864622+0000 mon.a (mon.0 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: ) 583 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: T20:58:16.871236+0000 mon.a (mon.0) 584 : audit 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[107122]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: T20:58:16.290231+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cluster 2023-06-10T20:58:16.305184+0000 mon.a (mon.0) 578 2023-06-10T20:58:17.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06-10T20:58:16.434102+0000 mon.a (mon 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: .0) 579 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:16 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: .796197+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06-10T20:58:17.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:16.797149+0000 mon.a ( 2023-06-10T20:58:17.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: mon.0) 581 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:17.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cephadm 2023-06- 2023-06-10T20:58:17.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 10T20:58:16.798522+0000 mgr.y ( 2023-06-10T20:58:17.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: mgr.14427) 5 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:58:17.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 06-10T20:58:16.798791+0000 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: mgr.y (mgr.14427) 6 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:16.824594+0000 mgr.y 2023-06-10T20:58:17.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: (mgr.14427) 7 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:16.832502+0000 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: mgr.y (mgr.14427) 8 : cephadm [INF] 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06-10T20:58:16 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: .858182+0000 mon.a (mon.0) 582 : 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 20:58:16.864622+0000 mon.a (mon 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: .0) 583 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: audit 2023-06 2023-06-10T20:58:17.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: -10T20:58:16.871236+0000 2023-06-10T20:58:17.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: mon.a (mon.0) 584 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:17 smithi064 conmon[97224]: 2023-06-10T20:58:17.672 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cluster 2023-06-10T20:58:16.290231+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cluster 2023-06-10T20:58:16.305184+0000 mon.a (mon.0) 578 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.434102+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.796197+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.797149+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:16.798522+0000 mgr.y (mgr.14427) 5 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:16.798791+0000 mgr.y (mgr.14427) 6 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T20:58:17.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:16.824594+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:16.832502+0000 mgr.y (mgr.14427) 8 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T20:58:17.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.858182+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.864622+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:17.674 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:17 smithi152 conmon[97365]: audit 2023-06-10T20:58:16.871236+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:18.056 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled alertmanager update... 2023-06-10T20:58:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:18 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:16.889737+0000 mgr.y (mgr.14427) 9 2023-06-10T20:58:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:18 smithi152 conmon[97365]: : cephadm [INF] Deploying daemon prometheus.a on smithi152 2023-06-10T20:58:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:18 smithi152 conmon[97365]: audit 2023-06-10T20:58:18.052372+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:16.889737+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi152 2023-06-10T20:58:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[97224]: audit 2023-06-10T20:58:18.052372+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:16.889737+0000 mgr.y (mgr 2023-06-10T20:58:18.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[107122]: .14427) 9 : cephadm [INF] Deploying daemon prometheus.a on smithi152 2023-06-10T20:58:18.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[107122]: audit 2023-06-10T20:58:18.052372+0000 mon.a (mon. 2023-06-10T20:58:18.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:18 smithi064 conmon[107122]: 0) 585 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:18.740 DEBUG:teuthology.orchestra.run.smithi064:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@alertmanager.a.service 2023-06-10T20:58:18.742 INFO:tasks.cephadm:Adding grafana.a on smithi152 2023-06-10T20:58:18.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph orch apply grafana '1;smithi152=a' 2023-06-10T20:58:18.776 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T20:58:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: audit 2023-06-10T20:58:18.046615+0000 mgr.y 2023-06-10T20:58:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: (mgr.14427) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T20:58:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: 10T20:58:18.048305+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi064=a;count:1 2023-06-10T20:58:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: cluster 2023-06-10T20:58:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: 2023-06-10T20:58:18.290669+0000 mgr.y (mgr 2023-06-10T20:58:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[107122]: .14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:19.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[97224]: audit 2023-06-10T20:58:18.046615+0000 mgr.y (mgr.14427) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:19.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:18.048305+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi064=a;count:1 2023-06-10T20:58:19.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[97224]: cluster 2023-06-10T20:58:18.290669+0000 2023-06-10T20:58:19.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[97224]: mgr.y (mgr.14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:19.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:19 smithi064 conmon[97224]: 2023-06-10T20:58:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:19 smithi152 conmon[97365]: audit 2023-06-10T20:58:18.046615+0000 mgr.y (mgr.14427) 10 : audit [DBG] from='client.24361 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi064=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:19.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:19 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:18.048305+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Saving service alertmanager spec with placement smithi064=a;count:1 2023-06-10T20:58:19.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:19 smithi152 conmon[97365]: cluster 2023-06-10T20:58:18.290669+0000 mgr.y (mgr.14427) 12 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:21.105 INFO:teuthology.orchestra.run.smithi152.stdout:Scheduled grafana update... 2023-06-10T20:58:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:21 smithi152 conmon[97365]: cluster 2023-06-10T20:58:20.291212+0000 mgr.y (mgr. 2023-06-10T20:58:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:21 smithi152 conmon[97365]: 14427) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:21 smithi152 conmon[97365]: audit 2023-06-10T20:58:21.101426+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: cluster 2023-06-10T20:58:20.291212 2023-06-10T20:58:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: audit 2023-06-10T20:58:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: 2023-06-10T20:58: 2023-06-10T20:58:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: 21.101426+0000 mon.a (mon. 2023-06-10T20:58:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: 0) 586 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[107122]: 2023-06-10T20:58:21.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[97224]: cluster 2023-06-10T20:58:20. 2023-06-10T20:58:21.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[97224]: 291212+0000 mgr.y (mgr.14427) 13 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:21.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[97224]: audit 2023-06-10T20:58:21.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[97224]: 2023-06-10T20:58:21.101426 2023-06-10T20:58:21.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:21 smithi064 conmon[97224]: +0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:22.119 DEBUG:teuthology.orchestra.run.smithi152:grafana.a> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@grafana.a.service 2023-06-10T20:58:22.122 INFO:tasks.cephadm:Setting up client nodes... 2023-06-10T20:58:22.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-06-10T20:58:22.151 INFO:journalctl@ceph.grafana.a.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T20:58:22.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[107122]: audit 2023-06-10T20:58 2023-06-10T20:58:22.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[107122]: :21.093739+0000 mgr.y (mgr.14427) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:22.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:21.095348+0000 mgr.y (mgr.14427) 15 : cephadm [INF] Saving service grafana spec with placement smithi152=a;count:1 2023-06-10T20:58:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[97224]: audit 2023-06-10T20:58:21.093739+0000 mgr.y ( 2023-06-10T20:58:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[97224]: mgr.14427) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:22 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:21.095348+0000 mgr.y (mgr.14427) 15 : cephadm [INF] Saving service grafana spec with placement smithi152=a;count:1 2023-06-10T20:58:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:22 smithi152 conmon[97365]: audit 2023-06-10T20:58:21.093739+0000 mgr.y (mgr.14427) 14 : audit [DBG] from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi152=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:58:22.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:22 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:21.095348+0000 mgr.y (mgr.14427) 15 : cephadm [INF] Saving service grafana spec with placement smithi152=a;count:1 2023-06-10T20:58:23.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:23 smithi064 conmon[97224]: cluster 2023-06-10T20:58:22.291523+0000 mgr.y (mgr.14427) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:23 smithi064 conmon[107122]: cluster 2023-06-10T20:58:22. 2023-06-10T20:58:23.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:23 smithi064 conmon[107122]: 291523+0000 mgr.y (mgr.14427) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:23 smithi152 conmon[97365]: cluster 2023-06-10T20:58:22.291523+0000 mgr.y (mgr.14427 2023-06-10T20:58:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:23 smithi152 conmon[97365]: ) 16 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:24.162 INFO:teuthology.orchestra.run.smithi064.stdout:[client.0] 2023-06-10T20:58:24.162 INFO:teuthology.orchestra.run.smithi064.stdout: key = AQDw44Rksi5LCRAAfXYP+BbB5xs+hlRKChRTPg== 2023-06-10T20:58:24.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[97224]: audit 2023-06-10T20:58:24.155768+0000 mon.a (mon.0) 587 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:24.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[97224]: 2023-06-10T20:58:24.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[97224]: audit 2023-06-10T20:58:24.159105+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[107122]: audit 2023-06-10T20:58:24. 2023-06-10T20:58:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[107122]: 155768+0000 mon.a (mon.0) 587 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:24 smithi064 conmon[107122]: audit 2023-06-10T20:58:24.159105+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:24 smithi152 conmon[97365]: audit 2023-06-10T20:58:24.155768+0000 mon.a (mon 2023-06-10T20:58:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:24 smithi152 conmon[97365]: .0) 587 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:24 smithi152 conmon[97365]: audit 2023-06-10T20:58:24.159105+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.64:0/2467981979' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:24.793 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T20:58:24.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-06-10T20:58:24.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-06-10T20:58:24.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-06-10T20:58:25.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:25 smithi152 conmon[97365]: cluster 2023-06-10T20:58:24.291808+0000 mgr.y (mgr.14427) 17 : 2023-06-10T20:58:25.620 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:25 smithi152 conmon[97365]: cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:25 smithi064 conmon[97224]: cluster 2023-06-10T20:58:24.291808+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:25 smithi064 conmon[107122]: cluster 2023-06-10T20:58:24.291808+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:26 smithi064 conmon[107122]: cluster 2023-06-10T20:58:26.292243+0000 mgr.y 2023-06-10T20:58:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:26 smithi064 conmon[107122]: (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:26 smithi064 conmon[97224]: cluster 2023-06-10T20:58:26.292243 2023-06-10T20:58:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:26 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:27 smithi152 conmon[97365]: cluster 2023-06-10T20:58:26.292243+0000 mgr.y (mgr.14427) 18 : cluster 2023-06-10T20:58:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:27 smithi152 conmon[97365]: [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:29.011 INFO:teuthology.orchestra.run.smithi152.stdout:[client.1] 2023-06-10T20:58:29.011 INFO:teuthology.orchestra.run.smithi152.stdout: key = AQD144RkpRYrABAA51qcSOxVpJ9fTBUyMdV3sQ== 2023-06-10T20:58:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: cluster 2023-06-10T20:58:28.292635+0000 mgr.y 2023-06-10T20:58:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: audit 2023-06-10T20:58:29 2023-06-10T20:58:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: .001927+0000 mon.b (mon.1) 23 : audit [INF] from='client.? 172.21.15.152:0/2835908060' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:58:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: 06-10T20:58:29. 2023-06-10T20:58:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: 002637+0000 mon.a (mon.0 2023-06-10T20:58:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: ) 589 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: T20:58:29.007427+0000 mon.a (mon.0 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[107122]: ) 590 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: cluster 2023- 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: 06-10T20:58:28.292635+0000 mgr.y (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:29.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: audit 2023-06-10T20:58:29. 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: 001927+0000 mon.b (mon.1) 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: 23 : audit [INF] from='client.? 172.21.15.152:0/2835908060' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: audit 2023 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: -06-10T20:58:29.002637+0000 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: mon.a (mon.0) 589 : audit 2023-06-10T20:58:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: audit 2023 2023-06-10T20:58:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: -06-10T20:58:29. 2023-06-10T20:58:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: 007427+0000 mon.a (mon.0) 2023-06-10T20:58:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:29 smithi064 conmon[97224]: 590 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:29 smithi152 conmon[97365]: cluster 2023-06-10T20:58:28.292635+0000 mgr.y (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:29.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:29 smithi152 conmon[97365]: audit 2023-06-10T20:58:29.001927+0000 mon.b (mon.1) 23 : audit [INF] from='client.? 172.21.15.152:0/2835908060' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:29 smithi152 conmon[97365]: audit 2023-06-10T20:58:29.002637+0000 mon.a (mon.0) 589 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-10T20:58:29.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:29 smithi152 conmon[97365]: audit 2023-06-10T20:58:29.007427+0000 mon.a (mon.0) 590 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-10T20:58:30.155 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2023-06-10T20:58:30.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-06-10T20:58:30.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-06-10T20:58:30.257 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-06-10T20:58:30.258 DEBUG:teuthology.orchestra.run.smithi064:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-06-10T20:58:30.444 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 systemd[1]: Starting Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:58:30.608 INFO:teuthology.misc.health.smithi064.stdout: 2023-06-10T20:58:30.608 INFO:teuthology.misc.health.smithi064.stdout:{"epoch":51,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:58:13.320272+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:58:30.620 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-06-10T20:58:30.621 INFO:tasks.cephadm:Creating RBD pool 2023-06-10T20:58:30.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- sudo ceph --cluster ceph osd pool create rbd 8 2023-06-10T20:58:31.194 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 bash[126737]: bc2b0c5cd82151b5e7909f19b04342084eac6859dfdcb8692f9410361b827e6c 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.839Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.840Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.840Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi152 (none))" 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.840Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.840Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-10T20:58:31.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.841Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.842Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.843Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 systemd[1]: Started Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.846Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.846Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.971µs 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.846Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-10T20:58:31.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.846Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.846Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.252µs wal_replay_duration=174.482µs total_replay_duration=214.959µs 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.847Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.847Z caller=main.go:947 level=info msg="TSDB started" 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.847Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.862Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.900092ms db_storage=720ns remote_storage=1.207µs web_handler=328ns query_engine=614ns scrape=475.533µs scrape_sd=22.766µs notify=1.282µs notify_sd=2.16µs rules=14.043951ms 2023-06-10T20:58:31.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 20:58:30 smithi152 conmon[126778]: ts=2023-06-10T20:58:30.862Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-10T20:58:31.352 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[97224]: cluster 2023-06-10T20:58:30.292995+0000 mgr.y (mgr 2023-06-10T20:58:31.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[107122]: cluster 2023-06-10T20:58:30.292995+0000 2023-06-10T20:58:31.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[107122]: mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:31.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[107122]: audit 2023-06-10T20:58:30.604552+0000 mon.a (mon.0) 591 : audit [DBG] from='client.? 172.21.15.64:0/664282883' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:31.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[107122]: audit 2023-06-10T20:58:30.903002+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:31.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[97224]: .14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:31.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[97224]: audit 2023-06-10T20:58:30.604552+0000 mon.a (mon.0) 591 : audit [DBG] from='client.? 172.21.15.64:0/664282883' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:31.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:31 smithi064 conmon[97224]: audit 2023-06-10T20:58:30.903002+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:31 smithi152 conmon[97365]: cluster 2023-06-10T20:58:30.292995+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:31 smithi152 conmon[97365]: audit 2023-06-10T20:58:30.604552+0000 mon.a (mon.0) 591 : audit [DBG] from='client.? 172.21.15.64:0/664282883' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:31 smithi152 conmon[97365]: audit 2023-06-10T20:58:30.903002+0000 mon.a (mon.0) 592 2023-06-10T20:58:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:31 smithi152 conmon[97365]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:31.908 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:31 smithi064 systemd[1]: Starting Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:58:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:32 smithi064 conmon[107122]: cephadm 2023-06-10T20: 2023-06-10T20:58:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:32 smithi064 conmon[107122]: 58:30.905024+0000 mgr.y (mgr.14427) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi064 2023-06-10T20:58:32.659 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:32 smithi064 bash[135658]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-06-10T20:58:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:32 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:30.905024 2023-06-10T20:58:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:32 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 21 : cephadm [INF] Deploying daemon node-exporter.a on smithi064 2023-06-10T20:58:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:32 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:30.905024+0000 mgr.y (mgr.14427) 21 : cephadm [INF] 2023-06-10T20:58:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:32 smithi152 conmon[97365]: Deploying daemon node-exporter.a on smithi064 2023-06-10T20:58:32.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:32 smithi064 bash[135658]: Getting image source signatures 2023-06-10T20:58:32.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:32 smithi064 bash[135658]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-06-10T20:58:32.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:32 smithi064 bash[135658]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-06-10T20:58:33.371 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'rbd' created 2023-06-10T20:58:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:58:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: :58:32.293398+0000 mgr.y (mgr.14427) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: audit 2023-06-10T20:58:33.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: 2023-06-10T20:58:32.616508+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.64:0/3651315135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T20:58:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: T20:58:32.616845+0000 mon.a (mon.0) 2023-06-10T20:58:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[97224]: 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: cluster 2023-06-10T20:58:32 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: .293398+0000 mgr.y (mgr.14427) 22 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: audit 2023-06-10T20:58:32.616508+0000 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.64:0/3651315135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: -10T20:58:32.616845+0000 mon.a (mon. 2023-06-10T20:58:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:33 smithi064 conmon[107122]: 0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.662 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:32 smithi064 bash[135658]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-06-10T20:58:33.662 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:33 smithi064 bash[135658]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-06-10T20:58:33.662 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:33 smithi064 bash[135658]: Writing manifest to image destination 2023-06-10T20:58:33.662 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:33 smithi064 bash[135658]: Storing signatures 2023-06-10T20:58:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:33 smithi152 conmon[97365]: cluster 2023-06-10T20:58:32.293398+0000 mgr.y (mgr.14427) 22 : cluster 2023-06-10T20:58:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:33 smithi152 conmon[97365]: [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:33 smithi152 conmon[97365]: audit 2023-06-10T20:58:32.616508+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.64:0/3651315135' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:33 smithi152 conmon[97365]: audit 2023-06-10T20:58:32.616845+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-10T20:58:33.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:33 smithi152 conmon[97365]: 2023-06-10T20:58:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[97224]: audit 2023-06-10T20:58:33.362129 2023-06-10T20:58:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[97224]: +0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-10T20:58:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[97224]: cluster 2023-06-10T20:58:33.362214+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-10T20:58:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[97224]: audit 2023-06-10T20:58:33.362580+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[97224]: audit 2023-06-10T20:58:33.363278+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: audit 2023-06-10T20:58:33.362129+0000 mon.a 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: cluster 2023 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: -06-10T20:58:33.362214+0000 mon.a (mon.0) 595 : cluster 2023-06-10T20:58:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: T20:58:33.362580+0000 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: audit 2023-06-10T20:58: 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: 33.363278+0000 mon.a (mon.0) 597 : 2023-06-10T20:58:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:34 smithi064 conmon[107122]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:58:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:34 smithi152 conmon[97365]: audit 2023-06-10T20:58:33.362129+0000 mon.a (mon 2023-06-10T20:58:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:34 smithi152 conmon[97365]: .0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-10T20:58:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:34 smithi152 conmon[97365]: cluster 2023-06-10T20:58:33.362214+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-10T20:58:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:34 smithi152 conmon[97365]: audit 2023-06-10T20:58:33.362580+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-10T20:58:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:34 smithi152 conmon[97365]: audit 2023-06-10T20:58:33.363278+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-10T20:58:35.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-06-10T20:58:35.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: cluster 2023-06-10T20:58:34.293840+0000 mgr.y (mgr.14427) 23 : cluster [DBG] pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:35.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: :58:34.372971+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: cluster 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: 2023-06-10T20:58:34.373127+0000 mon.a (mon.0) 599 : 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: audit 2023-06-10T20:58:35.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: 2023-06-10T20:58:34 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: .487540+0000 mon.a (mon.0) 600 : audit 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[107122]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[97224]: cluster 2023-06-10T20:58:34.293840+0000 mgr.y 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[97224]: (mgr.14427) 23 : cluster [DBG] pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[97224]: audit 2023-06-10T20:58:34.372971+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[97224]: cluster 2023-06-10T20:58:34.373127+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-10T20:58:35.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[97224]: audit 2023-06-10T20:58:34.487540+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: cluster 2023-06-10T20:58:34.293840+0000 mgr.y (mgr.14427) 23 : cluster [DBG] 2023-06-10T20:58:35.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: pgmap v14: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:35.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: audit 2023-06-10T20:58:34.372971+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-10T20:58:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: cluster 2023-06-10T20:58:34.373127+0000 mon.a 2023-06-10T20:58:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: (mon.0) 599 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-10T20:58:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: audit 2023-06-10T20:58:34 2023-06-10T20:58:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:35 smithi152 conmon[97365]: .487540+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:35.980 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.678Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.678Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.679Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.679Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=arp 2023-06-10T20:58:35.981 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-10T20:58:35.982 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=edac 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-10T20:58:35.983 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-10T20:58:35.984 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=os 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-10T20:58:35.985 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=stat 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=time 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=uname 2023-06-10T20:58:35.986 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 conmon[135865]: ts=2023-06-10T20:58:35.680Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 bash[135658]: bf0e1033c7ed03b77a062ad9ce4da94a1b033e72352315f9f9d2d6d51b4274f3 2023-06-10T20:58:35.987 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 20:58:35 smithi064 systemd[1]: Started Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:58:36.444 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:36 smithi152 systemd[1]: Starting Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:58:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:36 smithi152 conmon[97365]: cluster 2023-06-10T20:58:35.374013+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-10T20:58:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:36 smithi152 conmon[97365]: audit 2023-06-10T20:58:35.782872+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:36 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:35.783967+0000 mgr.y (mgr.14427) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi152 2023-06-10T20:58:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[107122]: cluster 2023-06-10T20:58:35.374013+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-10T20:58:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[107122]: audit 2023-06-10T20:58:35.782872+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:35.783967+0000 mgr.y (mgr.14427) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi152 2023-06-10T20:58:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[97224]: cluster 2023-06-10T20:58:35.374013+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-10T20:58:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[97224]: audit 2023-06-10T20:58:35.782872+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:36 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:35.783967+0000 mgr.y (mgr.14427) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi152 2023-06-10T20:58:36.944 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:36 smithi152 bash[127146]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2023-06-10T20:58:37.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[97224]: cluster 2023-06-10T20:58:36.294238+0000 mgr.y 2023-06-10T20:58:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[97224]: (mgr.14427) 25 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:58:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[97224]: 06-10T20:58:37.267724+0000 mon.a (mon.0) 603 : audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:58:37] "GET /metrics HTTP/1.1" 200 149597 "" "Prometheus/2.33.4" 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[107122]: cluster 2023-06-10T20:58:36.294238+0000 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[107122]: mgr.y (mgr.14427) 25 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[107122]: audit 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[107122]: 2023-06-10T20:58:37.267724+0000 mon.a (mon.0) 603 : 2023-06-10T20:58:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:37 smithi064 conmon[107122]: audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T20:58:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:37 smithi152 conmon[97365]: cluster 2023-06-10T20:58:36.294238+0000 mgr.y (mgr.14427) 25 : cluster [DBG] pgmap v17: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:37.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:37 smithi152 conmon[97365]: audit 2023-06-10T20:58:37.267724+0000 mon.a (mon.0) 603 : audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T20:58:38.004 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rbd' on pool 'rbd' 2023-06-10T20:58:38.117 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:37 smithi152 bash[127146]: Getting image source signatures 2023-06-10T20:58:38.444 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2023-06-10T20:58:38.444 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2023-06-10T20:58:38.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2023-06-10T20:58:38.739 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-10T20:58:38.740 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-10T20:58:38.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mgr dump --format=json 2023-06-10T20:58:38.944 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2023-06-10T20:58:38.944 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Writing manifest to image destination 2023-06-10T20:58:38.945 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:38 smithi152 bash[127146]: Storing signatures 2023-06-10T20:58:39.004 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[107122]: audit 2023-06-10T20:58:37.997835+0000 mon.a (mon.0) 604 : audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-10T20:58:39.004 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[107122]: cluster 2023-06-10T20:58:37.997916+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-10T20:58:39.105 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:39.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[107122]: cluster 2023-06-10T20:58:38.294596+0000 mgr.y (mgr.14427) 26 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:39.293 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[97224]: audit 2023-06-10T20:58:37.997835+0000 mon.a (mon.0) 604 : audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-10T20:58:39.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[97224]: cluster 2023-06-10T20:58:37.997916+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-10T20:58:39.294 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:39 smithi064 conmon[97224]: cluster 2023-06-10T20:58:38.294596+0000 mgr.y (mgr.14427) 26 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[97365]: audit 2023-06-10T20:58:37.997835+0000 mon.a (mon.0 2023-06-10T20:58:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[97365]: ) 604 : audit [INF] from='client.? 172.21.15.64:0/731402202' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-10T20:58:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[97365]: cluster 2023-06-10T20:58:37.997916+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-10T20:58:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[97365]: cluster 2023-06-10T20:58:38.294596+0000 mgr.y (mgr.14427) 2023-06-10T20:58:39.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[97365]: 26 : cluster [DBG] pgmap v19: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:40.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.967Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-10T20:58:40.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.967Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-10T20:58:40.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-10T20:58:40.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=arp 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-10T20:58:40.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=edac 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-10T20:58:40.447 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-10T20:58:40.448 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=os 2023-06-10T20:58:40.449 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-10T20:58:40.450 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-10T20:58:40.450 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-10T20:58:40.450 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-10T20:58:40.450 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-10T20:58:40.451 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-10T20:58:40.451 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=stat 2023-06-10T20:58:40.451 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-10T20:58:40.451 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-10T20:58:40.451 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=time 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=uname 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-10T20:58:40.452 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-10T20:58:40.453 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.968Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-10T20:58:40.453 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 conmon[127223]: ts=2023-06-10T20:58:39.969Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-10T20:58:40.453 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 bash[127146]: afdc33042e265e15f64b27d5525685d6d170c3ca5de4a07fc8bf795d067b11bf 2023-06-10T20:58:40.453 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 20:58:39 smithi152 systemd[1]: Started Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:58:40.819 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:58:41.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: audit 2023-06-10T20:58:40.049918+0000 mon.a 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: audit 2023-06-10T20:58:40.051184+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: cluster 2023-06-10T20:58:40.294940+0000 mgr.y (mgr. 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: 14427) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: audit 2023-06 2023-06-10T20:58:41.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[107122]: -10T20:58:40.811029+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.64:0/824852569' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[97224]: audit 2023-06-10T20:58:40.049918 2023-06-10T20:58:41.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[97224]: +0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:41.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[97224]: audit 2023-06-10T20:58:40.051184+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[97224]: cluster 2023-06-10T20:58:40.294940+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:41.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:41 smithi064 conmon[97224]: audit 2023-06-10T20:58:40.811029+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.64:0/824852569' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:41 smithi152 conmon[97365]: audit 2023-06-10T20:58:40.049918+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:41 smithi152 conmon[97365]: audit 2023-06-10T20:58:40.051184+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:41 smithi152 conmon[97365]: cluster 2023-06-10T20:58:40.294940+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v20: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:41 smithi152 conmon[97365]: audit 2023-06-10T20:58:40.811029+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.64:0/824852569' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:58:41.498 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":19,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":3618131890},{"type":"v1","addr":"172.21.15.64:6801","nonce":3618131890}]},"active_addr":"172.21.15.64:6801/3618131890","active_change":"2023-06-10T20:58:13.320627+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24323,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1955159746}]},{"addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":703163662}]}]} 2023-06-10T20:58:41.505 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-10T20:58:41.506 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-10T20:58:41.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T20:58:41.899 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:43.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:42 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:58:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit 2023-06-10T20:58:42.062298+0000 mon.a (mon.0) 608 : audit [INF] 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: cluster 2023-06-10T20:58:42.295295+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: -10T20:58:42.418568+0000 mon.a (mon.0) 2023-06-10T20:58:43.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: 609 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: -10T20:58:42.419380+0000 mon.a (mon. 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: 0) 610 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: :58:42.420061+0000 mon.a (mon.0) 611 : 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:43.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: audit 2023-06-10T20:58:42. 2023-06-10T20:58:43.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: 424638+0000 mon.a (mon.0) 612 : audit [INF] 2023-06-10T20:58:43.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: cephadm 2023-06-10T20:58:43.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: 2023-06-10T20:58:42.434888+0000 mgr.y (mgr.14427) 2023-06-10T20:58:43.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:43 smithi152 conmon[97365]: 29 : cephadm [INF] Deploying daemon alertmanager.a on smithi064 2023-06-10T20:58:43.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: audit 2023-06-10T20:58:42.062298 2023-06-10T20:58:43.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: +0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: cluster 2023-06-10T20:58:42.295295+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: audit 2023-06-10T20:58:42.418568+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: audit 2023-06-10T20:58:42.419380+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: audit 2023-06-10T20:58:42.420061+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: audit 2023-06-10T20:58:42.424638+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:42.434888+0000 mgr.y (mgr.14427) 29 : cephadm [INF] Deploying daemon alertmanager.a on smithi064 2023-06-10T20:58:43.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit 2023-06-10T20:58:42.062298+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: cluster 2023-06-10T20:58:42.295295+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v21: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:43.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit 2023-06-10T20:58:42.418568+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit 2023-06-10T20:58:42.419380+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:58:43.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit 2023-06-10T20:58:42.420061+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:58:43.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit 2023-06-10T20:58:42.424638+0000 mon.a (mon.0) 612 : 2023-06-10T20:58:43.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:43.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:43 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:42.434888+0000 mgr.y (mgr.14427) 29 : cephadm [INF] Deploying daemon alertmanager.a on smithi064 2023-06-10T20:58:43.681 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:58:43.681 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:58:37.993304+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:58:44.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:44 smithi064 conmon[97224]: audit 2023-06-10T20:58:43.677294+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.64:0/1086162789' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:44.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:44 smithi064 conmon[107122]: audit 2023-06-10T20:58:43.677294+0000 mon.a (mon.0) 613 : audit [DBG] from='client.? 172.21.15.64:0/1086162789' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:44.365 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-10T20:58:44.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T20:58:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:44 smithi152 conmon[97365]: audit 2023-06-10T20:58:43.677294+0000 mon.a (mon.0) 613 : audit [DBG] 2023-06-10T20:58:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:44 smithi152 conmon[97365]: from='client.? 172.21.15.64:0/1086162789' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:44.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:45.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[107122]: cluster 2023-06-10T20:58:44.295607 2023-06-10T20:58:45.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:45.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[107122]: audit 2023-06-10T20:58:44.495956+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:45.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[97224]: cluster 2023-06-10T20:58:44.295607 2023-06-10T20:58:45.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:45.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:45 smithi064 conmon[97224]: audit 2023-06-10T20:58:44.495956+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:45 smithi152 conmon[97365]: cluster 2023-06-10T20:58:44.295607+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:45 smithi152 conmon[97365]: audit 2023-06-10T20:58:44.495956+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:47.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:47 smithi064 conmon[107122]: cluster 2023-06-10T20:58:46.296176 2023-06-10T20:58:47.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:47 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:47.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:46 smithi064 conmon[97224]: cluster 2023-06-10T20:58:46.296176+0000 mgr.y (mgr.14427) 31 2023-06-10T20:58:47.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:46 smithi064 conmon[97224]: : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:46 smithi152 conmon[97365]: cluster 2023-06-10T20:58:46.296176+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:47.651 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:47 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:58:47] "GET /metrics HTTP/1.1" 200 152007 "" "Prometheus/2.33.4" 2023-06-10T20:58:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:58:48.056 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":55,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:58:37.993304+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:58:48.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:48 smithi064 conmon[97224]: audit 2023-06-10T20:58:48.052308+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.64:0/3619208804' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:48 smithi064 conmon[107122]: audit 2023-06-10T20:58:48.052308+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.64:0/3619208804' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:48 smithi152 conmon[97365]: audit 2023-06-10T20:58:48.052308+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.64:0/3619208804' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:58:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:49 smithi064 conmon[97224]: cluster 2023-06-10T20:58:48.296501+0000 mgr.y (mgr.14427 2023-06-10T20:58:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:49 smithi064 conmon[97224]: ) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:49 smithi064 conmon[107122]: cluster 2023-06-10T20:58:48.296501+0000 2023-06-10T20:58:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:49 smithi064 conmon[107122]: mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v24: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:49 smithi152 conmon[97365]: cluster 2023-06-10T20:58:48.296501+0000 mgr.y (mgr.14427) 32 : cluster [DBG] 2023-06-10T20:58:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:49 smithi152 conmon[97365]: pgmap v24: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:49.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-10T20:58:49.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-10T20:58:49.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-10T20:58:49.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-10T20:58:49.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-10T20:58:49.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-10T20:58:49.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-10T20:58:49.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-10T20:58:51.158 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:50 smithi064 systemd[1]: Starting Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:58:51.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:58:51.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[107122]: :58:50.301324+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:51.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97224]: cluster 2023-06-10T 2023-06-10T20:58:51.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97224]: 20:58:50.301324+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:51 smithi152 conmon[97365]: cluster 2023-06-10T20:58:50.301324+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:51.707 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:51.708 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:51.856 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:51.884 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:51.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE Bus STOPPING 2023-06-10T20:58:51.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T20:58:51.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE Bus STOPPED 2023-06-10T20:58:51.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE Bus STARTING 2023-06-10T20:58:51.909 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.629Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-06-10T20:58:51.910 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.629Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-06-10T20:58:51.910 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.630Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2023-06-10T20:58:51.910 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.632Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-06-10T20:58:51.911 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 bash[139531]: da4a0af553711b8b60756f51df04475b8a5675f77a35f8c69a0fbc63fd055c26 2023-06-10T20:58:51.911 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 systemd[1]: Started Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:58:51.911 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.669Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T20:58:51.911 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.669Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T20:58:51.912 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.672Z caller=main.go:518 msg=Listening address=:9093 2023-06-10T20:58:51.912 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:51.672Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-10T20:58:52.053 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:52.054 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:52.212 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:52.386 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE Serving on http://:::9283 2023-06-10T20:58:52.386 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:51 smithi064 conmon[97677]: [10/Jun/2023:20:58:51] ENGINE Bus STARTED 2023-06-10T20:58:52.671 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:58:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.703353 2023-06-10T20:58:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: +0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.826980+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.833048+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.835930+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.836319+0000 mgr.y (mgr.14427) 34 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: audit 2023-06-10T20:58:51.843560+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[107122]: cephadm 2023-06-10T20:58:51.849592+0000 mgr.y (mgr.14427) 35 : cephadm [INF] Deploying daemon grafana.a on smithi152 2023-06-10T20:58:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.703353+0000 mon.a ( 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: mon.0) 616 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.826980+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.833048+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.835930+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.836319+0000 mgr.y (mgr.14427) 34 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: audit 2023-06-10T20:58:51.843560+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:52 smithi064 conmon[97224]: cephadm 2023-06-10T20:58:51.849592+0000 mgr.y (mgr.14427) 35 : cephadm [INF] Deploying daemon grafana.a on smithi152 2023-06-10T20:58:53.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:58:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:58:53.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06-10T20:58:51.703353+0000 mon.a (mon.0) 616 : 2023-06-10T20:58:53.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06-10T20:58:51.826980+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06-10T20:58:53.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 2023-06-10T20:58:51.833048+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06 2023-06-10T20:58:53.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: -10T20:58:51.835930+0000 mon.a ( 2023-06-10T20:58:53.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 2023-06-10T20: 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 58:51.836319+0000 mgr.y (mgr.14427) 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 34 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 20:58:51.843560+0000 mon.a (mon. 2023-06-10T20:58:53.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: 0) 620 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:53.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: cephadm 2023-06 2023-06-10T20:58:53.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: -10T20:58:51.849592+0000 mgr.y 2023-06-10T20:58:53.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:52 smithi152 conmon[97365]: (mgr.14427) 35 : cephadm [INF] Deploying daemon grafana.a on smithi152 2023-06-10T20:58:53.826 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:58:53 smithi064 conmon[139765]: level=info ts=2023-06-10T20:58:53.633Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000838033s 2023-06-10T20:58:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:53 smithi064 conmon[97224]: cluster 2023-06-10T20:58:52.301765+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:54.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:53 smithi064 conmon[107122]: cluster 2023-06-10T20:58:52.301765+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:53 smithi152 conmon[97365]: cluster 2023-06-10T20:58:52.301765+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:53 smithi152 conmon[97365]: 2023-06-10T20:58:55.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:55 smithi064 conmon[107122]: cluster 2023-06-10T20:58:54.302121+0000 mgr.y (mgr.14427) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:55.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:55 smithi064 conmon[107122]: audit 2023-06-10T20:58:54.502494+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:55.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:55 smithi064 conmon[97224]: cluster 2023-06-10T20:58:54.302121+0000 mgr.y (mgr.14427) 37 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:55.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:55 smithi064 conmon[97224]: audit 2023-06-10T20:58:54.502494+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:55 smithi152 conmon[97365]: cluster 2023-06-10T20:58:54.302121+0000 mgr.y (mgr.14427) 37 : cluster [DBG] 2023-06-10T20:58:55.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:55 smithi152 conmon[97365]: pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:55.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:55 smithi152 conmon[97365]: audit 2023-06-10T20:58:54.502494+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:58:57.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:57 smithi064 conmon[107122]: cluster 2023-06-10T20:58:56.302461+0000 mgr.y (mgr.14427) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:57.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:57 smithi064 conmon[97224]: cluster 2023-06-10T20:58:56.302461+0000 mgr.y 2023-06-10T20:58:57.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:57 smithi064 conmon[97224]: (mgr.14427) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:57 smithi152 conmon[97365]: cluster 2023-06-10T20:58:56.302461+0000 mgr.y (mgr.14427) 38 : cluster 2023-06-10T20:58:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:57 smithi152 conmon[97365]: [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:57.568 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:58:57 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:58:57] "GET /metrics HTTP/1.1" 200 152007 "" "Prometheus/2.33.4" 2023-06-10T20:58:58.061 INFO:teuthology.orchestra.run.smithi064.stdout:206158430222 2023-06-10T20:58:58.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-10T20:58:58.777 INFO:teuthology.orchestra.run.smithi064.stdout:34359738412 2023-06-10T20:58:58.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-10T20:58:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:58:59 smithi064 conmon[107122]: cluster 2023-06-10T20:58:58.302853+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:59.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:58:59 smithi064 conmon[97224]: cluster 2023-06-10T20:58:58.302853+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:58:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:58:59 smithi152 conmon[97365]: cluster 2023-06-10T20:58:58.302853+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:01.173 INFO:teuthology.orchestra.run.smithi064.stdout:128849018906 2023-06-10T20:59:01.174 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-10T20:59:01.207 INFO:teuthology.orchestra.run.smithi064.stdout:107374182431 2023-06-10T20:59:01.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-10T20:59:01.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:01 smithi064 conmon[107122]: cluster 2023-06-10T20:59:00.303424+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:01 smithi064 conmon[97224]: cluster 2023-06-10T20:59:00.303424+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:01.652 INFO:teuthology.orchestra.run.smithi064.stdout:154618822678 2023-06-10T20:59:01.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-10T20:59:01.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:01 smithi152 conmon[97365]: cluster 2023-06-10T20:59:00.303424+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:01.908 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 20:59:01 smithi064 conmon[139765]: level=info ts=2023-06-10T20:59:01.636Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003965292s 2023-06-10T20:59:02.266 INFO:teuthology.orchestra.run.smithi064.stdout:55834574889 2023-06-10T20:59:02.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-10T20:59:02.299 INFO:teuthology.orchestra.run.smithi064.stdout:77309411364 2023-06-10T20:59:02.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-10T20:59:02.553 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:02.703 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:02.703 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:02.708 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:02.744 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:03.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:02 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:03.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:03 smithi064 conmon[107122]: cluster 2023-06-10T20:59:02.303725+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:03 smithi064 conmon[97224]: cluster 2023-06-10T20:59:02.303725+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:03.537 INFO:teuthology.orchestra.run.smithi064.stdout:180388626451 2023-06-10T20:59:03.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-10T20:59:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:03 smithi152 conmon[97365]: cluster 2023-06-10T20:59:02.303725+0000 mgr.y (mgr.14427) 2023-06-10T20:59:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:03 smithi152 conmon[97365]: 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:05.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:05 smithi064 conmon[97224]: cluster 2023-06-10T20:59:04.304013+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:05.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:05 smithi064 conmon[107122]: cluster 2023-06-10T20:59:04.304013+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:05 smithi152 conmon[97365]: cluster 2023-06-10T20:59:04.304013+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:06.437 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:06.446 INFO:teuthology.orchestra.run.smithi064.stdout:206158430223 2023-06-10T20:59:06.549 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:06.704 INFO:teuthology.orchestra.run.smithi064.stdout:154618822679 2023-06-10T20:59:07.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[107122]: cluster 2023-06-10T20:59:06.304416+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:07.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[107122]: audit 2023-06-10T20:59:06.442979+0000 mon.a (mon.0) 622 : audit [DBG] from='client.? 172.21.15.64:0/2684180969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T20:59:07.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[107122]: audit 2023-06-10T20:59:06.701301+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.64:0/1440586216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:07.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[97224]: cluster 2023-06-10T20:59:06.304416+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:07.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[97224]: audit 2023-06-10T20:59:06.442979+0000 mon.a (mon.0) 622 : audit [DBG] from='client.? 172.21.15.64:0/2684180969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T20:59:07.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[97224]: audit 2023-06-10T20:59:06.701301+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.64:0/1440586216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:07 smithi152 conmon[97365]: cluster 2023-06-10T20:59:06.304416+0000 mgr.y (mgr.14427) 43 : 2023-06-10T20:59:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:07 smithi152 conmon[97365]: cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:07 smithi152 conmon[97365]: audit 2023-06-10T20:59:06.442979+0000 mon.a (mon.0) 622 : audit [DBG] from='client.? 172.21.15.64:0/2684180969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T20:59:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:07 smithi152 conmon[97365]: audit 2023-06-10T20:59:06.701301+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.64:0/1440586216' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:07.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:07 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:07] "GET /metrics HTTP/1.1" 200 152006 "" "Prometheus/2.33.4" 2023-06-10T20:59:07.744 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430223 for osd.7 2023-06-10T20:59:07.745 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:08.612 INFO:teuthology.orchestra.run.smithi064.stdout:34359738414 2023-06-10T20:59:08.733 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822678 got 154618822679 for osd.5 2023-06-10T20:59:08.734 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:09.474 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:09.590 INFO:teuthology.orchestra.run.smithi064.stdout:107374182433 2023-06-10T20:59:09.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[107122]: cluster 2023-06-10T20: 2023-06-10T20:59:09.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[107122]: 59:08.304763+0000 mgr.y (mgr.14427) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:09.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[107122]: audit 2023-06-10T20:59:08.609408 2023-06-10T20:59:09.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[107122]: +0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.64:0/341268639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:09.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[97224]: cluster 2023-06-10T20:59:08.304763+0000 mgr.y (mgr.14427) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:09.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:09 smithi064 conmon[97224]: audit 2023-06-10T20:59:08.609408+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.64:0/341268639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:09.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:09 smithi152 conmon[97365]: cluster 2023-06-10T20:59:08.304763+0000 mgr.y (mgr.14427) 44 2023-06-10T20:59:09.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:09 smithi152 conmon[97365]: : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:09 smithi152 conmon[97365]: audit 2023-06-10T20:59:08.609408+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.64:0/341268639' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:10.393 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738412 got 34359738414 for osd.0 2023-06-10T20:59:10.393 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:10 smithi064 conmon[107122]: audit 2023-06-10T20:59:09. 2023-06-10T20:59:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:10 smithi064 conmon[107122]: 587629+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.64:0/2558341357' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T20:59:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:10 smithi064 conmon[97224]: audit 2023-06-10T20:59:09.587629+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.64:0/2558341357' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T20:59:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:10 smithi152 conmon[97365]: audit 2023-06-10T20:59:09.587629+0000 mon.c (mon.2) 2023-06-10T20:59:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:10 smithi152 conmon[97365]: 18 : audit [DBG] from='client.? 172.21.15.64:0/2558341357' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T20:59:10.834 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182431 got 107374182433 for osd.3 2023-06-10T20:59:10.834 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:10.860 INFO:teuthology.orchestra.run.smithi064.stdout:128849018909 2023-06-10T20:59:11.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:11 smithi064 conmon[97224]: cluster 2023-06-10T20:59:10.305388+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:11.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:11 smithi064 conmon[97224]: audit 2023-06-10T20:59:10.857623+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.64:0/1742529253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T20:59:11.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:11 smithi064 conmon[107122]: cluster 2023-06-10T20:59:10.305388+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:11.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:11 smithi064 conmon[107122]: audit 2023-06-10T20:59:10.857623+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.64:0/1742529253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T20:59:11.455 INFO:teuthology.orchestra.run.smithi064.stdout:55834574891 2023-06-10T20:59:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:11 smithi152 conmon[97365]: cluster 2023-06-10T20:59:10.305388+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:11 smithi152 conmon[97365]: audit 2023-06-10T20:59:10 2023-06-10T20:59:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:11 smithi152 conmon[97365]: .857623+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.64:0/1742529253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T20:59:12.014 INFO:teuthology.orchestra.run.smithi064.stdout:77309411367 2023-06-10T20:59:12.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:12 smithi064 conmon[107122]: audit 2023-06-10T20:59:11.452958+0000 mon.c ( 2023-06-10T20:59:12.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:12 smithi064 conmon[107122]: mon.2) 20 : audit [DBG] from='client.? 172.21.15.64:0/738123593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:12.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:12 smithi064 conmon[107122]: audit 2023-06-10T20:59:12.010952+0000 mon.a (mon.0) 623 : audit [DBG] from='client.? 172.21.15.64:0/1094145298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T20:59:12.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:12 smithi064 conmon[97224]: audit 2023-06-10T20:59:11.452958+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.64:0/738123593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:12.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:12 smithi064 conmon[97224]: audit 2023-06-10T20:59:12.010952+0000 mon.a (mon.0) 623 : audit [DBG] from='client.? 172.21.15.64:0/1094145298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T20:59:12.499 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018906 got 128849018909 for osd.4 2023-06-10T20:59:12.499 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:12 smithi152 conmon[97365]: audit 2023-06-10T20:59:11.452958+0000 mon.c (mon.2) 2023-06-10T20:59:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:12 smithi152 conmon[97365]: 20 : audit [DBG] from='client.? 172.21.15.64:0/738123593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:12 smithi152 conmon[97365]: audit 2023-06-10T20:59:12.010952+0000 mon.a (mon.0) 623 : audit [DBG] from='client.? 172.21.15.64:0/1094145298' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T20:59:13.025 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411367 for osd.2 2023-06-10T20:59:13.025 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:13.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:12 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:13.573 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574891 for osd.1 2023-06-10T20:59:13.574 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:13.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:13 smithi064 conmon[107122]: cluster 2023-06-10T20:59:12.305671+0000 2023-06-10T20:59:13.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:13 smithi064 conmon[107122]: mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:13.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:13 smithi064 conmon[97224]: cluster 2023-06-10T20:59:12.305671+0000 mgr.y ( 2023-06-10T20:59:13.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:13 smithi064 conmon[97224]: mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:13 smithi152 conmon[97365]: cluster 2023-06-10T20:59:12.305671+0000 2023-06-10T20:59:13.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:13 smithi152 conmon[97365]: mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:14.011 INFO:teuthology.orchestra.run.smithi064.stdout:180388626453 2023-06-10T20:59:14.585 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626451 got 180388626453 for osd.6 2023-06-10T20:59:14.585 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:14.586 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-10T20:59:14.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T20:59:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:14 smithi064 conmon[107122]: audit 2023-06-10T20:59:14.008428 2023-06-10T20:59:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:14 smithi064 conmon[107122]: +0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.64:0/2478710495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:14 smithi064 conmon[97224]: audit 2023-06-10T20:59:14.008428+0000 mon.c (mon.2) 21 : 2023-06-10T20:59:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:14 smithi064 conmon[97224]: audit [DBG] from='client.? 172.21.15.64:0/2478710495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:14 smithi152 conmon[97365]: audit 2023-06-10T20:59:14.008428+0000 mon.c (mon.2) 21 : audit [DBG] 2023-06-10T20:59:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:14 smithi152 conmon[97365]: from='client.? 172.21.15.64:0/2478710495' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:14.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:15.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:15 smithi064 conmon[97224]: cluster 2023-06-10T20:59:14.306153+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:15 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:59:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:15 smithi064 conmon[107122]: :59:14.306153+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:15 smithi152 conmon[97365]: cluster 2023-06-10T20:59:14.306153+0000 mgr.y (mgr.14427) 2023-06-10T20:59:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:15 smithi152 conmon[97365]: 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:16.523 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:16.524 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T20:59:17.069 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2023-06-10T20:59:16.306340+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44144,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":39872,"kb_avail":749917072,"statfs":{"total":767960285184,"available":767915081728,"internally_reserved":0,"allocated":4308992,"data_stored":1762465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40828928},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658722+0000","last_change":"2023-06-10T20:58:34.376483+0000","last_active":"2023-06-10T20:58:35.658722+0000","last_peered":"2023-06-10T20:58:35.658722+0000","last_clean":"2023-06-10T20:58:35.658722+0000","last_became_active":"2023-06-10T20:58:34.376121+0000","last_became_peered":"2023-06-10T20:58:34.376121+0000","last_unstale":"2023-06-10T20:58:35.658722+0000","last_undegraded":"2023-06-10T20:58:35.658722+0000","last_fullsized":"2023-06-10T20:58:35.658722+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658435+0000","last_change":"2023-06-10T20:58:34.481072+0000","last_active":"2023-06-10T20:58:35.658435+0000","last_peered":"2023-06-10T20:58:35.658435+0000","last_clean":"2023-06-10T20:58:35.658435+0000","last_became_active":"2023-06-10T20:58:34.480835+0000","last_became_peered":"2023-06-10T20:58:34.480835+0000","last_unstale":"2023-06-10T20:58:35.658435+0000","last_undegraded":"2023-06-10T20:58:35.658435+0000","last_fullsized":"2023-06-10T20:58:35.658435+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658754+0000","last_change":"2023-06-10T20:58:35.246897+0000","last_active":"2023-06-10T20:58:35.658754+0000","last_peered":"2023-06-10T20:58:35.658754+0000","last_clean":"2023-06-10T20:58:35.658754+0000","last_became_active":"2023-06-10T20:58:35.246501+0000","last_became_peered":"2023-06-10T20:58:35.246501+0000","last_unstale":"2023-06-10T20:58:35.658754+0000","last_undegraded":"2023-06-10T20:58:35.658754+0000","last_fullsized":"2023-06-10T20:58:35.658754+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658576+0000","last_change":"2023-06-10T20:58:35.246773+0000","last_active":"2023-06-10T20:58:35.658576+0000","last_peered":"2023-06-10T20:58:35.658576+0000","last_clean":"2023-06-10T20:58:35.658576+0000","last_became_active":"2023-06-10T20:58:35.246379+0000","last_became_peered":"2023-06-10T20:58:35.246379+0000","last_unstale":"2023-06-10T20:58:35.658576+0000","last_undegraded":"2023-06-10T20:58:35.658576+0000","last_fullsized":"2023-06-10T20:58:35.658576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658655+0000","last_change":"2023-06-10T20:57:57.149158+0000","last_active":"2023-06-10T20:58:35.658655+0000","last_peered":"2023-06-10T20:58:35.658655+0000","last_clean":"2023-06-10T20:58:35.658655+0000","last_became_active":"2023-06-10T20:57:57.148833+0000","last_became_peered":"2023-06-10T20:57:57.148833+0000","last_unstale":"2023-06-10T20:58:35.658655+0000","last_undegraded":"2023-06-10T20:58:35.658655+0000","last_fullsized":"2023-06-10T20:58:35.658655+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658537+0000","last_change":"2023-06-10T20:58:34.376607+0000","last_active":"2023-06-10T20:58:35.658537+0000","last_peered":"2023-06-10T20:58:35.658537+0000","last_clean":"2023-06-10T20:58:35.658537+0000","last_became_active":"2023-06-10T20:58:34.376302+0000","last_became_peered":"2023-06-10T20:58:34.376302+0000","last_unstale":"2023-06-10T20:58:35.658537+0000","last_undegraded":"2023-06-10T20:58:35.658537+0000","last_fullsized":"2023-06-10T20:58:35.658537+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.372845+0000","last_change":"2023-06-10T20:58:35.246831+0000","last_active":"2023-06-10T20:58:35.372845+0000","last_peered":"2023-06-10T20:58:35.372845+0000","last_clean":"2023-06-10T20:58:35.372845+0000","last_became_active":"2023-06-10T20:58:35.246208+0000","last_became_peered":"2023-06-10T20:58:35.246208+0000","last_unstale":"2023-06-10T20:58:35.372845+0000","last_undegraded":"2023-06-10T20:58:35.372845+0000","last_fullsized":"2023-06-10T20:58:35.372845+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658538+0000","last_change":"2023-06-10T20:58:34.376802+0000","last_active":"2023-06-10T20:58:35.658538+0000","last_peered":"2023-06-10T20:58:35.658538+0000","last_clean":"2023-06-10T20:58:35.658538+0000","last_became_active":"2023-06-10T20:58:34.376449+0000","last_became_peered":"2023-06-10T20:58:34.376449+0000","last_unstale":"2023-06-10T20:58:35.658538+0000","last_undegraded":"2023-06-10T20:58:35.658538+0000","last_fullsized":"2023-06-10T20:58:35.658538+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658398+0000","last_change":"2023-06-10T20:58:35.246694+0000","last_active":"2023-06-10T20:58:35.658398+0000","last_peered":"2023-06-10T20:58:35.658398+0000","last_clean":"2023-06-10T20:58:35.658398+0000","last_became_active":"2023-06-10T20:58:35.246275+0000","last_became_peered":"2023-06-10T20:58:35.246275+0000","last_unstale":"2023-06-10T20:58:35.658398+0000","last_undegraded":"2023-06-10T20:58:35.658398+0000","last_fullsized":"2023-06-10T20:58:35.658398+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5336,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.48499999999999999}]},{"osd":2,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.502},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.443},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52300000000000002}]},{"osd":4,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.41699999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626454,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5396,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739756,"statfs":{"total":95995035648,"available":95989510144,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.63300000000000001}]},{"osd":2,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.42299999999999999}]},{"osd":7,"last update":"Sat Jun 10 20:58:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57199999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.33900000000000002,"15min":0.316},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.32900000000000001},"min":{"1min":0.27600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.311}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53700000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44400000000000001,"15min":0.45000000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.56499999999999995,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.50700000000000001,"15min":0.51500000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Jun 10 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.55100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738416,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.36799999999999999},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.54500000000000004,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.373},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40400000000000003,"15min":0.39000000000000001},"min":{"1min":0.255,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Jun 10 20:58:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.46000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63400000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.624,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.46100000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.49199999999999999,"15min":0.502},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67900000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55800000000000005,"15min":0.56899999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77000000000000002,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68999999999999995,"15min":0.68799999999999994},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.67800000000000005,"15min":0.67600000000000005},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":0.64800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5588,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739564,"statfs":{"total":95995035648,"available":95989313536,"internally_reserved":0,"allocated":536576,"data_stored":219833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.39200000000000002,"15min":0.373},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.31},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Jun 10 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.50600000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Jun 10 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.56899999999999995,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.61299999999999999,"15min":0.60799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Jun 10 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.443}]},{"osd":6,"last update":"Sat Jun 10 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.46400000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39200000000000002,"15min":0.39800000000000002},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.248},"max":{"1min":0.56200000000000006,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.54500000000000004,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40500000000000003,"15min":0.40999999999999998},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.51800000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.47699999999999998,"15min":0.48299999999999998},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51100000000000001,"15min":0.51600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.31900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":0.622,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Jun 10 20:59:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81499999999999995,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Jun 10 20:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.52600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5524,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739628,"statfs":{"total":95995035648,"available":95989379072,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54900000000000004,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.40300000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.37},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.63}]},{"osd":2,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55200000000000005,"15min":0.55100000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.433}]},{"osd":5,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Jun 10 20:58:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.46600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822682,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5460,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739692,"statfs":{"total":95995035648,"available":95989444608,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.628}]},{"osd":1,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.505}]},{"osd":2,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.63700000000000001}]},{"osd":3,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T20:59:17.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T20:59:17.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[107122]: cluster 2023-06-10T20:59:16.306623+0000 2023-06-10T20:59:17.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[107122]: mgr.y (mgr.14427) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:17.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[107122]: audit 2023-06-10T20:59:16.516103+0000 mgr.y (mgr.14427) 49 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:17.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[97224]: cluster 2023-06-10T20:59:16.306623+0000 2023-06-10T20:59:17.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[97224]: mgr.y (mgr.14427) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:17.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[97224]: audit 2023-06-10T20:59:16.516103+0000 mgr.y (mgr.14427) 49 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:17.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:17 smithi152 conmon[97365]: cluster 2023-06-10T20:59:16.306623+0000 mgr.y (mgr.14427) 2023-06-10T20:59:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:17 smithi152 conmon[97365]: 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:17 smithi152 conmon[97365]: audit 2023-06-10T20:59:16.516103+0000 mgr.y (mgr.14427) 49 : audit [DBG] from='client.24500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:17.595 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:17 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:17] "GET /metrics HTTP/1.1" 200 152007 "" "Prometheus/2.33.4" 2023-06-10T20:59:18.993 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:18.994 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T20:59:19.595 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2023-06-10T20:59:18.306856+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44144,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":39872,"kb_avail":749917072,"statfs":{"total":767960285184,"available":767915081728,"internally_reserved":0,"allocated":4308992,"data_stored":1762465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40828928},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658722+0000","last_change":"2023-06-10T20:58:34.376483+0000","last_active":"2023-06-10T20:58:35.658722+0000","last_peered":"2023-06-10T20:58:35.658722+0000","last_clean":"2023-06-10T20:58:35.658722+0000","last_became_active":"2023-06-10T20:58:34.376121+0000","last_became_peered":"2023-06-10T20:58:34.376121+0000","last_unstale":"2023-06-10T20:58:35.658722+0000","last_undegraded":"2023-06-10T20:58:35.658722+0000","last_fullsized":"2023-06-10T20:58:35.658722+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658435+0000","last_change":"2023-06-10T20:58:34.481072+0000","last_active":"2023-06-10T20:58:35.658435+0000","last_peered":"2023-06-10T20:58:35.658435+0000","last_clean":"2023-06-10T20:58:35.658435+0000","last_became_active":"2023-06-10T20:58:34.480835+0000","last_became_peered":"2023-06-10T20:58:34.480835+0000","last_unstale":"2023-06-10T20:58:35.658435+0000","last_undegraded":"2023-06-10T20:58:35.658435+0000","last_fullsized":"2023-06-10T20:58:35.658435+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658754+0000","last_change":"2023-06-10T20:58:35.246897+0000","last_active":"2023-06-10T20:58:35.658754+0000","last_peered":"2023-06-10T20:58:35.658754+0000","last_clean":"2023-06-10T20:58:35.658754+0000","last_became_active":"2023-06-10T20:58:35.246501+0000","last_became_peered":"2023-06-10T20:58:35.246501+0000","last_unstale":"2023-06-10T20:58:35.658754+0000","last_undegraded":"2023-06-10T20:58:35.658754+0000","last_fullsized":"2023-06-10T20:58:35.658754+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658576+0000","last_change":"2023-06-10T20:58:35.246773+0000","last_active":"2023-06-10T20:58:35.658576+0000","last_peered":"2023-06-10T20:58:35.658576+0000","last_clean":"2023-06-10T20:58:35.658576+0000","last_became_active":"2023-06-10T20:58:35.246379+0000","last_became_peered":"2023-06-10T20:58:35.246379+0000","last_unstale":"2023-06-10T20:58:35.658576+0000","last_undegraded":"2023-06-10T20:58:35.658576+0000","last_fullsized":"2023-06-10T20:58:35.658576+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658655+0000","last_change":"2023-06-10T20:57:57.149158+0000","last_active":"2023-06-10T20:58:35.658655+0000","last_peered":"2023-06-10T20:58:35.658655+0000","last_clean":"2023-06-10T20:58:35.658655+0000","last_became_active":"2023-06-10T20:57:57.148833+0000","last_became_peered":"2023-06-10T20:57:57.148833+0000","last_unstale":"2023-06-10T20:58:35.658655+0000","last_undegraded":"2023-06-10T20:58:35.658655+0000","last_fullsized":"2023-06-10T20:58:35.658655+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658537+0000","last_change":"2023-06-10T20:58:34.376607+0000","last_active":"2023-06-10T20:58:35.658537+0000","last_peered":"2023-06-10T20:58:35.658537+0000","last_clean":"2023-06-10T20:58:35.658537+0000","last_became_active":"2023-06-10T20:58:34.376302+0000","last_became_peered":"2023-06-10T20:58:34.376302+0000","last_unstale":"2023-06-10T20:58:35.658537+0000","last_undegraded":"2023-06-10T20:58:35.658537+0000","last_fullsized":"2023-06-10T20:58:35.658537+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.372845+0000","last_change":"2023-06-10T20:58:35.246831+0000","last_active":"2023-06-10T20:58:35.372845+0000","last_peered":"2023-06-10T20:58:35.372845+0000","last_clean":"2023-06-10T20:58:35.372845+0000","last_became_active":"2023-06-10T20:58:35.246208+0000","last_became_peered":"2023-06-10T20:58:35.246208+0000","last_unstale":"2023-06-10T20:58:35.372845+0000","last_undegraded":"2023-06-10T20:58:35.372845+0000","last_fullsized":"2023-06-10T20:58:35.372845+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658538+0000","last_change":"2023-06-10T20:58:34.376802+0000","last_active":"2023-06-10T20:58:35.658538+0000","last_peered":"2023-06-10T20:58:35.658538+0000","last_clean":"2023-06-10T20:58:35.658538+0000","last_became_active":"2023-06-10T20:58:34.376449+0000","last_became_peered":"2023-06-10T20:58:34.376449+0000","last_unstale":"2023-06-10T20:58:35.658538+0000","last_undegraded":"2023-06-10T20:58:35.658538+0000","last_fullsized":"2023-06-10T20:58:35.658538+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-10T20:58:35.658398+0000","last_change":"2023-06-10T20:58:35.246694+0000","last_active":"2023-06-10T20:58:35.658398+0000","last_peered":"2023-06-10T20:58:35.658398+0000","last_clean":"2023-06-10T20:58:35.658398+0000","last_became_active":"2023-06-10T20:58:35.246275+0000","last_became_peered":"2023-06-10T20:58:35.246275+0000","last_unstale":"2023-06-10T20:58:35.658398+0000","last_undegraded":"2023-06-10T20:58:35.658398+0000","last_fullsized":"2023-06-10T20:58:35.658398+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430226,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5336,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.48499999999999999}]},{"osd":2,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.502},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.443},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.52300000000000002}]},{"osd":4,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.41699999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626454,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5396,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739756,"statfs":{"total":95995035648,"available":95989510144,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.63300000000000001}]},{"osd":2,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.42299999999999999}]},{"osd":7,"last update":"Sat Jun 10 20:58:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57199999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:50 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.33900000000000002,"15min":0.316},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.32900000000000001},"min":{"1min":0.27600000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.311}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53700000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.26800000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.44400000000000001,"15min":0.45000000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.56499999999999995,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.50700000000000001,"15min":0.51500000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Jun 10 20:58:18 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.55100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738416,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.36799999999999999},"min":{"1min":0.23899999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.54500000000000004,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.373},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40400000000000003,"15min":0.39000000000000001},"min":{"1min":0.255,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Jun 10 20:58:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.45600000000000002,"15min":0.46000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63400000000000001,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.624,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.46100000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.49199999999999999,"15min":0.502},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67900000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.55800000000000005,"15min":0.56899999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.77000000000000002,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68999999999999995,"15min":0.68799999999999994},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.67800000000000005,"15min":0.67600000000000005},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":0.64800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5588,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739564,"statfs":{"total":95995035648,"available":95989313536,"internally_reserved":0,"allocated":536576,"data_stored":219833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.39200000000000002,"15min":0.373},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.31},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Jun 10 20:58:42 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.50600000000000001,"15min":0.51000000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Jun 10 20:58:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.56899999999999995,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.61299999999999999,"15min":0.60799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.27,"5min":1.27,"15min":1.27},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Jun 10 20:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.443}]},{"osd":6,"last update":"Sat Jun 10 20:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.46400000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":0.39200000000000002,"15min":0.39800000000000002},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.248},"max":{"1min":0.56200000000000006,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.38,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.54500000000000004,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40500000000000003,"15min":0.40999999999999998},"min":{"1min":0.27800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.51800000000000002,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.47699999999999998,"15min":0.48299999999999998},"min":{"1min":0.34000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:58:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51100000000000001,"15min":0.51600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.31900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":0.622,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Jun 10 20:59:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81499999999999995,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Jun 10 20:58:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Jun 10 20:58:37 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Jun 10 20:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.52600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5524,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739628,"statfs":{"total":95995035648,"available":95989379072,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54900000000000004,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.40300000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.37},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.63}]},{"osd":2,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55200000000000005,"15min":0.55100000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Jun 10 20:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.433}]},{"osd":5,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Jun 10 20:58:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.46600000000000003}]}]},{"osd":5,"up_from":36,"seq":154618822682,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5460,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739692,"statfs":{"total":95995035648,"available":95989444608,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.628}]},{"osd":1,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.505}]},{"osd":2,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.63700000000000001}]},{"osd":3,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Jun 10 20:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Jun 10 20:58:36 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Jun 10 20:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57899999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T20:59:19.598 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-10T20:59:19.598 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-10T20:59:19.598 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-10T20:59:19.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph health --format=json 2023-06-10T20:59:19.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:19 smithi064 conmon[107122]: cluster 2023-06-10T20:59:18.307090+0000 mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:19.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:19 smithi064 conmon[97224]: cluster 2023-06-10T20:59:18.307090+0000 mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:19 smithi152 conmon[97365]: cluster 2023-06-10T20:59:18.307090+0000 2023-06-10T20:59:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:19 smithi152 conmon[97365]: mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:19.961 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:20.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:20 smithi064 conmon[107122]: audit 2023-06-10T20:59:18.986401+0000 mgr.y (mgr.14427) 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:20 smithi064 conmon[97224]: audit 2023-06-10T20:59:18.986401+0000 mgr.y (mgr.14427) 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:20 smithi152 conmon[97365]: audit 2023-06-10T20:59:18.986401+0000 mgr.y (mgr.14427) 2023-06-10T20:59:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:20 smithi152 conmon[97365]: 51 : audit [DBG] from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T20:59:21.491 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:21.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:21 smithi064 conmon[97224]: cluster 2023-06-10T20:59:20.307385+0000 mgr.y (mgr.14427) 52 : 2023-06-10T20:59:21.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:21 smithi064 conmon[97224]: cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:21 smithi064 conmon[107122]: cluster 2023-06-10T20:59: 2023-06-10T20:59:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:21 smithi064 conmon[107122]: 20.307385+0000 mgr.y (mgr.14427) 52 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:21 smithi152 conmon[97365]: cluster 2023-06-10T20:59:20.307385+0000 mgr.y (mgr.14427) 52 2023-06-10T20:59:21.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:21 smithi152 conmon[97365]: : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:22.102 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-10T20:59:22.102 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-10T20:59:22.103 INFO:tasks.cephadm:Setup complete, yielding 2023-06-10T20:59:22.103 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-06-10T20:59:22.113 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2023-06-10T20:59:22.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- bash -c 'ceph fs volume create foo' 2023-06-10T20:59:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:22 smithi064 conmon[97224]: audit 2023-06-10T20:59:21.487823+0000 mon.a (mon. 2023-06-10T20:59:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:22 smithi064 conmon[97224]: 0) 624 : audit [DBG] from='client.? 172.21.15.64:0/57261325' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T20:59:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:22 smithi064 conmon[107122]: audit 2023-06-10T20:59:21. 2023-06-10T20:59:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:22 smithi064 conmon[107122]: 487823+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.64:0/57261325' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T20:59:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:22 smithi152 conmon[97365]: audit 2023-06-10T20:59:21.487823+0000 mon.a (mon.0) 2023-06-10T20:59:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:22 smithi152 conmon[97365]: 624 : audit [DBG] from='client.? 172.21.15.64:0/57261325' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T20:59:23.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:22 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:23.511 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:23 smithi064 conmon[97224]: cluster 2023-06-10T20:59:22.307705+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:23.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:23 smithi064 conmon[107122]: cluster 2023-06-10T20:59:22.307705+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:23 smithi152 conmon[97365]: cluster 2023-06-10T20:59:22.307705+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:24.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[97224]: audit 2023-06-10T20:59:23.866529+0000 mgr.y (mgr. 2023-06-10T20:59:24.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[97224]: 14427) 54 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:59:24.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[97224]: audit 2023-06-10T20:59:23.867045+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-10T20:59:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[107122]: audit 2023-06-10T20:59:23.866529+0000 2023-06-10T20:59:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[107122]: mgr.y (mgr.14427) 54 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:59:24.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:59:24.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:24 smithi064 conmon[107122]: 20:59:23.867045+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-10T20:59:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:24 smithi152 conmon[97365]: audit 2023-06-10T20:59:23.866529+0000 mgr.y (mgr.14427) 54 : audit [DBG] from='client.14610 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T20:59:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:24 smithi152 conmon[97365]: audit 2023-06-10T20:59:23.867045+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-10T20:59:25.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[107122]: cluster 2023-06-10T20:59:24.308035+0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:25.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[107122]: audit 2023-06-10T20:59:24.391760+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[107122]: cluster 2023-06-10T20:59:24.391839+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[107122]: audit 2023-06-10T20:59:24.394552+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: cluster 2023-06-10T20:59:24.308035 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: audit 2023-06-10T20:59:24.391760+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-10T20:59:25.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: cluster 2023-06-10T20:59: 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: 24.391839+0000 mon.a (mon.0) 627 : cluster [DBG] 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: osdmap e56: 8 total, 8 up, 8 in 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: 20:59:24.394552+0000 mon.a (mon.0) 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: 628 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:25.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:25 smithi064 conmon[97224]: debug 2023-06-10T20:59:25.412+0000 7f7c986e1700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-10T20:59:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:25 smithi152 conmon[97365]: cluster 2023-06-10T20:59:24.308035+0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:25 smithi152 conmon[97365]: audit 2023-06-10T20:59:24.391760+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-10T20:59:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:25 smithi152 conmon[97365]: cluster 2023-06-10T20:59:24.391839+0000 mon.a (mon.0) 627 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-06-10T20:59:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:25 smithi152 conmon[97365]: audit 2023-06-10T20:59:24.394552+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:26.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-06-10T20:59:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: audit 2023-06-10T20:59:25.396968+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cluster 2023-06-10T20:59:25.397125+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: audit 2023-06-10T20:59:25.412625+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 20:59:25.413470+0000 mon.a (mon.0) 632 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cluster 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 2023-06-10T20:59:25.413504+0000 mon.a (mon.0) 633 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-10T20:59:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cluster 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 2023-06-10T20:59:25.421425+0000 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: mon.a (mon.0) 634 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 10T20:59:25.421533+0000 mon.a (mon.0) 635 : audit 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cluster 2023-06-10T20:59:26.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 2023-06-10T20:59:25 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: .421600+0000 mon.a (mon.0) 636 : cluster [DBG] 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: fsmap foo:0 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: cephadm 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: 2023-06-10T20:59:25.425804+0000 mgr.y ( 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: mgr.14427) 56 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: T20:59:25.433087+0000 mon.a (mon.0) 637 : audit 2023-06-10T20:59:26.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:26 smithi152 conmon[97365]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:26.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: audit 2023-06-10T20:59:25.396968+0000 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cluster 2023-06-10T20:59:25.397125+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: audit 2023-06-10T20:59:25.412625+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cluster 2023-06-10T20:59:25.413470+0000 mon.a (mon.0) 632 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cluster 2023-06-10T20:59:25.413504+0000 mon.a (mon.0) 633 : cluster 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-10T20:59:26.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: -06-10T20:59: 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: 25.421425+0000 mon.a (mon.0 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: ) 634 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: audit 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: 2023-06-10T20:59:25 2023-06-10T20:59:26.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: .421533+0000 mon.a (mon.0) 635 : 2023-06-10T20:59:26.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:59:26.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: -06-10T20:59:25.421600+0000 2023-06-10T20:59:26.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: mon.a (mon.0) 636 : cluster [DBG] fsmap foo:0 2023-06-10T20:59:26.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: cephadm 2023-06 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: -10T20:59:25.425804+0000 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: mgr.y (mgr.14427) 56 : cephadm 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: [INF] Saving service mds.foo spec with placement count:2 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: audit 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: 2023-06- 2023-06-10T20:59:26.782 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: 10T20:59:25.433087+0000 2023-06-10T20:59:26.783 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: mon.a (mon.0) 637 : audit 2023-06-10T20:59:26.783 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[97224]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:26.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: audit 2023-06-10T20 2023-06-10T20:59:26.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: :59:25.396968+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.783 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20:59 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: :25.397125+0000 mon.a ( 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: mon.0) 630 : cluster [DBG] 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: osdmap e57: 8 total, 8 up, 8 in 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: audit 2023-06-10T20:59:25 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .412625+0000 mon.a (mon.0) 631 : audit [INF] 2023-06-10T20:59:26.784 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20:59 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: :25.413470+0000 mon.a (mon.0) 632 : cluster 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20:59:25 2023-06-10T20:59:26.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .413504+0000 mon.a (mon.0) 633 : cluster 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023- 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 06-10T20:59:25 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .421425+0000 mon.a (mon. 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 0) 634 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-10T20:59:26.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: audit 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: T20:59:25.421533+0000 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: mon.a (mon.0) 635 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: :59:25.421600+0000 mon.a 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: (mon.0) 636 : cluster 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: [DBG] fsmap foo:0 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: cephadm 2023 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: -06-10T20:59:25 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .425804+0000 mgr.y (mgr. 2023-06-10T20:59:26.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 14427) 56 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: audit 2023- 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: 06-10T20:59:25 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .433087+0000 mon.a (mon 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: .0) 637 : audit [INF] 2023-06-10T20:59:26.789 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:26 smithi064 conmon[107122]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:27.562 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:27] "GET /metrics HTTP/1.1" 200 152007 "" "Prometheus/2.33.4" 2023-06-10T20:59:27.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[97224]: cluster 2023-06-10T20:59:26.308428+0000 mgr.y (mgr 2023-06-10T20:59:27.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[97224]: .14427) 57 : cluster [DBG] pgmap v46: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:27.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[97224]: cluster 2023-06-10T20:59:26.421846+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-10T20:59:27.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[107122]: cluster 2023-06-10T20:59:26.308428 2023-06-10T20:59:27.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v46: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:27.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T20:59:27.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:27 smithi064 conmon[107122]: T20:59:26.421846+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-10T20:59:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:27 smithi152 conmon[97365]: cluster 2023-06-10T20:59:26.308428+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v46: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:27 smithi152 conmon[97365]: cluster 2023-06-10T20:59:26.421846+0000 mon.a (mon.0) 638 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-10T20:59:28.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:28 smithi152 conmon[97365]: cluster 2023-06-10T20:59:27.428282+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-10T20:59:28.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-06-10T20:59:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:28 smithi064 conmon[97224]: cluster 2023-06-10T20:59:27 2023-06-10T20:59:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:28 smithi064 conmon[97224]: .428282+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-10T20:59:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:28 smithi064 conmon[107122]: cluster 2023-06-10T20:59:27.428282+0000 2023-06-10T20:59:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:28 smithi064 conmon[107122]: mon.a (mon.0) 639 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-10T20:59:29.444 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:29 smithi152 systemd[1]: Starting Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T20:59:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:29 smithi152 conmon[97365]: cluster 2023-06-10T20:59:28.309077+0000 mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v49: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:29 smithi064 conmon[97224]: cluster 2023-06-10T20:59:28.309077 2023-06-10T20:59:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:29 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v49: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:29 smithi064 conmon[107122]: cluster 2023-06-10T20:59:28.309077+0000 2023-06-10T20:59:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:29 smithi064 conmon[107122]: mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v49: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:30.318 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 bash[128625]: 9249018f5990b07027ecd762167d7b33417c77c9c80f57fa8ead13094e79c461 2023-06-10T20:59:30.318 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-06-10T20:59:30.318 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-06-10T20:59:30.319 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-06-10T20:59:30.319 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-06-10T20:59:30.319 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-06-10T20:59:30.319 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-06-10T20:59:30.320 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-06-10T20:59:30.320 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-06-10T20:59:30.320 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="App mode production" logger=settings 2023-06-10T20:59:30.321 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 systemd[1]: Started Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-06-10T20:59:30.322 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-06-10T20:59:30.323 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-06-10T20:59:30.323 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-06-10T20:59:30.323 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-06-10T20:59:30.595 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: audit 2023-06-10T20:59:29.490896+0000 2023-06-10T20:59:30.595 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.595 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: audit 2023-06-10T20:59:30.109812+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.595 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: 2023-06-10T20:59:30.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: audit 2023-06-10T20:59:30.111030+0000 mon.a (mon.0) 642 : audit 2023-06-10T20:59:30.596 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[97365]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:30.596 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-06-10T20:59:30.596 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-06-10T20:59:30.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-06-10T20:59:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[97224]: audit 2023-06-10T20:59:29.490896+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[97224]: audit 2023-06-10T20:59:30.109812+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[97224]: audit 2023-06-10T20:59:30.111030+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[107122]: audit 2023-06-10T20:59:29.490896+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[107122]: audit 2023-06-10T20:59:30.109812+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:30 smithi064 conmon[107122]: audit 2023-06-10T20:59:30.111030+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:30.897 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-06-10T20:59:30.898 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-06-10T20:59:30.898 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-06-10T20:59:30.898 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-06-10T20:59:30.898 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-06-10T20:59:30.911 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-06-10T20:59:30.922 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-10T20:59:30.922 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-10T20:59:30.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mgr dump --format=json 2023-06-10T20:59:31.149 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:30 smithi152 conmon[128666]: t=2023-06-10T20:59:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-06-10T20:59:31.149 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-06-10T20:59:31.149 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-06-10T20:59:31.149 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-06-10T20:59:31.418 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-06-10T20:59:31.419 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-06-10T20:59:31.519 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[97365]: cluster 2023-06-10T20:59:30.309589+0000 mgr.y (mgr.14427) 2023-06-10T20:59:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[97365]: 59 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:31.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-06-10T20:59:31.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-06-10T20:59:31.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-06-10T20:59:31.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-06-10T20:59:31.696 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-06-10T20:59:31.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:31 smithi064 conmon[107122]: cluster 2023-06-10T20: 2023-06-10T20:59:31.755 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:31 smithi064 conmon[107122]: 59:30.309589+0000 mgr.y (mgr.14427) 59 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:31.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:31 smithi064 conmon[97224]: cluster 2023-06-10T20:59:30.309589+0000 mgr.y (mgr.14427) 59 : 2023-06-10T20:59:31.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:31 smithi064 conmon[97224]: cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:31.975 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-06-10T20:59:31.975 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-06-10T20:59:31.975 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-06-10T20:59:31.975 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-06-10T20:59:31.976 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-06-10T20:59:31.976 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:31 smithi152 conmon[128666]: t=2023-06-10T20:59:31+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-06-10T20:59:32.300 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-06-10T20:59:32.300 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-06-10T20:59:32.300 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-06-10T20:59:32.300 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-06-10T20:59:32.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-06-10T20:59:32.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-06-10T20:59:32.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-06-10T20:59:32.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-06-10T20:59:32.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-06-10T20:59:32.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-06-10T20:59:32.601 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-06-10T20:59:32.862 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-06-10T20:59:32.863 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-06-10T20:59:32.863 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-06-10T20:59:32.863 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-06-10T20:59:32.863 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-06-10T20:59:33.116 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:33.150 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:33.150 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:32 smithi152 conmon[128666]: t=2023-06-10T20:59:32+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-06-10T20:59:33.150 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-06-10T20:59:33.150 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-06-10T20:59:33.433 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-06-10T20:59:33.433 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-06-10T20:59:33.433 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-06-10T20:59:33.434 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-06-10T20:59:33.434 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-06-10T20:59:33.434 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-06-10T20:59:33.685 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[97365]: cluster 2023-06-10T20:59:32.309952+0000 mgr.y (mgr. 2023-06-10T20:59:33.685 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[97365]: 14427) 60 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[97365]: audit 2023-06-10T20:59:33.108512+0000 mon.a (mon.0) 643 : audit [DBG] from='client.? 172.21.15.64:0/352585451' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:59:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[97365]: audit 2023-06-10T20:59:33.353511+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:33.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[97365]: 2023-06-10T20:59:33.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-06-10T20:59:33.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-06-10T20:59:33.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-06-10T20:59:33.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-06-10T20:59:33.687 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-06-10T20:59:33.688 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-06-10T20:59:33.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[97224]: cluster 2023-06-10T20:59:32.309952+0000 mgr.y (mgr.14427) 60 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:33.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[97224]: audit 2023-06-10T20:59:33.108512+0000 mon.a (mon.0) 643 : audit [DBG] from='client.? 172.21.15.64:0/352585451' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:59:33.895 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[97224]: audit 2023-06-10T20:59:33.353511+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:33.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: cluster 2023-06-10T20:59:32.309952+0000 mgr.y (mgr.14427) 60 : cluster [DBG] 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: pgmap v51: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: audit 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: 2023-06-10T20:59:33.108512+0000 mon.a (mon 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: .0) 643 : audit [DBG] from='client.? 172.21.15.64:0/352585451' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: audit 2023 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: -06-10T20:59:33.353511+0000 2023-06-10T20:59:33.896 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: mon.a (mon.0) 644 : audit [INF] 2023-06-10T20:59:33.897 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:33 smithi064 conmon[107122]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:33.911 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":19,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":3618131890},{"type":"v1","addr":"172.21.15.64:6801","nonce":3618131890}]},"active_addr":"172.21.15.64:6801/3618131890","active_change":"2023-06-10T20:58:13.320627+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24323,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":1955159746}]},{"addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":703163662}]}]} 2023-06-10T20:59:33.915 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-10T20:59:33.915 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-10T20:59:33.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T20:59:33.944 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-06-10T20:59:33.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-06-10T20:59:33.946 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-06-10T20:59:33.946 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-06-10T20:59:34.224 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-06-10T20:59:34.224 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:33 smithi152 conmon[128666]: t=2023-06-10T20:59:33+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-06-10T20:59:34.224 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-06-10T20:59:34.224 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-06-10T20:59:34.225 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-06-10T20:59:34.294 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:34.485 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-06-10T20:59:34.485 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-06-10T20:59:34.485 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-06-10T20:59:34.485 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-06-10T20:59:34.486 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-06-10T20:59:34.486 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-06-10T20:59:34.486 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-06-10T20:59:34.486 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-06-10T20:59:34.756 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-06-10T20:59:34.757 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-06-10T20:59:34.757 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-06-10T20:59:34.757 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-06-10T20:59:34.757 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-06-10T20:59:34.757 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-06-10T20:59:34.758 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-06-10T20:59:34.758 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-06-10T20:59:34.758 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-06-10T20:59:35.023 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-06-10T20:59:35.023 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[128666]: t=2023-06-10T20:59:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-06-10T20:59:35.024 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20:59:33.753623+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20:59:34.093290+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: :59:34.094016+0000 mon.a (mon.0) 647 : audit [INF] 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:59:35.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20:59: 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 34.101728+0000 mon.a (mon.0) 648 : audit [INF] 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 2023-06-10T20:59:34.112625+0000 mon.a (mon.0) 649 : 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: :59:34.113523+0000 mon.a (mon 2023-06-10T20:59:35.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: .0) 650 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: T20:59:34.119593+0000 mon.a (mon.0) 651 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: -06-10T20:59:34 2023-06-10T20:59:35.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: .120518+0000 mon.a (mon. 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 0) 652 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: audit 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 2023-06-10T20:59: 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: 34.530704+0000 mon.a (mon.0 2023-06-10T20:59:35.028 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:34 smithi152 conmon[97365]: ) 653 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:33.753623+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.093290+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.094016+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:59:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.101728+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.112625+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.113523+0000 2023-06-10T20:59:35.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: mon.a (mon.0) 650 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:35.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.119593+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:35.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.120518+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[97224]: audit 2023-06-10T20:59:34.530704+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:33.753623+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.093290+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.094016+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.101728+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.112625+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.113523+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:35.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.119593+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:35.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T20:59:35.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: 10T20:59:34.120518+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:35.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:34 smithi064 conmon[107122]: audit 2023-06-10T20:59:34.530704+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:35.300 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-06-10T20:59:35.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-06-10T20:59:35.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-06-10T20:59:35.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-06-10T20:59:35.301 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-06-10T20:59:35.302 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-06-10T20:59:35.302 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-06-10T20:59:35.302 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-06-10T20:59:35.553 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-06-10T20:59:35.554 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-06-10T20:59:35.830 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[97365]: cephadm 2023-06-10T20:59:34.121226+0000 mgr.y (mgr.14427) 61 : cephadm [INF] Deploying daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T20:59:35.831 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:59:35.831 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[97365]: 59:34.310269+0000 mgr.y (mgr.14427) 62 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:35.831 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-06-10T20:59:35.832 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-06-10T20:59:35.832 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-06-10T20:59:35.832 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-06-10T20:59:35.833 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-06-10T20:59:35.833 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-06-10T20:59:35.833 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-06-10T20:59:35.833 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-06-10T20:59:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[107122]: cephadm 2023-06-10T20:59 2023-06-10T20:59:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[107122]: :34.121226+0000 mgr.y (mgr.14427) 61 : cephadm [INF] Deploying daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T20:59:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[107122]: cluster 2023-06-10T20:59:34.310269+0000 mgr.y (mgr.14427) 62 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[97224]: cephadm 2023-06-10T20:59:34.121226+0000 2023-06-10T20:59:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[97224]: mgr.y (mgr.14427) 61 : cephadm [INF] Deploying daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T20:59:35.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:35 smithi064 conmon[97224]: cluster 2023-06-10T20:59:34.310269+0000 mgr.y (mgr.14427) 62 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:36.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-06-10T20:59:36.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-06-10T20:59:36.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-06-10T20:59:36.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-06-10T20:59:36.099 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:35 smithi152 conmon[128666]: t=2023-06-10T20:59:35+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-06-10T20:59:36.100 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-06-10T20:59:36.100 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-06-10T20:59:36.100 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-06-10T20:59:36.268 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:36.268 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":60,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:59:27.419785+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:59:36.359 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-06-10T20:59:36.360 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-06-10T20:59:36.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-06-10T20:59:36.361 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-06-10T20:59:36.628 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-06-10T20:59:36.629 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-06-10T20:59:36.630 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-06-10T20:59:36.630 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-06-10T20:59:36.902 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-10T20:59:36.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T20:59:36.914 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-06-10T20:59:36.914 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-06-10T20:59:36.914 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-06-10T20:59:36.915 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-06-10T20:59:36.915 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-06-10T20:59:36.915 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-06-10T20:59:36.915 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-06-10T20:59:36.915 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-06-10T20:59:37.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[107122]: audit 2023-06-10T20:59 2023-06-10T20:59:37.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[107122]: :36.263601+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.64:0/3192585741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:37.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[107122]: cluster 2023-06-10T20:59:36.310761+0000 mgr.y (mgr.14427) 63 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:37.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[97224]: audit 2023-06-10T20: 2023-06-10T20:59:37.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[97224]: 59:36.263601+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.64:0/3192585741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:37.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[97224]: cluster 2023-06-10T20:59:36.310761+0000 mgr.y (mgr.14427) 63 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:37.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[97365]: audit 2023-06-10T20:59:36.263601+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.64:0/3192585741' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:37.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[97365]: cluster 2023-06-10T20:59: 2023-06-10T20:59:37.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[97365]: 36.310761+0000 mgr.y (mgr.14427) 63 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:37.168 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-06-10T20:59:37.168 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-06-10T20:59:37.168 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:36 smithi152 conmon[128666]: t=2023-06-10T20:59:36+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-06-10T20:59:37.169 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-06-10T20:59:37.170 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-06-10T20:59:37.170 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-06-10T20:59:37.170 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-06-10T20:59:37.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-06-10T20:59:37.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-06-10T20:59:37.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-06-10T20:59:37.423 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-06-10T20:59:37.423 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-06-10T20:59:37.423 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-06-10T20:59:37.423 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-06-10T20:59:37.466 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:37.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:37 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:37] "GET /metrics HTTP/1.1" 200 157029 "" "Prometheus/2.33.4" 2023-06-10T20:59:37.694 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-06-10T20:59:37.694 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-06-10T20:59:37.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-06-10T20:59:38.018 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-06-10T20:59:38.018 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-06-10T20:59:38.018 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-06-10T20:59:38.018 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-06-10T20:59:38.018 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-06-10T20:59:38.019 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-06-10T20:59:38.019 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-06-10T20:59:38.019 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:37 smithi152 conmon[128666]: t=2023-06-10T20:59:37+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-06-10T20:59:38.019 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-06-10T20:59:38.295 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-06-10T20:59:38.296 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-06-10T20:59:38.296 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-06-10T20:59:38.296 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-06-10T20:59:38.296 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-06-10T20:59:38.572 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-06-10T20:59:38.572 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-06-10T20:59:38.572 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-06-10T20:59:38.572 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: audit 2023-06-10T20:59:37.443463+0000 mon.a ( 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: audit 2023-06-10T20:59:37.444876+0000 mon.a (mon 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: .0) 655 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: audit 2023-06-10T20 2023-06-10T20:59:38.573 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: :59:37.448625+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: audit 2023-06-10T20:59:37.449533+0000 mon.a (mon. 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 0) 657 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cephadm 2023-06-10T20:59:37.450219+0000 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: mgr.y (mgr.14427) 64 : cephadm [INF] Deploying daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10T20:59 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: :38.016874+0000 mon.a (mon.0) 658 : cluster [DBG] mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:boot 2023-06-10T20:59:38.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: T20:59:38.016919+0000 mon.a (mon.0) 659 : cluster [INF] daemon mds.foo.smithi064.wuuunt assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10T20: 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 59:38.016973+0000 mon.a (mon.0) 660 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10 2023-06-10T20:59:38.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: T20:59:38.016991+0000 mon.a (mon.0) 661 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-10T20:59:38.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T20:59:38.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 20:59:38.017003+0000 mon.a (mon.0) 662 : cluster [INF] 2023-06-10T20:59:38.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: Cluster is now healthy 2023-06-10T20:59:38.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023 2023-06-10T20:59:38.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: -06-10T20:59:38.017203+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-10T20:59:38.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T20:59:38.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: T20:59:38.017311+0000 mon.a (mon.0) 664 : audit 2023-06-10T20:59:38.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T20:59:38.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023-06-10T20:59:38.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 2023-06-10T20:59:38.020140+0000 mon.a (mon.0) 665 : cluster [DBG] 2023-06-10T20:59:38.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:creating} 2023-06-10T20:59:38.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: cluster 2023 2023-06-10T20:59:38.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: -06-10T20:59:38.031559+0000 mon.a (mon.0) 666 : cluster [INF] daemon mds.foo.smithi064.wuuunt is now active in filesystem foo as rank 0 2023-06-10T20:59:38.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[97365]: 2023-06-10T20:59:38.702 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: audit 2023-06-10T20:59:37.443463+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: audit 2023-06-10T20:59:37.444876+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: audit 2023-06-10T20:59:37.448625+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: audit 2023-06-10T20:59:37.449533+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cephadm 2023-06-10T20:59:37.450219+0000 mgr.y (mgr.14427) 64 : cephadm [INF] Deploying daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T20:59:38.703 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: 20:59:38.016874+0000 mon.a (mon.0) 658 : cluster [DBG] mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:boot 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.016919+0000 mon.a (mon.0) 659 : cluster [INF] daemon mds.foo.smithi064.wuuunt assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.016973+0000 mon.a (mon.0) 660 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.016991+0000 mon.a (mon.0) 661 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.017003+0000 mon.a (mon.0) 662 : cluster [INF] Cluster is now healthy 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.017203+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: audit 2023-06-10T20:59:38.017311+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T20:59:38.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.020140+0000 mon.a (mon.0) 665 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:creating} 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.031559+0000 mon.a (mon.0) 666 : cluster [INF] daemon mds.foo.smithi064.wuuunt is now active in filesystem foo as rank 0 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: audit 2023-06-10T20:59:37.443463+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: audit 2023-06-10T20:59:37.444876+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: audit 2023-06-10T20:59:37.448625+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: audit 2023-06-10T20:59:37.449533+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T20:59:38.705 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cephadm 2023-06- 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: 10T20:59:37.450219+0000 mgr.y (mgr.14427) 64 : cephadm [INF] Deploying daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.016874+0000 mon.a (mon.0) 658 : cluster [DBG] mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:boot 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.016919+0000 mon.a (mon.0) 659 : cluster [INF] daemon mds.foo.smithi064.wuuunt assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.016973+0000 mon.a (mon.0) 660 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.016991+0000 mon.a (mon.0) 661 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.017003+0000 mon.a (mon.0) 662 : cluster [INF] Cluster is now healthy 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.017203+0000 mon.a (mon.0) 663 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-10T20:59:38.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: audit 2023-06-10T20:59:38.017311+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: 2023-06-10T20:59:38.020140+0000 mon.a (mon.0) 665 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:creating} 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: -06-10T20:59:38.031559 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: +0000 mon.a (mon.0) 666 : 2023-06-10T20:59:38.707 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:38 smithi064 conmon[97224]: cluster [INF] daemon mds.foo.smithi064.wuuunt is now active in filesystem foo as rank 0 2023-06-10T20:59:38.935 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-06-10T20:59:38.935 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:38 smithi152 conmon[128666]: t=2023-06-10T20:59:38+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-06-10T20:59:39.041 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T20:59:39.041 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":60,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T20:59:27.419785+0000","last_up_change":"2023-06-10T20:57:55.181726+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6803","nonce":2269841417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6805","nonce":2269841417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6809","nonce":2269841417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":2269841417},{"type":"v1","addr":"172.21.15.64:6807","nonce":2269841417}]},"public_addr":"172.21.15.64:6803/2269841417","cluster_addr":"172.21.15.64:6805/2269841417","heartbeat_back_addr":"172.21.15.64:6809/2269841417","heartbeat_front_addr":"172.21.15.64:6807/2269841417","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6811","nonce":3223495478}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6813","nonce":3223495478}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6817","nonce":3223495478}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":3223495478},{"type":"v1","addr":"172.21.15.64:6815","nonce":3223495478}]},"public_addr":"172.21.15.64:6811/3223495478","cluster_addr":"172.21.15.64:6813/3223495478","heartbeat_back_addr":"172.21.15.64:6817/3223495478","heartbeat_front_addr":"172.21.15.64:6815/3223495478","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6819","nonce":208030224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6821","nonce":208030224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6825","nonce":208030224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":208030224},{"type":"v1","addr":"172.21.15.64:6823","nonce":208030224}]},"public_addr":"172.21.15.64:6819/208030224","cluster_addr":"172.21.15.64:6821/208030224","heartbeat_back_addr":"172.21.15.64:6825/208030224","heartbeat_front_addr":"172.21.15.64:6823/208030224","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6827","nonce":791323195}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6829","nonce":791323195}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6833","nonce":791323195}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":791323195},{"type":"v1","addr":"172.21.15.64:6831","nonce":791323195}]},"public_addr":"172.21.15.64:6827/791323195","cluster_addr":"172.21.15.64:6829/791323195","heartbeat_back_addr":"172.21.15.64:6833/791323195","heartbeat_front_addr":"172.21.15.64:6831/791323195","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T20:59:39.194 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-06-10T20:59:39.195 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-06-10T20:59:39.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: cluster 2023-06-10T20:59:38.311299+0000 mgr.y ( 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: mgr.14427) 65 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: cluster 2023-06-10T20:59:39.023939+0000 mon.a (mon.0) 667 : cluster [DBG] 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:active 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: cluster 2023-06-10T20:59:39.023996+0000 mon.a (mon.0) 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: 668 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T20:59:39.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[97365]: 10T20:59:39.037098+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.64:0/85503695' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-06-10T20:59:39.497 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-06-10T20:59:39.498 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-06-10T20:59:39.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: cluster 2023-06-10T20:59:38.311299+0000 mgr.y (mgr.14427) 65 : 2023-06-10T20:59:39.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:39.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: cluster 2023-06-10T20:59:39.023939+0000 mon.a (mon.0) 667 : cluster [DBG] 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:active 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: cluster 2023-06-10T20:59:39.023996+0000 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: mon.a (mon.0) 668 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: audit 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: 2023-06-10T20:59:39.037098+0000 mon.a (mon. 2023-06-10T20:59:39.708 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[107122]: 0) 669 : audit [DBG] from='client.? 172.21.15.64:0/85503695' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:39.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[97224]: cluster 2023-06-10T20:59:38.311299+0000 mgr.y ( 2023-06-10T20:59:39.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[97224]: mgr.14427) 65 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:39.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[97224]: cluster 2023-06-10T20:59:39.023939+0000 mon.a (mon.0) 667 : cluster [DBG] mds.? [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] up:active 2023-06-10T20:59:39.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[97224]: cluster 2023-06-10T20:59:39.023996+0000 mon.a (mon.0) 668 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 2023-06-10T20:59:39.709 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:39 smithi064 conmon[97224]: audit 2023-06-10T20:59:39.037098+0000 mon.a (mon.0) 669 : audit [DBG] from='client.? 172.21.15.64:0/85503695' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T20:59:39.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-10T20:59:39.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-10T20:59:39.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-10T20:59:39.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-10T20:59:39.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-10T20:59:39.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-10T20:59:39.713 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-10T20:59:39.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-06-10T20:59:39.790 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-06-10T20:59:39.791 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-06-10T20:59:39.791 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-06-10T20:59:40.042 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-06-10T20:59:40.042 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:39 smithi152 conmon[128666]: t=2023-06-10T20:59:39+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-06-10T20:59:40.337 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-06-10T20:59:40.337 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-06-10T20:59:40.337 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-06-10T20:59:40.338 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-06-10T20:59:40.338 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-06-10T20:59:40.338 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-06-10T20:59:40.597 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-06-10T20:59:40.598 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-06-10T20:59:40.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-06-10T20:59:40.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-06-10T20:59:40.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-06-10T20:59:40.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[97365]: audit 2023-06-10T20:59:39.536812+0000 mon.a (mon.0) 670 2023-06-10T20:59:40.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[97365]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[97365]: audit 2023-06-10T20:59:40.322718+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[97365]: audit 2023-06-10T20:59:40.324228+0000 mon.a (mon.0) 672 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:40.872 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-06-10T20:59:40.872 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-06-10T20:59:40.873 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-06-10T20:59:40.873 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-06-10T20:59:40.873 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-06-10T20:59:40.873 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-06-10T20:59:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[107122]: audit 2023-06-10T20:59:39.536812+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[107122]: audit 2023-06-10T20:59:40.322718+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[107122]: audit 2023-06-10T20:59:40.324228+0000 mon.a (mon.0) 672 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:40.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[97224]: audit 2023-06-10T20:59:39.536812+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[97224]: audit 2023-06-10T20:59:40.322718+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:40 smithi064 conmon[97224]: audit 2023-06-10T20:59:40.324228+0000 mon.a (mon.0) 672 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T20:59:41.194 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-06-10T20:59:41.195 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:40 smithi152 conmon[128666]: t=2023-06-10T20:59:40+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-06-10T20:59:41.195 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-06-10T20:59:41.195 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-06-10T20:59:41.195 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-06-10T20:59:41.471 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-06-10T20:59:41.471 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-06-10T20:59:41.471 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-06-10T20:59:41.472 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-06-10T20:59:41.472 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-06-10T20:59:41.472 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-06-10T20:59:41.572 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.623 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.723 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: cluster 2023-06-10T20:59:40.311847+0000 mgr.y (mgr.14427) 66 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:41.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: 2023-06-10T20:59:41.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: cluster 2023-06-10T20:59:40.542482+0000 mon.a (mon.0) 673 : cluster [DBG] mds.? [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] up:boot 2023-06-10T20:59:41.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: cluster 2023-06-10T20:59:40. 2023-06-10T20:59:41.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: 542792+0000 mon.a (mon.0) 674 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: audit 2023-06-10T20:59:40.542920+0000 2023-06-10T20:59:41.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: mon.a (mon.0) 675 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T20:59:41.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: cluster 2023-06 2023-06-10T20:59:41.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[97365]: -10T20:59:40.545595+0000 mon.a (mon.0) 676 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.725 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-06-10T20:59:41.725 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-06-10T20:59:41.726 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: cluster 2023-06-10T20:59:40.311847 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 66 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: cluster 2023-06-10T20 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: :59:40.542482+0000 mon.a (mon.0) 673 : cluster [DBG] mds.? [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] up:boot 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: cluster 2023-06-10T20:59:40.542792+0000 2023-06-10T20:59:41.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: mon.a (mon.0) 674 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: 20:59:40.542920+0000 mon.a (mon.0) 675 : audit 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: cluster 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: 2023-06-10T20:59:40.545595+0000 mon.a (mon.0) 676 : cluster [DBG] 2023-06-10T20:59:41.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[107122]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: cluster 2023-06-10T20:59:40.311847+0000 mgr.y (mgr.14427) 2023-06-10T20:59:41.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: 66 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:41.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: cluster 2023-06-10T20:59:40.542482+0000 mon.a (mon.0) 673 : cluster [DBG] mds.? [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] up:boot 2023-06-10T20:59:41.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: cluster 2023-06-10T20 2023-06-10T20:59:41.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: :59:40.542792+0000 mon.a ( 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: mon.0) 674 : cluster [DBG] 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: audit 2023- 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: 06-10T20:59:40. 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: 542920+0000 mon.a (mon.0 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: ) 675 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T20:59:41.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: cluster 2023 2023-06-10T20:59:41.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: -06-10T20:59:40 2023-06-10T20:59:41.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: .545595+0000 mon.a (mon.0 2023-06-10T20:59:41.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: ) 676 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T20:59:41.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:41 smithi064 conmon[97224]: 2023-06-10T20:59:41.846 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.846 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.886 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:41.976 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-06-10T20:59:41.976 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-06-10T20:59:41.976 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-06-10T20:59:41.977 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-06-10T20:59:42.261 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:41 smithi152 conmon[128666]: t=2023-06-10T20:59:41+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-06-10T20:59:42.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-06-10T20:59:42.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-06-10T20:59:42.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-06-10T20:59:42.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-06-10T20:59:42.262 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-06-10T20:59:42.396 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:42.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-06-10T20:59:42.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-06-10T20:59:42.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-06-10T20:59:42.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-06-10T20:59:42.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-06-10T20:59:42.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-06-10T20:59:42.626 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:42.841 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-06-10T20:59:42.841 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-06-10T20:59:42.841 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-06-10T20:59:42.842 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-06-10T20:59:42.842 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-06-10T20:59:42.842 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-06-10T20:59:43.093 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:43.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-06-10T20:59:43.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:42 smithi152 conmon[128666]: t=2023-06-10T20:59:42+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-06-10T20:59:43.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-06-10T20:59:43.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-06-10T20:59:43.094 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-06-10T20:59:43.379 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-06-10T20:59:43.379 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-06-10T20:59:43.379 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-06-10T20:59:43.380 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-06-10T20:59:43.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[97365]: cluster 2023-06-10T20:59:42.312200+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:43.640 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T20:59:43.640 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[97365]: 20:59:43.417503+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:43.640 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-06-10T20:59:43.640 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-06-10T20:59:43.641 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-06-10T20:59:43.642 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-06-10T20:59:43.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[97224]: cluster 2023-06-10T20:59:42.312200+0000 mgr.y (mgr.14427) 67 : cluster 2023-06-10T20:59:43.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[97224]: [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:43.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[97224]: audit 2023-06-10T20:59:43.417503+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[107122]: cluster 2023-06-10T20:59:42.312200+0000 2023-06-10T20:59:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[107122]: mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[107122]: audit 2023-06-10T20:59:43.417503 2023-06-10T20:59:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[107122]: +0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:43 smithi064 conmon[107122]: 2023-06-10T20:59:43.917 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-06-10T20:59:43.917 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-06-10T20:59:43.918 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-06-10T20:59:44.178 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:43 smithi152 conmon[128666]: t=2023-06-10T20:59:43+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-06-10T20:59:44.179 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-06-10T20:59:44.180 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-06-10T20:59:44.180 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-06-10T20:59:44.180 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-06-10T20:59:44.180 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-06-10T20:59:44.180 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-06-10T20:59:44.431 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-06-10T20:59:44.431 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-06-10T20:59:44.431 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-06-10T20:59:44.431 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-06-10T20:59:44.432 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-06-10T20:59:44.432 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-06-10T20:59:44.432 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-06-10T20:59:44.432 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-06-10T20:59:44.694 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-06-10T20:59:44.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-06-10T20:59:44.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-06-10T20:59:44.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-06-10T20:59:44.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-06-10T20:59:44.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-06-10T20:59:44.696 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-06-10T20:59:44.696 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-06-10T20:59:44.993 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-06-10T20:59:44.994 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-06-10T20:59:44.994 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-06-10T20:59:44.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-06-10T20:59:44.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-06-10T20:59:44.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-06-10T20:59:44.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-06-10T20:59:44.995 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-06-10T20:59:44.996 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:44 smithi152 conmon[128666]: t=2023-06-10T20:59:44+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-06-10T20:59:45.279 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-06-10T20:59:45.279 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-06-10T20:59:45.279 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-06-10T20:59:45.279 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-06-10T20:59:45.279 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-06-10T20:59:45.280 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-06-10T20:59:45.280 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-06-10T20:59:45.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[97365]: cluster 2023-06-10T20:59:44.312619+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:45.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[97365]: audit 2023-06-10T20:59:44.491880+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[97365]: audit 2023-06-10T20:59: 2023-06-10T20:59:45.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[97365]: 44.542746+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.561 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-06-10T20:59:45.561 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-06-10T20:59:45.561 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=15.318004439s 2023-06-10T20:59:45.561 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-06-10T20:59:45.561 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-06-10T20:59:45.562 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-06-10T20:59:45.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[97224]: cluster 2023-06-10T20:59:44.312619+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:45.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[97224]: audit 2023-06-10T20:59:44.491880+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[97224]: audit 2023-06-10T20:59:44.542746+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[107122]: cluster 2023-06-10T20:59:44.312619+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[107122]: audit 2023-06-10T20:59:44.491880+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:45 smithi064 conmon[107122]: audit 2023-06-10T20:59:44.542746+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T20:59:45.944 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-06-10T20:59:45.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-06-10T20:59:45.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 20:59:45 smithi152 conmon[128666]: t=2023-06-10T20:59:45+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-06-10T20:59:46.986 INFO:teuthology.orchestra.run.smithi064.stdout:55834574898 2023-06-10T20:59:46.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-10T20:59:47.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:47 smithi064 conmon[107122]: cluster 2023-06-10T20:59:46. 2023-06-10T20:59:47.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:47 smithi064 conmon[107122]: 313119+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:47.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:47 smithi064 conmon[97224]: cluster 2023-06-10T20:59:46.313119+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:47 smithi152 conmon[97365]: cluster 2023-06-10T20:59:46.313119+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:47.557 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:47 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:47] "GET /metrics HTTP/1.1" 200 187818 "" "Prometheus/2.33.4" 2023-06-10T20:59:49.276 INFO:teuthology.orchestra.run.smithi064.stdout:107374182442 2023-06-10T20:59:49.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-10T20:59:49.438 INFO:teuthology.orchestra.run.smithi064.stdout:180388626461 2023-06-10T20:59:49.439 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-10T20:59:49.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:49 smithi064 conmon[107122]: cluster 2023-06-10T20:59:48.313566+0000 2023-06-10T20:59:49.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:49 smithi064 conmon[107122]: mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:49 smithi064 conmon[97224]: cluster 2023-06-10T20:59:48.313566+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:49 smithi152 conmon[97365]: cluster 2023-06-10T20:59:48.313566+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:51.190 INFO:teuthology.orchestra.run.smithi064.stdout:34359738423 2023-06-10T20:59:51.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-10T20:59:51.442 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:51 smithi064 conmon[107122]: cluster 2023-06-10T20:59:50.314080+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:51.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:51 smithi064 conmon[97224]: cluster 2023-06-10T20:59:50.314080+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:51 smithi152 conmon[97365]: cluster 2023-06-10T20:59:50.314080+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:51 smithi152 conmon[97365]: 2023-06-10T20:59:52.091 INFO:teuthology.orchestra.run.smithi064.stdout:154618822689 2023-06-10T20:59:52.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-10T20:59:52.489 INFO:teuthology.orchestra.run.smithi064.stdout:206158430233 2023-06-10T20:59:52.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-10T20:59:52.588 INFO:teuthology.orchestra.run.smithi064.stdout:128849018917 2023-06-10T20:59:52.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-10T20:59:52.637 INFO:teuthology.orchestra.run.smithi064.stdout:77309411376 2023-06-10T20:59:52.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-10T20:59:52.664 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:52.725 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:53.001 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:53.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 20:59:52 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T20:59:53.199 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:53.288 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:53 smithi064 conmon[97224]: cluster 2023-06-10T20:59:52. 2023-06-10T20:59:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:53 smithi064 conmon[97224]: 314439+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:53.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:53 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T20:59:53.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:53 smithi064 conmon[107122]: T20:59:52.314439+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:53 smithi152 conmon[97365]: cluster 2023-06-10T20:59:52.314439+0000 mgr.y (mgr.14427) 72 : cluster 2023-06-10T20:59:53.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:53 smithi152 conmon[97365]: [DBG] pgmap v61: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:55.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:55 smithi064 conmon[97224]: cluster 2023-06-10T20:59:54.314811+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:55 smithi064 conmon[107122]: cluster 2023-06-10T20:59:54.314811+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:55 smithi152 conmon[97365]: cluster 2023-06-10T20:59:54.314811+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v62: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:56.268 INFO:teuthology.orchestra.run.smithi064.stdout:180388626462 2023-06-10T20:59:56.700 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:57.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[97224]: audit 2023-06-10T20:59:56.265172+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.64:0/1759493717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:57.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[97224]: cluster 2023-06-10T20:59:56.315578+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:57.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[97224]: audit 2023-06-10T20:59:56.663082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.64:0/1565077442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:57.105 INFO:teuthology.orchestra.run.smithi064.stdout:55834574900 2023-06-10T20:59:57.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[107122]: audit 2023-06-10T20:59:56.265172+0000 mon.a (mon 2023-06-10T20:59:57.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[107122]: .0) 680 : audit [DBG] from='client.? 172.21.15.64:0/1759493717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:57.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[107122]: cluster 2023-06-10T20:59:56.315578+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:57.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[107122]: audit 2023-06-10T20:59:56.663082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.64:0/1565077442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:57 smithi152 conmon[97365]: audit 2023-06-10T20:59:56.265172+0000 mon.a (mon.0) 680 : audit [DBG] from='client.? 172.21.15.64:0/1759493717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T20:59:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:57 smithi152 conmon[97365]: cluster 2023-06-10T20:59:56.315578+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v63: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-10T20:59:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:57 smithi152 conmon[97365]: audit 2023-06-10T20:59:56.663082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.64:0/1565077442' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T20:59:57.626 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 20:59:57 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:20:59:57] "GET /metrics HTTP/1.1" 200 187818 "" "Prometheus/2.33.4" 2023-06-10T20:59:58.213 INFO:teuthology.orchestra.run.smithi064.stdout:34359738425 2023-06-10T20:59:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:58 smithi064 conmon[97224]: audit 2023-06-10T20:59:58.210113+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.64:0/4087277866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:58.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:58 smithi064 conmon[107122]: audit 2023-06-10T20:59:58.210113+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.64:0/4087277866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:58.687 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574898 got 55834574900 for osd.1 2023-06-10T20:59:58.687 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:58.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:58 smithi152 conmon[97365]: audit 2023-06-10T20:59:58.210113+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.64:0/4087277866' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T20:59:58.858 INFO:teuthology.orchestra.run.smithi064.stdout:154618822691 2023-06-10T20:59:58.915 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738423 got 34359738425 for osd.0 2023-06-10T20:59:58.916 DEBUG:teuthology.parallel:result is None 2023-06-10T20:59:59.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[97224]: cluster 2023-06-10T20:59:58.315975+0000 mgr.y (mgr. 2023-06-10T20:59:59.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[97224]: 14427) 75 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:59.523 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T20:59:59.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[107122]: cluster 2023-06-10T20:59:58. 2023-06-10T20:59:59.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[107122]: 315975+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:59.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[107122]: audit 2023-06-10T20:59:58.581078+0000 mon.a 2023-06-10T20:59:59.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[107122]: (mon.0) 681 : audit [DBG] from='client.? 172.21.15.64:0/2179002250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:59.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 20:59:59 smithi064 conmon[97224]: audit 2023-06-10T20:59:58.581078+0000 mon.a (mon.0) 681 : audit [DBG] from='client.? 172.21.15.64:0/2179002250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:59 smithi152 conmon[97365]: cluster 2023-06-10T20:59:58.315975+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v64: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T20:59:59.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 20:59:59 smithi152 conmon[97365]: audit 2023-06-10T20:59:58.581078+0000 mon.a (mon.0) 681 : audit [DBG] from='client.? 172.21.15.64:0/2179002250' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T20:59:59.863 INFO:teuthology.orchestra.run.smithi064.stdout:107374182444 2023-06-10T21:00:00.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: audit 2023-06-10T20:59:59.688988+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.64:0/2078161015' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:00:00.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: cluster 2023-06-10T21:00:00.000103+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2023-06-10T21:00:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: audit 2023-06-10T21:00:00.118852+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: audit 2023-06-10T21:00:00.119687+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: audit 2023-06-10T21:00:00.120387+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: audit 2023-06-10T21:00:00.124437+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[97224]: 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: audit 2023-06-10T20:59:59.688988+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.64:0/2078161015' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: cluster 2023-06-10T21:00:00.000103+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: audit 2023-06-10T21:00:00.118852+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: audit 2023-06-10T21:00:00.119687+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: audit 2023-06-10T21:00:00.120387+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:00:00.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:00 smithi064 conmon[107122]: T21:00:00.124437+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.683 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626461 got 180388626462 for osd.6 2023-06-10T21:00:00.683 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: audit 2023-06-10T20:59:59.688988+0000 mon.a (mon.0) 682 : audit [DBG] from='client.? 172.21.15.64:0/2078161015' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: cluster 2023-06-10T21:00:00.000103+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: audit 2023-06-10T21:00:00.118852+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: audit 2023-06-10T21:00:00.119687+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: audit 2023-06-10T21:00:00.120387+0000 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: audit 2023-06-10T21:00:00.124437 2023-06-10T21:00:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: +0000 mon.a (mon.0) 687 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:00 smithi152 conmon[97365]: 2023-06-10T21:00:00.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822689 got 154618822691 for osd.5 2023-06-10T21:00:00.724 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:01.161 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:01.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:00.128129+0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:00.131229+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[97224]: cluster 2023-06-10T21:00:00.316665+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: cephadm 2023-06-10T21:00:00.128129 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T21:00:01.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: 10T21:00:00.131229+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:00:01.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: cluster 2023 2023-06-10T21:00:01.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: -06-10T21:00:00.316665+0000 mgr.y ( 2023-06-10T21:00:01.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:01 smithi064 conmon[107122]: mgr.14427) 78 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:01.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:01 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:00.128129+0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:00:01.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:01 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:00.131229+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:00:01.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:01 smithi152 conmon[97365]: cluster 2023-06-10T21:00:00.316665+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:02.527 INFO:teuthology.orchestra.run.smithi064.stdout:206158430236 2023-06-10T21:00:02.769 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182442 got 107374182444 for osd.3 2023-06-10T21:00:02.769 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:03.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:00:02 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:00:03.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:03 smithi064 conmon[97224]: cluster 2023-06-10T21:00:02.317034+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:03.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:03 smithi064 conmon[97224]: audit 2023-06-10T21:00:02.524637+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.64:0/48812477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:00:03.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:03 smithi064 conmon[107122]: cluster 2023-06-10T21:00:02.317034+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:03.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:03 smithi064 conmon[107122]: audit 2023-06-10T21:00:02.524637+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.64:0/48812477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:00:03.528 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430236 for osd.7 2023-06-10T21:00:03.529 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:03 smithi152 conmon[97365]: cluster 2023-06-10T21:00:02.317034+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:03 smithi152 conmon[97365]: audit 2023-06-10T21:00:02.524637+0000 mon.a (mon.0) 688 : audit [DBG] from='client.? 172.21.15.64:0/48812477' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:00:04.620 INFO:teuthology.orchestra.run.smithi064.stdout:77309411378 2023-06-10T21:00:05.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[107122]: cluster 2023-06-10T21:00: 2023-06-10T21:00:05.378 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:05 smithi064 systemd[1]: Stopping Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:00:05.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[97224]: cluster 2023-06-10T21:00:04.317477 2023-06-10T21:00:05.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 80 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:05.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[97224]: audit 2023-06-10T21:00:04.617507+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.64:0/3230169970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:00:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[107122]: 04.317477+0000 mgr.y (mgr.14427) 80 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[107122]: audit 2023-06-10T21:00:04.617507+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.64:0/3230169970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:00:05.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:05 smithi064 bash[157620]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-alertmanager.a found: no such container 2023-06-10T21:00:05.685 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411376 got 77309411378 for osd.2 2023-06-10T21:00:05.685 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:05 smithi152 conmon[97365]: cluster 2023-06-10T21:00:04.317477+0000 mgr.y (mgr.14427) 80 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:05.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:05 smithi152 conmon[97365]: audit 2023-06-10T21:00:04.617507+0000 mon.a (mon.0) 689 : audit [DBG] from='client.? 172.21.15.64:0/3230169970' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:00:06.136 INFO:teuthology.orchestra.run.smithi064.stdout:128849018920 2023-06-10T21:00:06.407 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:05 smithi064 conmon[139765]: level=info ts=2023-06-10T21:00:05.943Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:00:07.172 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018917 got 128849018920 for osd.4 2023-06-10T21:00:07.172 DEBUG:teuthology.parallel:result is None 2023-06-10T21:00:07.173 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-10T21:00:07.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:00:07.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[97224]: audit 2023-06-10T21:00:06. 2023-06-10T21:00:07.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[97224]: 132547+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.64:0/1530918557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:00:07.232 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[97224]: cluster 2023-06-10T21:00:06.318231+0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:07.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[107122]: audit 2023-06-10T21:00:06.132547+0000 2023-06-10T21:00:07.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[107122]: mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.64:0/1530918557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:00:07.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[107122]: cluster 2023-06-10T21:00:06.318231+0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:07.233 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:06 smithi064 bash[157835]: da4a0af553711b8b60756f51df04475b8a5675f77a35f8c69a0fbc63fd055c26 2023-06-10T21:00:07.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:07 smithi152 conmon[97365]: audit 2023-06-10T21:00:06.132547+0000 mon.a (mon.0) 690 : audit [DBG] from='client.? 172.21.15.64:0/1530918557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:00:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:07 smithi152 conmon[97365]: cluster 2023-06-10T21:00:06.318231 2023-06-10T21:00:07.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:07 smithi152 conmon[97365]: +0000 mgr.y (mgr.14427) 81 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:07.595 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:07.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:07] "GET /metrics HTTP/1.1" 200 187818 "" "Prometheus/2.33.4" 2023-06-10T21:00:07.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-10T21:00:07.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 systemd[1]: Stopped Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:00:07.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 systemd[1]: Starting Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:00:08.158 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.886Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-06-10T21:00:08.158 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.886Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-06-10T21:00:08.158 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.887Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.888Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 bash[158300]: 163fca1ec9a54a7e49b48476f0249f1d643cefeac1e7019799d3f2af5fe46010 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 systemd[1]: Started Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.922Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.923Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.925Z caller=main.go:518 msg=Listening address=:9093 2023-06-10T21:00:08.159 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:07 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:07.925Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-10T21:00:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:08 smithi152 conmon[97365]: audit 2023-06-10T21:00:07.925176+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:08 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:07.926538+0000 mgr.y (mgr.14427) 82 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:00:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:08 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:07.930141+0000 mgr.y (mgr.14427) 83 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:00:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:08 smithi152 conmon[97365]: cluster 2023-06-10T21:00:08.318772+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:09.335 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:00:09.336 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:00:09.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[107122]: audit 2023-06-10T21:00:07.925176+0000 2023-06-10T21:00:09.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[107122]: mon.a (mon.0) 691 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:09.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[107122]: cephadm 2023-06-10T21:00:07.926538+0000 mgr.y (mgr.14427) 82 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:00:09.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[107122]: cephadm 2023-06-10T21:00:07.930141+0000 mgr.y (mgr.14427) 83 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:00:09.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[107122]: cluster 2023-06-10T21:00:08.318772+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[97224]: audit 2023-06-10T21:00:07.925176+0000 2023-06-10T21:00:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[97224]: mon.a (mon.0) 691 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:07.926538+0000 mgr.y (mgr.14427) 82 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:00:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:07.930141+0000 mgr.y (mgr.14427) 83 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:00:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:08 smithi064 conmon[97224]: cluster 2023-06-10T21:00:08.318772+0000 mgr.y (mgr.14427) 84 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:09.922 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-06-10T21:00:08.318488+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":46128,"kb_used_data":4784,"kb_used_omap":0,"kb_used_meta":41280,"kb_avail":749915088,"statfs":{"total":767960285184,"available":767913050112,"internally_reserved":0,"allocated":4898816,"data_stored":2054375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42270720},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003400"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654540+0000","last_change":"2023-06-10T20:59:26.107674+0000","last_active":"2023-06-10T20:59:26.654540+0000","last_peered":"2023-06-10T20:59:26.654540+0000","last_clean":"2023-06-10T20:59:26.654540+0000","last_became_active":"2023-06-10T20:59:26.107295+0000","last_became_peered":"2023-06-10T20:59:26.107295+0000","last_unstale":"2023-06-10T20:59:26.654540+0000","last_undegraded":"2023-06-10T20:59:26.654540+0000","last_fullsized":"2023-06-10T20:59:26.654540+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.026194+0000","last_change":"2023-06-10T20:59:25.411031+0000","last_active":"2023-06-10T20:59:38.026194+0000","last_peered":"2023-06-10T20:59:38.026194+0000","last_clean":"2023-06-10T20:59:38.026194+0000","last_became_active":"2023-06-10T20:59:25.410713+0000","last_became_peered":"2023-06-10T20:59:25.410713+0000","last_unstale":"2023-06-10T20:59:38.026194+0000","last_undegraded":"2023-06-10T20:59:38.026194+0000","last_fullsized":"2023-06-10T20:59:38.026194+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423649+0000","last_change":"2023-06-10T20:59:25.435382+0000","last_active":"2023-06-10T20:59:26.423649+0000","last_peered":"2023-06-10T20:59:26.423649+0000","last_clean":"2023-06-10T20:59:26.423649+0000","last_became_active":"2023-06-10T20:59:25.435084+0000","last_became_peered":"2023-06-10T20:59:25.435084+0000","last_unstale":"2023-06-10T20:59:26.423649+0000","last_undegraded":"2023-06-10T20:59:26.423649+0000","last_fullsized":"2023-06-10T20:59:26.423649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:27.427652+0000","last_change":"2023-06-10T20:59:25.407062+0000","last_active":"2023-06-10T20:59:27.427652+0000","last_peered":"2023-06-10T20:59:27.427652+0000","last_clean":"2023-06-10T20:59:27.427652+0000","last_became_active":"2023-06-10T20:59:25.406722+0000","last_became_peered":"2023-06-10T20:59:25.406722+0000","last_unstale":"2023-06-10T20:59:27.427652+0000","last_undegraded":"2023-06-10T20:59:27.427652+0000","last_fullsized":"2023-06-10T20:59:27.427652+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654830+0000","last_change":"2023-06-10T20:59:25.428869+0000","last_active":"2023-06-10T20:59:26.654830+0000","last_peered":"2023-06-10T20:59:26.654830+0000","last_clean":"2023-06-10T20:59:26.654830+0000","last_became_active":"2023-06-10T20:59:25.428527+0000","last_became_peered":"2023-06-10T20:59:25.428527+0000","last_unstale":"2023-06-10T20:59:26.654830+0000","last_undegraded":"2023-06-10T20:59:26.654830+0000","last_fullsized":"2023-06-10T20:59:26.654830+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029303+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T20:59:38.029303+0000","last_peered":"2023-06-10T20:59:38.029303+0000","last_clean":"2023-06-10T20:59:38.029303+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T20:59:38.029303+0000","last_undegraded":"2023-06-10T20:59:38.029303+0000","last_fullsized":"2023-06-10T20:59:38.029303+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655872+0000","last_change":"2023-06-10T20:59:25.431228+0000","last_active":"2023-06-10T20:59:26.655872+0000","last_peered":"2023-06-10T20:59:26.655872+0000","last_clean":"2023-06-10T20:59:26.655872+0000","last_became_active":"2023-06-10T20:59:25.430927+0000","last_became_peered":"2023-06-10T20:59:25.430927+0000","last_unstale":"2023-06-10T20:59:26.655872+0000","last_undegraded":"2023-06-10T20:59:26.655872+0000","last_fullsized":"2023-06-10T20:59:26.655872+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.025712+0000","last_change":"2023-06-10T20:59:25.405538+0000","last_active":"2023-06-10T20:59:38.025712+0000","last_peered":"2023-06-10T20:59:38.025712+0000","last_clean":"2023-06-10T20:59:38.025712+0000","last_became_active":"2023-06-10T20:59:25.405084+0000","last_became_peered":"2023-06-10T20:59:25.405084+0000","last_unstale":"2023-06-10T20:59:38.025712+0000","last_undegraded":"2023-06-10T20:59:38.025712+0000","last_fullsized":"2023-06-10T20:59:38.025712+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653749+0000","last_change":"2023-06-10T20:59:25.432132+0000","last_active":"2023-06-10T20:59:26.653749+0000","last_peered":"2023-06-10T20:59:26.653749+0000","last_clean":"2023-06-10T20:59:26.653749+0000","last_became_active":"2023-06-10T20:59:25.431840+0000","last_became_peered":"2023-06-10T20:59:25.431840+0000","last_unstale":"2023-06-10T20:59:26.653749+0000","last_undegraded":"2023-06-10T20:59:26.653749+0000","last_fullsized":"2023-06-10T20:59:26.653749+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808163+0000","last_change":"2023-06-10T20:59:25.412193+0000","last_active":"2023-06-10T20:59:28.808163+0000","last_peered":"2023-06-10T20:59:28.808163+0000","last_clean":"2023-06-10T20:59:28.808163+0000","last_became_active":"2023-06-10T20:59:25.411853+0000","last_became_peered":"2023-06-10T20:59:25.411853+0000","last_unstale":"2023-06-10T20:59:28.808163+0000","last_undegraded":"2023-06-10T20:59:28.808163+0000","last_fullsized":"2023-06-10T20:59:28.808163+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655257+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T20:59:26.655257+0000","last_peered":"2023-06-10T20:59:26.655257+0000","last_clean":"2023-06-10T20:59:26.655257+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T20:59:26.655257+0000","last_undegraded":"2023-06-10T20:59:26.655257+0000","last_fullsized":"2023-06-10T20:59:26.655257+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808370+0000","last_change":"2023-06-10T20:59:25.412194+0000","last_active":"2023-06-10T20:59:28.808370+0000","last_peered":"2023-06-10T20:59:28.808370+0000","last_clean":"2023-06-10T20:59:28.808370+0000","last_became_active":"2023-06-10T20:59:25.411886+0000","last_became_peered":"2023-06-10T20:59:25.411886+0000","last_unstale":"2023-06-10T20:59:28.808370+0000","last_undegraded":"2023-06-10T20:59:28.808370+0000","last_fullsized":"2023-06-10T20:59:28.808370+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421617+0000","last_change":"2023-06-10T20:58:34.376483+0000","last_active":"2023-06-10T20:59:26.421617+0000","last_peered":"2023-06-10T20:59:26.421617+0000","last_clean":"2023-06-10T20:59:26.421617+0000","last_became_active":"2023-06-10T20:58:34.376121+0000","last_became_peered":"2023-06-10T20:58:34.376121+0000","last_unstale":"2023-06-10T20:59:26.421617+0000","last_undegraded":"2023-06-10T20:59:26.421617+0000","last_fullsized":"2023-06-10T20:59:26.421617+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423469+0000","last_change":"2023-06-10T20:59:25.930882+0000","last_active":"2023-06-10T20:59:26.423469+0000","last_peered":"2023-06-10T20:59:26.423469+0000","last_clean":"2023-06-10T20:59:26.423469+0000","last_became_active":"2023-06-10T20:59:25.930538+0000","last_became_peered":"2023-06-10T20:59:25.930538+0000","last_unstale":"2023-06-10T20:59:26.423469+0000","last_undegraded":"2023-06-10T20:59:26.423469+0000","last_fullsized":"2023-06-10T20:59:26.423469+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421323+0000","last_change":"2023-06-10T20:58:34.481072+0000","last_active":"2023-06-10T20:59:26.421323+0000","last_peered":"2023-06-10T20:59:26.421323+0000","last_clean":"2023-06-10T20:59:26.421323+0000","last_became_active":"2023-06-10T20:58:34.480835+0000","last_became_peered":"2023-06-10T20:58:34.480835+0000","last_unstale":"2023-06-10T20:59:26.421323+0000","last_undegraded":"2023-06-10T20:59:26.421323+0000","last_fullsized":"2023-06-10T20:59:26.421323+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.025408+0000","last_change":"2023-06-10T20:59:25.407671+0000","last_active":"2023-06-10T20:59:38.025408+0000","last_peered":"2023-06-10T20:59:38.025408+0000","last_clean":"2023-06-10T20:59:38.025408+0000","last_became_active":"2023-06-10T20:59:25.407266+0000","last_became_peered":"2023-06-10T20:59:25.407266+0000","last_unstale":"2023-06-10T20:59:38.025408+0000","last_undegraded":"2023-06-10T20:59:38.025408+0000","last_fullsized":"2023-06-10T20:59:38.025408+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653646+0000","last_change":"2023-06-10T20:59:25.930871+0000","last_active":"2023-06-10T20:59:26.653646+0000","last_peered":"2023-06-10T20:59:26.653646+0000","last_clean":"2023-06-10T20:59:26.653646+0000","last_became_active":"2023-06-10T20:59:25.930516+0000","last_became_peered":"2023-06-10T20:59:25.930516+0000","last_unstale":"2023-06-10T20:59:26.653646+0000","last_undegraded":"2023-06-10T20:59:26.653646+0000","last_fullsized":"2023-06-10T20:59:26.653646+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421395+0000","last_change":"2023-06-10T20:59:25.411265+0000","last_active":"2023-06-10T20:59:26.421395+0000","last_peered":"2023-06-10T20:59:26.421395+0000","last_clean":"2023-06-10T20:59:26.421395+0000","last_became_active":"2023-06-10T20:59:25.410947+0000","last_became_peered":"2023-06-10T20:59:25.410947+0000","last_unstale":"2023-06-10T20:59:26.421395+0000","last_undegraded":"2023-06-10T20:59:26.421395+0000","last_fullsized":"2023-06-10T20:59:26.421395+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655287+0000","last_change":"2023-06-10T20:58:35.246897+0000","last_active":"2023-06-10T20:59:26.655287+0000","last_peered":"2023-06-10T20:59:26.655287+0000","last_clean":"2023-06-10T20:59:26.655287+0000","last_became_active":"2023-06-10T20:58:35.246501+0000","last_became_peered":"2023-06-10T20:58:35.246501+0000","last_unstale":"2023-06-10T20:59:26.655287+0000","last_undegraded":"2023-06-10T20:59:26.655287+0000","last_fullsized":"2023-06-10T20:59:26.655287+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654113+0000","last_change":"2023-06-10T20:59:25.432741+0000","last_active":"2023-06-10T20:59:26.654113+0000","last_peered":"2023-06-10T20:59:26.654113+0000","last_clean":"2023-06-10T20:59:26.654113+0000","last_became_active":"2023-06-10T20:59:25.432445+0000","last_became_peered":"2023-06-10T20:59:25.432445+0000","last_unstale":"2023-06-10T20:59:26.654113+0000","last_undegraded":"2023-06-10T20:59:26.654113+0000","last_fullsized":"2023-06-10T20:59:26.654113+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654179+0000","last_change":"2023-06-10T20:59:25.411497+0000","last_active":"2023-06-10T20:59:26.654179+0000","last_peered":"2023-06-10T20:59:26.654179+0000","last_clean":"2023-06-10T20:59:26.654179+0000","last_became_active":"2023-06-10T20:59:25.411149+0000","last_became_peered":"2023-06-10T20:59:25.411149+0000","last_unstale":"2023-06-10T20:59:26.654179+0000","last_undegraded":"2023-06-10T20:59:26.654179+0000","last_fullsized":"2023-06-10T20:59:26.654179+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654970+0000","last_change":"2023-06-10T20:58:35.246773+0000","last_active":"2023-06-10T20:59:26.654970+0000","last_peered":"2023-06-10T20:59:26.654970+0000","last_clean":"2023-06-10T20:59:26.654970+0000","last_became_active":"2023-06-10T20:58:35.246379+0000","last_became_peered":"2023-06-10T20:58:35.246379+0000","last_unstale":"2023-06-10T20:59:26.654970+0000","last_undegraded":"2023-06-10T20:59:26.654970+0000","last_fullsized":"2023-06-10T20:59:26.654970+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.658012+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T20:59:26.658012+0000","last_peered":"2023-06-10T20:59:26.658012+0000","last_clean":"2023-06-10T20:59:26.658012+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T20:59:26.658012+0000","last_undegraded":"2023-06-10T20:59:26.658012+0000","last_fullsized":"2023-06-10T20:59:26.658012+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423764+0000","last_change":"2023-06-10T20:59:25.409300+0000","last_active":"2023-06-10T20:59:26.423764+0000","last_peered":"2023-06-10T20:59:26.423764+0000","last_clean":"2023-06-10T20:59:26.423764+0000","last_became_active":"2023-06-10T20:59:25.408999+0000","last_became_peered":"2023-06-10T20:59:25.408999+0000","last_unstale":"2023-06-10T20:59:26.423764+0000","last_undegraded":"2023-06-10T20:59:26.423764+0000","last_fullsized":"2023-06-10T20:59:26.423764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654887+0000","last_change":"2023-06-10T20:57:57.149158+0000","last_active":"2023-06-10T20:59:26.654887+0000","last_peered":"2023-06-10T20:59:26.654887+0000","last_clean":"2023-06-10T20:59:26.654887+0000","last_became_active":"2023-06-10T20:57:57.148833+0000","last_became_peered":"2023-06-10T20:57:57.148833+0000","last_unstale":"2023-06-10T20:59:26.654887+0000","last_undegraded":"2023-06-10T20:59:26.654887+0000","last_fullsized":"2023-06-10T20:59:26.654887+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654285+0000","last_change":"2023-06-10T20:58:34.376607+0000","last_active":"2023-06-10T20:59:26.654285+0000","last_peered":"2023-06-10T20:59:26.654285+0000","last_clean":"2023-06-10T20:59:26.654285+0000","last_became_active":"2023-06-10T20:58:34.376302+0000","last_became_peered":"2023-06-10T20:58:34.376302+0000","last_unstale":"2023-06-10T20:59:26.654285+0000","last_undegraded":"2023-06-10T20:59:26.654285+0000","last_fullsized":"2023-06-10T20:59:26.654285+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.422223+0000","last_change":"2023-06-10T20:59:25.435968+0000","last_active":"2023-06-10T20:59:26.422223+0000","last_peered":"2023-06-10T20:59:26.422223+0000","last_clean":"2023-06-10T20:59:26.422223+0000","last_became_active":"2023-06-10T20:59:25.435664+0000","last_became_peered":"2023-06-10T20:59:25.435664+0000","last_unstale":"2023-06-10T20:59:26.422223+0000","last_undegraded":"2023-06-10T20:59:26.422223+0000","last_fullsized":"2023-06-10T20:59:26.422223+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420887+0000","last_change":"2023-06-10T20:59:25.408598+0000","last_active":"2023-06-10T20:59:26.420887+0000","last_peered":"2023-06-10T20:59:26.420887+0000","last_clean":"2023-06-10T20:59:26.420887+0000","last_became_active":"2023-06-10T20:59:25.408257+0000","last_became_peered":"2023-06-10T20:59:25.408257+0000","last_unstale":"2023-06-10T20:59:26.420887+0000","last_undegraded":"2023-06-10T20:59:26.420887+0000","last_fullsized":"2023-06-10T20:59:26.420887+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653445+0000","last_change":"2023-06-10T20:58:35.246831+0000","last_active":"2023-06-10T20:59:26.653445+0000","last_peered":"2023-06-10T20:59:26.653445+0000","last_clean":"2023-06-10T20:59:26.653445+0000","last_became_active":"2023-06-10T20:58:35.246208+0000","last_became_peered":"2023-06-10T20:58:35.246208+0000","last_unstale":"2023-06-10T20:59:26.653445+0000","last_undegraded":"2023-06-10T20:59:26.653445+0000","last_fullsized":"2023-06-10T20:59:26.653445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421296+0000","last_change":"2023-06-10T20:59:25.433487+0000","last_active":"2023-06-10T20:59:26.421296+0000","last_peered":"2023-06-10T20:59:26.421296+0000","last_clean":"2023-06-10T20:59:26.421296+0000","last_became_active":"2023-06-10T20:59:25.432968+0000","last_became_peered":"2023-06-10T20:59:25.432968+0000","last_unstale":"2023-06-10T20:59:26.421296+0000","last_undegraded":"2023-06-10T20:59:26.421296+0000","last_fullsized":"2023-06-10T20:59:26.421296+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655792+0000","last_change":"2023-06-10T20:59:25.412473+0000","last_active":"2023-06-10T20:59:26.655792+0000","last_peered":"2023-06-10T20:59:26.655792+0000","last_clean":"2023-06-10T20:59:26.655792+0000","last_became_active":"2023-06-10T20:59:25.412173+0000","last_became_peered":"2023-06-10T20:59:25.412173+0000","last_unstale":"2023-06-10T20:59:26.655792+0000","last_undegraded":"2023-06-10T20:59:26.655792+0000","last_fullsized":"2023-06-10T20:59:26.655792+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654267+0000","last_change":"2023-06-10T20:58:34.376802+0000","last_active":"2023-06-10T20:59:26.654267+0000","last_peered":"2023-06-10T20:59:26.654267+0000","last_clean":"2023-06-10T20:59:26.654267+0000","last_became_active":"2023-06-10T20:58:34.376449+0000","last_became_peered":"2023-06-10T20:58:34.376449+0000","last_unstale":"2023-06-10T20:59:26.654267+0000","last_undegraded":"2023-06-10T20:59:26.654267+0000","last_fullsized":"2023-06-10T20:59:26.654267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421081+0000","last_change":"2023-06-10T20:59:25.430136+0000","last_active":"2023-06-10T20:59:26.421081+0000","last_peered":"2023-06-10T20:59:26.421081+0000","last_clean":"2023-06-10T20:59:26.421081+0000","last_became_active":"2023-06-10T20:59:25.429815+0000","last_became_peered":"2023-06-10T20:59:25.429815+0000","last_unstale":"2023-06-10T20:59:26.421081+0000","last_undegraded":"2023-06-10T20:59:26.421081+0000","last_fullsized":"2023-06-10T20:59:26.421081+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655336+0000","last_change":"2023-06-10T20:59:25.408917+0000","last_active":"2023-06-10T20:59:26.655336+0000","last_peered":"2023-06-10T20:59:26.655336+0000","last_clean":"2023-06-10T20:59:26.655336+0000","last_became_active":"2023-06-10T20:59:25.408502+0000","last_became_peered":"2023-06-10T20:59:25.408502+0000","last_unstale":"2023-06-10T20:59:26.655336+0000","last_undegraded":"2023-06-10T20:59:26.655336+0000","last_fullsized":"2023-06-10T20:59:26.655336+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421214+0000","last_change":"2023-06-10T20:58:35.246694+0000","last_active":"2023-06-10T20:59:26.421214+0000","last_peered":"2023-06-10T20:59:26.421214+0000","last_clean":"2023-06-10T20:59:26.421214+0000","last_became_active":"2023-06-10T20:58:35.246275+0000","last_became_peered":"2023-06-10T20:58:35.246275+0000","last_unstale":"2023-06-10T20:59:26.421214+0000","last_undegraded":"2023-06-10T20:59:26.421214+0000","last_fullsized":"2023-06-10T20:59:26.421214+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.422161+0000","last_change":"2023-06-10T20:59:25.431108+0000","last_active":"2023-06-10T20:59:26.422161+0000","last_peered":"2023-06-10T20:59:26.422161+0000","last_clean":"2023-06-10T20:59:26.422161+0000","last_became_active":"2023-06-10T20:59:25.430328+0000","last_became_peered":"2023-06-10T20:59:25.430328+0000","last_unstale":"2023-06-10T20:59:26.422161+0000","last_undegraded":"2023-06-10T20:59:26.422161+0000","last_fullsized":"2023-06-10T20:59:26.422161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423492+0000","last_change":"2023-06-10T20:59:25.410112+0000","last_active":"2023-06-10T20:59:26.423492+0000","last_peered":"2023-06-10T20:59:26.423492+0000","last_clean":"2023-06-10T20:59:26.423492+0000","last_became_active":"2023-06-10T20:59:25.409782+0000","last_became_peered":"2023-06-10T20:59:25.409782+0000","last_unstale":"2023-06-10T20:59:26.423492+0000","last_undegraded":"2023-06-10T20:59:26.423492+0000","last_fullsized":"2023-06-10T20:59:26.423492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655104+0000","last_change":"2023-06-10T20:59:26.106380+0000","last_active":"2023-06-10T20:59:26.655104+0000","last_peered":"2023-06-10T20:59:26.655104+0000","last_clean":"2023-06-10T20:59:26.655104+0000","last_became_active":"2023-06-10T20:59:26.106011+0000","last_became_peered":"2023-06-10T20:59:26.106011+0000","last_unstale":"2023-06-10T20:59:26.655104+0000","last_undegraded":"2023-06-10T20:59:26.655104+0000","last_fullsized":"2023-06-10T20:59:26.655104+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655308+0000","last_change":"2023-06-10T20:59:25.409183+0000","last_active":"2023-06-10T20:59:26.655308+0000","last_peered":"2023-06-10T20:59:26.655308+0000","last_clean":"2023-06-10T20:59:26.655308+0000","last_became_active":"2023-06-10T20:59:25.408876+0000","last_became_peered":"2023-06-10T20:59:25.408876+0000","last_unstale":"2023-06-10T20:59:26.655308+0000","last_undegraded":"2023-06-10T20:59:26.655308+0000","last_fullsized":"2023-06-10T20:59:26.655308+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655561+0000","last_change":"2023-06-10T20:59:25.431005+0000","last_active":"2023-06-10T20:59:26.655561+0000","last_peered":"2023-06-10T20:59:26.655561+0000","last_clean":"2023-06-10T20:59:26.655561+0000","last_became_active":"2023-06-10T20:59:25.430701+0000","last_became_peered":"2023-06-10T20:59:25.430701+0000","last_unstale":"2023-06-10T20:59:26.655561+0000","last_undegraded":"2023-06-10T20:59:26.655561+0000","last_fullsized":"2023-06-10T20:59:26.655561+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421606+0000","last_change":"2023-06-10T20:59:25.408524+0000","last_active":"2023-06-10T20:59:26.421606+0000","last_peered":"2023-06-10T20:59:26.421606+0000","last_clean":"2023-06-10T20:59:26.421606+0000","last_became_active":"2023-06-10T20:59:25.408052+0000","last_became_peered":"2023-06-10T20:59:25.408052+0000","last_unstale":"2023-06-10T20:59:26.421606+0000","last_undegraded":"2023-06-10T20:59:26.421606+0000","last_fullsized":"2023-06-10T20:59:26.421606+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654835+0000","last_change":"2023-06-10T20:59:26.107188+0000","last_active":"2023-06-10T20:59:26.654835+0000","last_peered":"2023-06-10T20:59:26.654835+0000","last_clean":"2023-06-10T20:59:26.654835+0000","last_became_active":"2023-06-10T20:59:26.106822+0000","last_became_peered":"2023-06-10T20:59:26.106822+0000","last_unstale":"2023-06-10T20:59:26.654835+0000","last_undegraded":"2023-06-10T20:59:26.654835+0000","last_fullsized":"2023-06-10T20:59:26.654835+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.026573+0000","last_change":"2023-06-10T20:59:25.411558+0000","last_active":"2023-06-10T20:59:38.026573+0000","last_peered":"2023-06-10T20:59:38.026573+0000","last_clean":"2023-06-10T20:59:38.026573+0000","last_became_active":"2023-06-10T20:59:25.411214+0000","last_became_peered":"2023-06-10T20:59:25.411214+0000","last_unstale":"2023-06-10T20:59:38.026573+0000","last_undegraded":"2023-06-10T20:59:38.026573+0000","last_fullsized":"2023-06-10T20:59:38.026573+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653540+0000","last_change":"2023-06-10T20:59:25.434771+0000","last_active":"2023-06-10T20:59:26.653540+0000","last_peered":"2023-06-10T20:59:26.653540+0000","last_clean":"2023-06-10T20:59:26.653540+0000","last_became_active":"2023-06-10T20:59:25.432415+0000","last_became_peered":"2023-06-10T20:59:25.432415+0000","last_unstale":"2023-06-10T20:59:26.653540+0000","last_undegraded":"2023-06-10T20:59:26.653540+0000","last_fullsized":"2023-06-10T20:59:26.653540+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654511+0000","last_change":"2023-06-10T20:59:25.408811+0000","last_active":"2023-06-10T20:59:26.654511+0000","last_peered":"2023-06-10T20:59:26.654511+0000","last_clean":"2023-06-10T20:59:26.654511+0000","last_became_active":"2023-06-10T20:59:25.408478+0000","last_became_peered":"2023-06-10T20:59:25.408478+0000","last_unstale":"2023-06-10T20:59:26.654511+0000","last_undegraded":"2023-06-10T20:59:26.654511+0000","last_fullsized":"2023-06-10T20:59:26.654511+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.657590+0000","last_change":"2023-06-10T20:59:26.656990+0000","last_active":"2023-06-10T20:59:26.657590+0000","last_peered":"2023-06-10T20:59:26.657590+0000","last_clean":"2023-06-10T20:59:26.657590+0000","last_became_active":"2023-06-10T20:59:26.656676+0000","last_became_peered":"2023-06-10T20:59:26.656676+0000","last_unstale":"2023-06-10T20:59:26.657590+0000","last_undegraded":"2023-06-10T20:59:26.657590+0000","last_fullsized":"2023-06-10T20:59:26.657590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808402+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T20:59:28.808402+0000","last_peered":"2023-06-10T20:59:28.808402+0000","last_clean":"2023-06-10T20:59:28.808402+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T20:59:28.808402+0000","last_undegraded":"2023-06-10T20:59:28.808402+0000","last_fullsized":"2023-06-10T20:59:28.808402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655172+0000","last_change":"2023-06-10T20:59:25.431587+0000","last_active":"2023-06-10T20:59:26.655172+0000","last_peered":"2023-06-10T20:59:26.655172+0000","last_clean":"2023-06-10T20:59:26.655172+0000","last_became_active":"2023-06-10T20:59:25.431318+0000","last_became_peered":"2023-06-10T20:59:25.431318+0000","last_unstale":"2023-06-10T20:59:26.655172+0000","last_undegraded":"2023-06-10T20:59:26.655172+0000","last_fullsized":"2023-06-10T20:59:26.655172+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028700+0000","last_change":"2023-06-10T20:59:25.412893+0000","last_active":"2023-06-10T20:59:38.028700+0000","last_peered":"2023-06-10T20:59:38.028700+0000","last_clean":"2023-06-10T20:59:38.028700+0000","last_became_active":"2023-06-10T20:59:25.412557+0000","last_became_peered":"2023-06-10T20:59:25.412557+0000","last_unstale":"2023-06-10T20:59:38.028700+0000","last_undegraded":"2023-06-10T20:59:38.028700+0000","last_fullsized":"2023-06-10T20:59:38.028700+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654786+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T20:59:26.654786+0000","last_peered":"2023-06-10T20:59:26.654786+0000","last_clean":"2023-06-10T20:59:26.654786+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T20:59:26.654786+0000","last_undegraded":"2023-06-10T20:59:26.654786+0000","last_fullsized":"2023-06-10T20:59:26.654786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808422+0000","last_change":"2023-06-10T20:59:25.415629+0000","last_active":"2023-06-10T20:59:28.808422+0000","last_peered":"2023-06-10T20:59:28.808422+0000","last_clean":"2023-06-10T20:59:28.808422+0000","last_became_active":"2023-06-10T20:59:25.415333+0000","last_became_peered":"2023-06-10T20:59:25.415333+0000","last_unstale":"2023-06-10T20:59:28.808422+0000","last_undegraded":"2023-06-10T20:59:28.808422+0000","last_fullsized":"2023-06-10T20:59:28.808422+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.424182+0000","last_change":"2023-06-10T20:59:25.930923+0000","last_active":"2023-06-10T20:59:26.424182+0000","last_peered":"2023-06-10T20:59:26.424182+0000","last_clean":"2023-06-10T20:59:26.424182+0000","last_became_active":"2023-06-10T20:59:25.930583+0000","last_became_peered":"2023-06-10T20:59:25.930583+0000","last_unstale":"2023-06-10T20:59:26.424182+0000","last_undegraded":"2023-06-10T20:59:26.424182+0000","last_fullsized":"2023-06-10T20:59:26.424182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028959+0000","last_change":"2023-06-10T20:59:25.404800+0000","last_active":"2023-06-10T20:59:38.028959+0000","last_peered":"2023-06-10T20:59:38.028959+0000","last_clean":"2023-06-10T20:59:38.028959+0000","last_became_active":"2023-06-10T20:59:25.404428+0000","last_became_peered":"2023-06-10T20:59:25.404428+0000","last_unstale":"2023-06-10T20:59:38.028959+0000","last_undegraded":"2023-06-10T20:59:38.028959+0000","last_fullsized":"2023-06-10T20:59:38.028959+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653896+0000","last_change":"2023-06-10T20:59:25.930888+0000","last_active":"2023-06-10T20:59:26.653896+0000","last_peered":"2023-06-10T20:59:26.653896+0000","last_clean":"2023-06-10T20:59:26.653896+0000","last_became_active":"2023-06-10T20:59:25.930539+0000","last_became_peered":"2023-06-10T20:59:25.930539+0000","last_unstale":"2023-06-10T20:59:26.653896+0000","last_undegraded":"2023-06-10T20:59:26.653896+0000","last_fullsized":"2023-06-10T20:59:26.653896+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.656007+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T20:59:26.656007+0000","last_peered":"2023-06-10T20:59:26.656007+0000","last_clean":"2023-06-10T20:59:26.656007+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T20:59:26.656007+0000","last_undegraded":"2023-06-10T20:59:26.656007+0000","last_fullsized":"2023-06-10T20:59:26.656007+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655260+0000","last_change":"2023-06-10T20:59:25.930888+0000","last_active":"2023-06-10T20:59:26.655260+0000","last_peered":"2023-06-10T20:59:26.655260+0000","last_clean":"2023-06-10T20:59:26.655260+0000","last_became_active":"2023-06-10T20:59:25.930503+0000","last_became_peered":"2023-06-10T20:59:25.930503+0000","last_unstale":"2023-06-10T20:59:26.655260+0000","last_undegraded":"2023-06-10T20:59:26.655260+0000","last_fullsized":"2023-06-10T20:59:26.655260+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653617+0000","last_change":"2023-06-10T20:59:25.404676+0000","last_active":"2023-06-10T20:59:26.653617+0000","last_peered":"2023-06-10T20:59:26.653617+0000","last_clean":"2023-06-10T20:59:26.653617+0000","last_became_active":"2023-06-10T20:59:25.404305+0000","last_became_peered":"2023-06-10T20:59:25.404305+0000","last_unstale":"2023-06-10T20:59:26.653617+0000","last_undegraded":"2023-06-10T20:59:26.653617+0000","last_fullsized":"2023-06-10T20:59:26.653617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423873+0000","last_change":"2023-06-10T20:59:25.930933+0000","last_active":"2023-06-10T20:59:26.423873+0000","last_peered":"2023-06-10T20:59:26.423873+0000","last_clean":"2023-06-10T20:59:26.423873+0000","last_became_active":"2023-06-10T20:59:25.930600+0000","last_became_peered":"2023-06-10T20:59:25.930600+0000","last_unstale":"2023-06-10T20:59:26.423873+0000","last_undegraded":"2023-06-10T20:59:26.423873+0000","last_fullsized":"2023-06-10T20:59:26.423873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028696+0000","last_change":"2023-06-10T20:59:25.410958+0000","last_active":"2023-06-10T20:59:38.028696+0000","last_peered":"2023-06-10T20:59:38.028696+0000","last_clean":"2023-06-10T20:59:38.028696+0000","last_became_active":"2023-06-10T20:59:25.410668+0000","last_became_peered":"2023-06-10T20:59:25.410668+0000","last_unstale":"2023-06-10T20:59:38.028696+0000","last_undegraded":"2023-06-10T20:59:38.028696+0000","last_fullsized":"2023-06-10T20:59:38.028696+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655610+0000","last_change":"2023-06-10T20:59:26.106185+0000","last_active":"2023-06-10T20:59:26.655610+0000","last_peered":"2023-06-10T20:59:26.655610+0000","last_clean":"2023-06-10T20:59:26.655610+0000","last_became_active":"2023-06-10T20:59:26.105815+0000","last_became_peered":"2023-06-10T20:59:26.105815+0000","last_unstale":"2023-06-10T20:59:26.655610+0000","last_undegraded":"2023-06-10T20:59:26.655610+0000","last_fullsized":"2023-06-10T20:59:26.655610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029994+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T20:59:38.029994+0000","last_peered":"2023-06-10T20:59:38.029994+0000","last_clean":"2023-06-10T20:59:38.029994+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T20:59:38.029994+0000","last_undegraded":"2023-06-10T20:59:38.029994+0000","last_fullsized":"2023-06-10T20:59:38.029994+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420999+0000","last_change":"2023-06-10T20:59:26.105976+0000","last_active":"2023-06-10T20:59:26.420999+0000","last_peered":"2023-06-10T20:59:26.420999+0000","last_clean":"2023-06-10T20:59:26.420999+0000","last_became_active":"2023-06-10T20:59:26.105591+0000","last_became_peered":"2023-06-10T20:59:26.105591+0000","last_unstale":"2023-06-10T20:59:26.420999+0000","last_undegraded":"2023-06-10T20:59:26.420999+0000","last_fullsized":"2023-06-10T20:59:26.420999+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029396+0000","last_change":"2023-06-10T20:59:25.410259+0000","last_active":"2023-06-10T20:59:38.029396+0000","last_peered":"2023-06-10T20:59:38.029396+0000","last_clean":"2023-06-10T20:59:38.029396+0000","last_became_active":"2023-06-10T20:59:25.409862+0000","last_became_peered":"2023-06-10T20:59:25.409862+0000","last_unstale":"2023-06-10T20:59:38.029396+0000","last_undegraded":"2023-06-10T20:59:38.029396+0000","last_fullsized":"2023-06-10T20:59:38.029396+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423711+0000","last_change":"2023-06-10T20:59:26.106353+0000","last_active":"2023-06-10T20:59:26.423711+0000","last_peered":"2023-06-10T20:59:26.423711+0000","last_clean":"2023-06-10T20:59:26.423711+0000","last_became_active":"2023-06-10T20:59:26.105974+0000","last_became_peered":"2023-06-10T20:59:26.105974+0000","last_unstale":"2023-06-10T20:59:26.423711+0000","last_undegraded":"2023-06-10T20:59:26.423711+0000","last_fullsized":"2023-06-10T20:59:26.423711+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654434+0000","last_change":"2023-06-10T20:59:25.404046+0000","last_active":"2023-06-10T20:59:26.654434+0000","last_peered":"2023-06-10T20:59:26.654434+0000","last_clean":"2023-06-10T20:59:26.654434+0000","last_became_active":"2023-06-10T20:59:25.403707+0000","last_became_peered":"2023-06-10T20:59:25.403707+0000","last_unstale":"2023-06-10T20:59:26.654434+0000","last_undegraded":"2023-06-10T20:59:26.654434+0000","last_fullsized":"2023-06-10T20:59:26.654434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423560+0000","last_change":"2023-06-10T20:59:26.107602+0000","last_active":"2023-06-10T20:59:26.423560+0000","last_peered":"2023-06-10T20:59:26.423560+0000","last_clean":"2023-06-10T20:59:26.423560+0000","last_became_active":"2023-06-10T20:59:26.107262+0000","last_became_peered":"2023-06-10T20:59:26.107262+0000","last_unstale":"2023-06-10T20:59:26.423560+0000","last_undegraded":"2023-06-10T20:59:26.423560+0000","last_fullsized":"2023-06-10T20:59:26.423560+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654058+0000","last_change":"2023-06-10T20:59:25.413422+0000","last_active":"2023-06-10T20:59:26.654058+0000","last_peered":"2023-06-10T20:59:26.654058+0000","last_clean":"2023-06-10T20:59:26.654058+0000","last_became_active":"2023-06-10T20:59:25.412850+0000","last_became_peered":"2023-06-10T20:59:25.412850+0000","last_unstale":"2023-06-10T20:59:26.654058+0000","last_undegraded":"2023-06-10T20:59:26.654058+0000","last_fullsized":"2023-06-10T20:59:26.654058+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.657444+0000","last_change":"2023-06-10T20:59:26.656575+0000","last_active":"2023-06-10T20:59:26.657444+0000","last_peered":"2023-06-10T20:59:26.657444+0000","last_clean":"2023-06-10T20:59:26.657444+0000","last_became_active":"2023-06-10T20:59:26.656167+0000","last_became_peered":"2023-06-10T20:59:26.656167+0000","last_unstale":"2023-06-10T20:59:26.657444+0000","last_undegraded":"2023-06-10T20:59:26.657444+0000","last_fullsized":"2023-06-10T20:59:26.657444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423589+0000","last_change":"2023-06-10T20:59:25.405537+0000","last_active":"2023-06-10T20:59:26.423589+0000","last_peered":"2023-06-10T20:59:26.423589+0000","last_clean":"2023-06-10T20:59:26.423589+0000","last_became_active":"2023-06-10T20:59:25.405233+0000","last_became_peered":"2023-06-10T20:59:25.405233+0000","last_unstale":"2023-06-10T20:59:26.423589+0000","last_undegraded":"2023-06-10T20:59:26.423589+0000","last_fullsized":"2023-06-10T20:59:26.423589+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653984+0000","last_change":"2023-06-10T20:59:25.931230+0000","last_active":"2023-06-10T20:59:26.653984+0000","last_peered":"2023-06-10T20:59:26.653984+0000","last_clean":"2023-06-10T20:59:26.653984+0000","last_became_active":"2023-06-10T20:59:25.930826+0000","last_became_peered":"2023-06-10T20:59:25.930826+0000","last_unstale":"2023-06-10T20:59:26.653984+0000","last_undegraded":"2023-06-10T20:59:26.653984+0000","last_fullsized":"2023-06-10T20:59:26.653984+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420912+0000","last_change":"2023-06-10T20:59:25.411727+0000","last_active":"2023-06-10T20:59:26.420912+0000","last_peered":"2023-06-10T20:59:26.420912+0000","last_clean":"2023-06-10T20:59:26.420912+0000","last_became_active":"2023-06-10T20:59:25.411353+0000","last_became_peered":"2023-06-10T20:59:25.411353+0000","last_unstale":"2023-06-10T20:59:26.420912+0000","last_undegraded":"2023-06-10T20:59:26.420912+0000","last_fullsized":"2023-06-10T20:59:26.420912+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.424097+0000","last_change":"2023-06-10T20:59:25.435781+0000","last_active":"2023-06-10T20:59:26.424097+0000","last_peered":"2023-06-10T20:59:26.424097+0000","last_clean":"2023-06-10T20:59:26.424097+0000","last_became_active":"2023-06-10T20:59:25.435478+0000","last_became_peered":"2023-06-10T20:59:25.435478+0000","last_unstale":"2023-06-10T20:59:26.424097+0000","last_undegraded":"2023-06-10T20:59:26.424097+0000","last_fullsized":"2023-06-10T20:59:26.424097+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:27.427094+0000","last_change":"2023-06-10T20:59:25.405915+0000","last_active":"2023-06-10T20:59:27.427094+0000","last_peered":"2023-06-10T20:59:27.427094+0000","last_clean":"2023-06-10T20:59:27.427094+0000","last_became_active":"2023-06-10T20:59:25.405233+0000","last_became_peered":"2023-06-10T20:59:25.405233+0000","last_unstale":"2023-06-10T20:59:27.427094+0000","last_undegraded":"2023-06-10T20:59:27.427094+0000","last_fullsized":"2023-06-10T20:59:27.427094+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430237,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":630784,"data_stored":257887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.58599999999999997,"15min":0.55100000000000005},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0140000000000002,"5min":6.0140000000000002,"15min":6.0140000000000002},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.60199999999999998,"15min":0.56000000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.3019999999999996,"5min":7.3019999999999996,"15min":7.3019999999999996},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.54600000000000004,"15min":0.52300000000000002},"min":{"1min":0.439,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.50800000000000001}]},{"osd":2,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":0.65900000000000003,"15min":0.58999999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":8.9949999999999992,"5min":8.9949999999999992,"15min":8.9949999999999992},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.63900000000000001,"15min":0.57499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.8789999999999996,"5min":8.8789999999999996,"15min":8.8789999999999996},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":0.66400000000000003,"15min":0.58399999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.9489999999999998,"5min":7.9489999999999998,"15min":7.9489999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.61099999999999999,"15min":0.58499999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.016,"5min":5.016,"15min":5.016},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.51100000000000001,"15min":0.51900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64500000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.52400000000000002,"15min":0.53300000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71899999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.497}]},{"osd":5,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.51400000000000001,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69499999999999995,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57499999999999996},"min":{"1min":0.42599999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.73599999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.60399999999999998,"15min":0.61599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.54900000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626465,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":257426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.57999999999999996,"15min":0.59099999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67800000000000005,"5min":0.754,"15min":0.754},"last":0.66800000000000004}]},{"osd":1,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.753,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.55800000000000005,"15min":0.56699999999999995},"min":{"1min":0.32900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.72399999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":1.51},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.70799999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.76100000000000001}]},{"osd":3,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":1.444},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.42399999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.68500000000000005,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.69399999999999995}]},{"osd":4,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.44,"15min":0.432},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.45400000000000001,"15min":0.44800000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.63600000000000001}]},{"osd":5,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46300000000000002,"15min":0.45400000000000001},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.498,"15min":0.49399999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.68400000000000005}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79400000000000004,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.72099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574903,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5908,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739244,"statfs":{"total":95995035648,"available":95988985856,"internally_reserved":0,"allocated":602112,"data_stored":256135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.318},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.62,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.379},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.376,"15min":0.33400000000000002},"min":{"1min":0.26800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59199999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.498},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53700000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Jun 10 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.433,"15min":0.44700000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.49199999999999999,"15min":0.51000000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.441}]},{"osd":4,"last update":"Sat Jun 10 20:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.63,"15min":0.61399999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.593,"5min":4.593,"15min":4.593},"last":0.56399999999999995}]},{"osd":5,"last update":"Sat Jun 10 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42499999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Jun 10 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Jun 10 20:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63200000000000001,"15min":0.64100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.86399999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.439,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.59599999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738427,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5916,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739236,"statfs":{"total":95995035648,"available":95988977664,"internally_reserved":0,"allocated":610304,"data_stored":256711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":0.57799999999999996,"15min":0.434},"min":{"1min":0.26900000000000002,"5min":0.224,"15min":0.224},"max":{"1min":15.949999999999999,"5min":15.949999999999999,"15min":15.949999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.51600000000000001,"15min":0.42699999999999999},"min":{"1min":0.252,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.0500000000000007,"5min":9.0500000000000007,"15min":9.0500000000000007},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.67700000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46400000000000002,"15min":0.46500000000000002},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.72999999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Jun 10 20:59:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.58799999999999997,"15min":0.53400000000000003},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.128,"5min":11.128,"15min":11.128},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.3180000000000001,"5min":0.70699999999999996,"15min":0.61899999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":15.895,"5min":15.895,"15min":15.895},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:00:03 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.72599999999999998,"15min":0.69999999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.1900000000000004,"5min":5.1900000000000004,"15min":5.1900000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.879,"5min":0.71799999999999997,"15min":0.68999999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.9729999999999999,"5min":4.9729999999999999,"15min":4.9729999999999999},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.621,"15min":0.624},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.76200000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.85199999999999998,"5min":1.393,"15min":1.393},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Jun 10 20:59:39 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.69499999999999995,"15min":0.66200000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.212,"5min":0.751,"15min":0.67400000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":11.167,"5min":11.167,"15min":11.167},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:00:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.72799999999999998,"15min":0.70499999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":5.1520000000000001,"5min":5.1520000000000001,"15min":5.1520000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.74299999999999999,"15min":0.72199999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":5.0490000000000004},"last":0.62}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":256045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.39200000000000002,"15min":0.373},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.34200000000000003}]},{"osd":1,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.48599999999999999,"15min":0.48999999999999999},"min":{"1min":0.29199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.069,"5min":1.161,"15min":1.161},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.49099999999999999,"15min":0.505},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71399999999999997,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.749},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60399999999999998,"15min":0.60499999999999998},"min":{"1min":0.41699999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80500000000000005,"5min":1.27,"15min":1.27},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Jun 10 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.60199999999999998,"15min":0.59199999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.61599999999999999,"15min":0.61299999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.61399999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.68100000000000005,"15min":0.69899999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72799999999999998,"5min":1.254,"15min":1.254},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.66100000000000003,"15min":0.67000000000000004},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.84099999999999997,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.80300000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:00:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.622,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.46400000000000002,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.78000000000000003,"5min":1.075,"15min":1.075},"last":0.623}]}]},{"osd":3,"up_from":25,"seq":107374182446,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":256225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.38500000000000001,"15min":0.39500000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.60099999999999998,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.377,"5min":0.42299999999999999,"15min":0.443},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.59599999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46400000000000002,"15min":0.47799999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66400000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Jun 10 20:59:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57099999999999995,"15min":0.56899999999999995},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.81499999999999995,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Jun 10 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.62,"15min":0.622},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81299999999999994,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Jun 10 20:59:40 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.65600000000000003,"15min":0.66600000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61399999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.753}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.64900000000000002,"15min":0.65900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.62}]}]},{"osd":4,"up_from":30,"seq":128849018921,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5800,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739352,"statfs":{"total":95995035648,"available":95989096448,"internally_reserved":0,"allocated":622592,"data_stored":257502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.53200000000000003,"15min":0.54600000000000004},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69799999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.497,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.38200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.70799999999999996}]},{"osd":1,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.52900000000000003,"15min":0.54300000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.64800000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.749},{"interface":"front","average":{"1min":0.502,"5min":0.54300000000000004,"15min":0.55300000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68799999999999994,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.57699999999999996,"15min":0.55900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.4550000000000001,"5min":4.4550000000000001,"15min":4.4550000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.499,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73799999999999999,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.68700000000000006}]},{"osd":3,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.55900000000000005,"15min":0.57099999999999995},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67300000000000004,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.60599999999999998,"15min":0.58999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.4649999999999999,"5min":4.4649999999999999,"15min":4.4649999999999999},"last":0.72299999999999998}]},{"osd":5,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.40600000000000003,"15min":0.40000000000000002},"min":{"1min":0.33400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.78300000000000003,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.443}]},{"osd":6,"last update":"Sat Jun 10 20:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46800000000000003,"15min":0.46300000000000002},"min":{"1min":0.308,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.5},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.443,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.746,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77500000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77000000000000002,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.46500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822693,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":256444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77700000000000002,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.617,"5min":1.617,"15min":1.617},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.752,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.435},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.36599999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45000000000000001,"15min":0.45100000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.44,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66800000000000004,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68600000000000005,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Jun 10 21:00:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77200000000000002,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.66700000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:00:09.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:00:10.158 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:09 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:09.889Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000717282s 2023-06-10T21:00:10.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:09 smithi064 conmon[107122]: audit 2023-06-10T21:00:09.320591+0000 mgr.y ( 2023-06-10T21:00:10.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:09 smithi064 conmon[107122]: mgr.14427) 85 : audit [DBG] from='client.14751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:09 smithi064 conmon[97224]: audit 2023-06-10T21:00:09. 2023-06-10T21:00:10.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:09 smithi064 conmon[97224]: 320591+0000 mgr.y (mgr.14427) 85 : audit [DBG] from='client.14751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:09 smithi152 conmon[97365]: audit 2023-06-10T21:00:09.320591+0000 mgr.y (mgr. 2023-06-10T21:00:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:09 smithi152 conmon[97365]: 14427) 85 : audit [DBG] from='client.14751 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:10.320 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:10.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 systemd[1]: Stopping Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:00:10.695 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 bash[131534]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-prometheus.a found: no such container 2023-06-10T21:00:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[97365]: cluster 2023-06-10T21:00:10.319301+0000 mgr.y (mgr.14427) 86 2023-06-10T21:00:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[97365]: : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:11.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:00:11.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-10T21:00:11.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-10T21:00:11.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-10T21:00:11.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-10T21:00:11.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.740Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-10T21:00:11.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-10T21:00:11.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-10T21:00:11.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-10T21:00:11.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-10T21:00:11.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-10T21:00:11.197 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:10 smithi152 conmon[126778]: ts=2023-06-10T21:00:10.741Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-10T21:00:11.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:10 smithi064 conmon[107122]: cluster 2023-06-10T21:00:10.319301 2023-06-10T21:00:11.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:10 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:11.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:10 smithi064 conmon[97224]: cluster 2023-06-10T21:00:10.319301+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:11.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:11 smithi152 bash[131650]: bc2b0c5cd82151b5e7909f19b04342084eac6859dfdcb8692f9410361b827e6c 2023-06-10T21:00:11.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:11 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-10T21:00:11.695 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:11 smithi152 systemd[1]: Stopped Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:00:11.695 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:11 smithi152 systemd[1]: Starting Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:00:11.986 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:00:11.987 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:00:12.336 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPING 2023-06-10T21:00:12.336 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:00:12.336 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPED 2023-06-10T21:00:12.336 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTING 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Serving on http://:::9283 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTED 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPING 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPED 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTING 2023-06-10T21:00:12.337 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Serving on http://:::9283 2023-06-10T21:00:12.338 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTED 2023-06-10T21:00:12.444 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.036Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.036Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.036Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi152 (none))" 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.036Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.036Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.038Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.039Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-10T21:00:12.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.040Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 bash[131978]: a4c5e6078a1577f28577b11f6a68a62df38b211ea20ef502995fae1c1a8ce5be 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.042Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.042Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.274µs 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.042Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 systemd[1]: Started Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.268Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-06-10T21:00:12.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.268Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.268Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=38.351µs wal_replay_duration=226.060524ms total_replay_duration=226.119804ms 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.270Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.270Z caller=main.go:947 level=info msg="TSDB started" 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.270Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.285Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.674716ms db_storage=720ns remote_storage=1.473µs web_handler=320ns query_engine=586ns scrape=426.902µs scrape_sd=24.796µs notify=27.357µs notify_sd=7.946µs rules=13.954965ms 2023-06-10T21:00:12.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:00:12 smithi152 conmon[132019]: ts=2023-06-10T21:00:12.285Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-10T21:00:12.577 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":70,"stamp":"2023-06-10T21:00:10.318941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":46128,"kb_used_data":4784,"kb_used_omap":0,"kb_used_meta":41280,"kb_avail":749915088,"statfs":{"total":767960285184,"available":767913050112,"internally_reserved":0,"allocated":4898816,"data_stored":2054375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42270720},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003196"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654540+0000","last_change":"2023-06-10T20:59:26.107674+0000","last_active":"2023-06-10T20:59:26.654540+0000","last_peered":"2023-06-10T20:59:26.654540+0000","last_clean":"2023-06-10T20:59:26.654540+0000","last_became_active":"2023-06-10T20:59:26.107295+0000","last_became_peered":"2023-06-10T20:59:26.107295+0000","last_unstale":"2023-06-10T20:59:26.654540+0000","last_undegraded":"2023-06-10T20:59:26.654540+0000","last_fullsized":"2023-06-10T20:59:26.654540+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":16,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.026194+0000","last_change":"2023-06-10T20:59:25.411031+0000","last_active":"2023-06-10T20:59:38.026194+0000","last_peered":"2023-06-10T20:59:38.026194+0000","last_clean":"2023-06-10T20:59:38.026194+0000","last_became_active":"2023-06-10T20:59:25.410713+0000","last_became_peered":"2023-06-10T20:59:25.410713+0000","last_unstale":"2023-06-10T20:59:38.026194+0000","last_undegraded":"2023-06-10T20:59:38.026194+0000","last_fullsized":"2023-06-10T20:59:38.026194+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423649+0000","last_change":"2023-06-10T20:59:25.435382+0000","last_active":"2023-06-10T20:59:26.423649+0000","last_peered":"2023-06-10T20:59:26.423649+0000","last_clean":"2023-06-10T20:59:26.423649+0000","last_became_active":"2023-06-10T20:59:25.435084+0000","last_became_peered":"2023-06-10T20:59:25.435084+0000","last_unstale":"2023-06-10T20:59:26.423649+0000","last_undegraded":"2023-06-10T20:59:26.423649+0000","last_fullsized":"2023-06-10T20:59:26.423649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:27.427652+0000","last_change":"2023-06-10T20:59:25.407062+0000","last_active":"2023-06-10T20:59:27.427652+0000","last_peered":"2023-06-10T20:59:27.427652+0000","last_clean":"2023-06-10T20:59:27.427652+0000","last_became_active":"2023-06-10T20:59:25.406722+0000","last_became_peered":"2023-06-10T20:59:25.406722+0000","last_unstale":"2023-06-10T20:59:27.427652+0000","last_undegraded":"2023-06-10T20:59:27.427652+0000","last_fullsized":"2023-06-10T20:59:27.427652+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654830+0000","last_change":"2023-06-10T20:59:25.428869+0000","last_active":"2023-06-10T20:59:26.654830+0000","last_peered":"2023-06-10T20:59:26.654830+0000","last_clean":"2023-06-10T20:59:26.654830+0000","last_became_active":"2023-06-10T20:59:25.428527+0000","last_became_peered":"2023-06-10T20:59:25.428527+0000","last_unstale":"2023-06-10T20:59:26.654830+0000","last_undegraded":"2023-06-10T20:59:26.654830+0000","last_fullsized":"2023-06-10T20:59:26.654830+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029303+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T20:59:38.029303+0000","last_peered":"2023-06-10T20:59:38.029303+0000","last_clean":"2023-06-10T20:59:38.029303+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T20:59:38.029303+0000","last_undegraded":"2023-06-10T20:59:38.029303+0000","last_fullsized":"2023-06-10T20:59:38.029303+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655872+0000","last_change":"2023-06-10T20:59:25.431228+0000","last_active":"2023-06-10T20:59:26.655872+0000","last_peered":"2023-06-10T20:59:26.655872+0000","last_clean":"2023-06-10T20:59:26.655872+0000","last_became_active":"2023-06-10T20:59:25.430927+0000","last_became_peered":"2023-06-10T20:59:25.430927+0000","last_unstale":"2023-06-10T20:59:26.655872+0000","last_undegraded":"2023-06-10T20:59:26.655872+0000","last_fullsized":"2023-06-10T20:59:26.655872+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.025712+0000","last_change":"2023-06-10T20:59:25.405538+0000","last_active":"2023-06-10T20:59:38.025712+0000","last_peered":"2023-06-10T20:59:38.025712+0000","last_clean":"2023-06-10T20:59:38.025712+0000","last_became_active":"2023-06-10T20:59:25.405084+0000","last_became_peered":"2023-06-10T20:59:25.405084+0000","last_unstale":"2023-06-10T20:59:38.025712+0000","last_undegraded":"2023-06-10T20:59:38.025712+0000","last_fullsized":"2023-06-10T20:59:38.025712+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653749+0000","last_change":"2023-06-10T20:59:25.432132+0000","last_active":"2023-06-10T20:59:26.653749+0000","last_peered":"2023-06-10T20:59:26.653749+0000","last_clean":"2023-06-10T20:59:26.653749+0000","last_became_active":"2023-06-10T20:59:25.431840+0000","last_became_peered":"2023-06-10T20:59:25.431840+0000","last_unstale":"2023-06-10T20:59:26.653749+0000","last_undegraded":"2023-06-10T20:59:26.653749+0000","last_fullsized":"2023-06-10T20:59:26.653749+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808163+0000","last_change":"2023-06-10T20:59:25.412193+0000","last_active":"2023-06-10T20:59:28.808163+0000","last_peered":"2023-06-10T20:59:28.808163+0000","last_clean":"2023-06-10T20:59:28.808163+0000","last_became_active":"2023-06-10T20:59:25.411853+0000","last_became_peered":"2023-06-10T20:59:25.411853+0000","last_unstale":"2023-06-10T20:59:28.808163+0000","last_undegraded":"2023-06-10T20:59:28.808163+0000","last_fullsized":"2023-06-10T20:59:28.808163+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655257+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T20:59:26.655257+0000","last_peered":"2023-06-10T20:59:26.655257+0000","last_clean":"2023-06-10T20:59:26.655257+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T20:59:26.655257+0000","last_undegraded":"2023-06-10T20:59:26.655257+0000","last_fullsized":"2023-06-10T20:59:26.655257+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808370+0000","last_change":"2023-06-10T20:59:25.412194+0000","last_active":"2023-06-10T20:59:28.808370+0000","last_peered":"2023-06-10T20:59:28.808370+0000","last_clean":"2023-06-10T20:59:28.808370+0000","last_became_active":"2023-06-10T20:59:25.411886+0000","last_became_peered":"2023-06-10T20:59:25.411886+0000","last_unstale":"2023-06-10T20:59:28.808370+0000","last_undegraded":"2023-06-10T20:59:28.808370+0000","last_fullsized":"2023-06-10T20:59:28.808370+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421617+0000","last_change":"2023-06-10T20:58:34.376483+0000","last_active":"2023-06-10T20:59:26.421617+0000","last_peered":"2023-06-10T20:59:26.421617+0000","last_clean":"2023-06-10T20:59:26.421617+0000","last_became_active":"2023-06-10T20:58:34.376121+0000","last_became_peered":"2023-06-10T20:58:34.376121+0000","last_unstale":"2023-06-10T20:59:26.421617+0000","last_undegraded":"2023-06-10T20:59:26.421617+0000","last_fullsized":"2023-06-10T20:59:26.421617+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423469+0000","last_change":"2023-06-10T20:59:25.930882+0000","last_active":"2023-06-10T20:59:26.423469+0000","last_peered":"2023-06-10T20:59:26.423469+0000","last_clean":"2023-06-10T20:59:26.423469+0000","last_became_active":"2023-06-10T20:59:25.930538+0000","last_became_peered":"2023-06-10T20:59:25.930538+0000","last_unstale":"2023-06-10T20:59:26.423469+0000","last_undegraded":"2023-06-10T20:59:26.423469+0000","last_fullsized":"2023-06-10T20:59:26.423469+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421323+0000","last_change":"2023-06-10T20:58:34.481072+0000","last_active":"2023-06-10T20:59:26.421323+0000","last_peered":"2023-06-10T20:59:26.421323+0000","last_clean":"2023-06-10T20:59:26.421323+0000","last_became_active":"2023-06-10T20:58:34.480835+0000","last_became_peered":"2023-06-10T20:58:34.480835+0000","last_unstale":"2023-06-10T20:59:26.421323+0000","last_undegraded":"2023-06-10T20:59:26.421323+0000","last_fullsized":"2023-06-10T20:59:26.421323+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.025408+0000","last_change":"2023-06-10T20:59:25.407671+0000","last_active":"2023-06-10T20:59:38.025408+0000","last_peered":"2023-06-10T20:59:38.025408+0000","last_clean":"2023-06-10T20:59:38.025408+0000","last_became_active":"2023-06-10T20:59:25.407266+0000","last_became_peered":"2023-06-10T20:59:25.407266+0000","last_unstale":"2023-06-10T20:59:38.025408+0000","last_undegraded":"2023-06-10T20:59:38.025408+0000","last_fullsized":"2023-06-10T20:59:38.025408+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653646+0000","last_change":"2023-06-10T20:59:25.930871+0000","last_active":"2023-06-10T20:59:26.653646+0000","last_peered":"2023-06-10T20:59:26.653646+0000","last_clean":"2023-06-10T20:59:26.653646+0000","last_became_active":"2023-06-10T20:59:25.930516+0000","last_became_peered":"2023-06-10T20:59:25.930516+0000","last_unstale":"2023-06-10T20:59:26.653646+0000","last_undegraded":"2023-06-10T20:59:26.653646+0000","last_fullsized":"2023-06-10T20:59:26.653646+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421395+0000","last_change":"2023-06-10T20:59:25.411265+0000","last_active":"2023-06-10T20:59:26.421395+0000","last_peered":"2023-06-10T20:59:26.421395+0000","last_clean":"2023-06-10T20:59:26.421395+0000","last_became_active":"2023-06-10T20:59:25.410947+0000","last_became_peered":"2023-06-10T20:59:25.410947+0000","last_unstale":"2023-06-10T20:59:26.421395+0000","last_undegraded":"2023-06-10T20:59:26.421395+0000","last_fullsized":"2023-06-10T20:59:26.421395+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655287+0000","last_change":"2023-06-10T20:58:35.246897+0000","last_active":"2023-06-10T20:59:26.655287+0000","last_peered":"2023-06-10T20:59:26.655287+0000","last_clean":"2023-06-10T20:59:26.655287+0000","last_became_active":"2023-06-10T20:58:35.246501+0000","last_became_peered":"2023-06-10T20:58:35.246501+0000","last_unstale":"2023-06-10T20:59:26.655287+0000","last_undegraded":"2023-06-10T20:59:26.655287+0000","last_fullsized":"2023-06-10T20:59:26.655287+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654113+0000","last_change":"2023-06-10T20:59:25.432741+0000","last_active":"2023-06-10T20:59:26.654113+0000","last_peered":"2023-06-10T20:59:26.654113+0000","last_clean":"2023-06-10T20:59:26.654113+0000","last_became_active":"2023-06-10T20:59:25.432445+0000","last_became_peered":"2023-06-10T20:59:25.432445+0000","last_unstale":"2023-06-10T20:59:26.654113+0000","last_undegraded":"2023-06-10T20:59:26.654113+0000","last_fullsized":"2023-06-10T20:59:26.654113+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654179+0000","last_change":"2023-06-10T20:59:25.411497+0000","last_active":"2023-06-10T20:59:26.654179+0000","last_peered":"2023-06-10T20:59:26.654179+0000","last_clean":"2023-06-10T20:59:26.654179+0000","last_became_active":"2023-06-10T20:59:25.411149+0000","last_became_peered":"2023-06-10T20:59:25.411149+0000","last_unstale":"2023-06-10T20:59:26.654179+0000","last_undegraded":"2023-06-10T20:59:26.654179+0000","last_fullsized":"2023-06-10T20:59:26.654179+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654970+0000","last_change":"2023-06-10T20:58:35.246773+0000","last_active":"2023-06-10T20:59:26.654970+0000","last_peered":"2023-06-10T20:59:26.654970+0000","last_clean":"2023-06-10T20:59:26.654970+0000","last_became_active":"2023-06-10T20:58:35.246379+0000","last_became_peered":"2023-06-10T20:58:35.246379+0000","last_unstale":"2023-06-10T20:59:26.654970+0000","last_undegraded":"2023-06-10T20:59:26.654970+0000","last_fullsized":"2023-06-10T20:59:26.654970+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.658012+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T20:59:26.658012+0000","last_peered":"2023-06-10T20:59:26.658012+0000","last_clean":"2023-06-10T20:59:26.658012+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T20:59:26.658012+0000","last_undegraded":"2023-06-10T20:59:26.658012+0000","last_fullsized":"2023-06-10T20:59:26.658012+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423764+0000","last_change":"2023-06-10T20:59:25.409300+0000","last_active":"2023-06-10T20:59:26.423764+0000","last_peered":"2023-06-10T20:59:26.423764+0000","last_clean":"2023-06-10T20:59:26.423764+0000","last_became_active":"2023-06-10T20:59:25.408999+0000","last_became_peered":"2023-06-10T20:59:25.408999+0000","last_unstale":"2023-06-10T20:59:26.423764+0000","last_undegraded":"2023-06-10T20:59:26.423764+0000","last_fullsized":"2023-06-10T20:59:26.423764+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":20,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654887+0000","last_change":"2023-06-10T20:57:57.149158+0000","last_active":"2023-06-10T20:59:26.654887+0000","last_peered":"2023-06-10T20:59:26.654887+0000","last_clean":"2023-06-10T20:59:26.654887+0000","last_became_active":"2023-06-10T20:57:57.148833+0000","last_became_peered":"2023-06-10T20:57:57.148833+0000","last_unstale":"2023-06-10T20:59:26.654887+0000","last_undegraded":"2023-06-10T20:59:26.654887+0000","last_fullsized":"2023-06-10T20:59:26.654887+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654285+0000","last_change":"2023-06-10T20:58:34.376607+0000","last_active":"2023-06-10T20:59:26.654285+0000","last_peered":"2023-06-10T20:59:26.654285+0000","last_clean":"2023-06-10T20:59:26.654285+0000","last_became_active":"2023-06-10T20:58:34.376302+0000","last_became_peered":"2023-06-10T20:58:34.376302+0000","last_unstale":"2023-06-10T20:59:26.654285+0000","last_undegraded":"2023-06-10T20:59:26.654285+0000","last_fullsized":"2023-06-10T20:59:26.654285+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.422223+0000","last_change":"2023-06-10T20:59:25.435968+0000","last_active":"2023-06-10T20:59:26.422223+0000","last_peered":"2023-06-10T20:59:26.422223+0000","last_clean":"2023-06-10T20:59:26.422223+0000","last_became_active":"2023-06-10T20:59:25.435664+0000","last_became_peered":"2023-06-10T20:59:25.435664+0000","last_unstale":"2023-06-10T20:59:26.422223+0000","last_undegraded":"2023-06-10T20:59:26.422223+0000","last_fullsized":"2023-06-10T20:59:26.422223+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420887+0000","last_change":"2023-06-10T20:59:25.408598+0000","last_active":"2023-06-10T20:59:26.420887+0000","last_peered":"2023-06-10T20:59:26.420887+0000","last_clean":"2023-06-10T20:59:26.420887+0000","last_became_active":"2023-06-10T20:59:25.408257+0000","last_became_peered":"2023-06-10T20:59:25.408257+0000","last_unstale":"2023-06-10T20:59:26.420887+0000","last_undegraded":"2023-06-10T20:59:26.420887+0000","last_fullsized":"2023-06-10T20:59:26.420887+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653445+0000","last_change":"2023-06-10T20:58:35.246831+0000","last_active":"2023-06-10T20:59:26.653445+0000","last_peered":"2023-06-10T20:59:26.653445+0000","last_clean":"2023-06-10T20:59:26.653445+0000","last_became_active":"2023-06-10T20:58:35.246208+0000","last_became_peered":"2023-06-10T20:58:35.246208+0000","last_unstale":"2023-06-10T20:59:26.653445+0000","last_undegraded":"2023-06-10T20:59:26.653445+0000","last_fullsized":"2023-06-10T20:59:26.653445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421296+0000","last_change":"2023-06-10T20:59:25.433487+0000","last_active":"2023-06-10T20:59:26.421296+0000","last_peered":"2023-06-10T20:59:26.421296+0000","last_clean":"2023-06-10T20:59:26.421296+0000","last_became_active":"2023-06-10T20:59:25.432968+0000","last_became_peered":"2023-06-10T20:59:25.432968+0000","last_unstale":"2023-06-10T20:59:26.421296+0000","last_undegraded":"2023-06-10T20:59:26.421296+0000","last_fullsized":"2023-06-10T20:59:26.421296+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655792+0000","last_change":"2023-06-10T20:59:25.412473+0000","last_active":"2023-06-10T20:59:26.655792+0000","last_peered":"2023-06-10T20:59:26.655792+0000","last_clean":"2023-06-10T20:59:26.655792+0000","last_became_active":"2023-06-10T20:59:25.412173+0000","last_became_peered":"2023-06-10T20:59:25.412173+0000","last_unstale":"2023-06-10T20:59:26.655792+0000","last_undegraded":"2023-06-10T20:59:26.655792+0000","last_fullsized":"2023-06-10T20:59:26.655792+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654267+0000","last_change":"2023-06-10T20:58:34.376802+0000","last_active":"2023-06-10T20:59:26.654267+0000","last_peered":"2023-06-10T20:59:26.654267+0000","last_clean":"2023-06-10T20:59:26.654267+0000","last_became_active":"2023-06-10T20:58:34.376449+0000","last_became_peered":"2023-06-10T20:58:34.376449+0000","last_unstale":"2023-06-10T20:59:26.654267+0000","last_undegraded":"2023-06-10T20:59:26.654267+0000","last_fullsized":"2023-06-10T20:59:26.654267+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421081+0000","last_change":"2023-06-10T20:59:25.430136+0000","last_active":"2023-06-10T20:59:26.421081+0000","last_peered":"2023-06-10T20:59:26.421081+0000","last_clean":"2023-06-10T20:59:26.421081+0000","last_became_active":"2023-06-10T20:59:25.429815+0000","last_became_peered":"2023-06-10T20:59:25.429815+0000","last_unstale":"2023-06-10T20:59:26.421081+0000","last_undegraded":"2023-06-10T20:59:26.421081+0000","last_fullsized":"2023-06-10T20:59:26.421081+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655336+0000","last_change":"2023-06-10T20:59:25.408917+0000","last_active":"2023-06-10T20:59:26.655336+0000","last_peered":"2023-06-10T20:59:26.655336+0000","last_clean":"2023-06-10T20:59:26.655336+0000","last_became_active":"2023-06-10T20:59:25.408502+0000","last_became_peered":"2023-06-10T20:59:25.408502+0000","last_unstale":"2023-06-10T20:59:26.655336+0000","last_undegraded":"2023-06-10T20:59:26.655336+0000","last_fullsized":"2023-06-10T20:59:26.655336+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421214+0000","last_change":"2023-06-10T20:58:35.246694+0000","last_active":"2023-06-10T20:59:26.421214+0000","last_peered":"2023-06-10T20:59:26.421214+0000","last_clean":"2023-06-10T20:59:26.421214+0000","last_became_active":"2023-06-10T20:58:35.246275+0000","last_became_peered":"2023-06-10T20:58:35.246275+0000","last_unstale":"2023-06-10T20:59:26.421214+0000","last_undegraded":"2023-06-10T20:59:26.421214+0000","last_fullsized":"2023-06-10T20:59:26.421214+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.422161+0000","last_change":"2023-06-10T20:59:25.431108+0000","last_active":"2023-06-10T20:59:26.422161+0000","last_peered":"2023-06-10T20:59:26.422161+0000","last_clean":"2023-06-10T20:59:26.422161+0000","last_became_active":"2023-06-10T20:59:25.430328+0000","last_became_peered":"2023-06-10T20:59:25.430328+0000","last_unstale":"2023-06-10T20:59:26.422161+0000","last_undegraded":"2023-06-10T20:59:26.422161+0000","last_fullsized":"2023-06-10T20:59:26.422161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423492+0000","last_change":"2023-06-10T20:59:25.410112+0000","last_active":"2023-06-10T20:59:26.423492+0000","last_peered":"2023-06-10T20:59:26.423492+0000","last_clean":"2023-06-10T20:59:26.423492+0000","last_became_active":"2023-06-10T20:59:25.409782+0000","last_became_peered":"2023-06-10T20:59:25.409782+0000","last_unstale":"2023-06-10T20:59:26.423492+0000","last_undegraded":"2023-06-10T20:59:26.423492+0000","last_fullsized":"2023-06-10T20:59:26.423492+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655104+0000","last_change":"2023-06-10T20:59:26.106380+0000","last_active":"2023-06-10T20:59:26.655104+0000","last_peered":"2023-06-10T20:59:26.655104+0000","last_clean":"2023-06-10T20:59:26.655104+0000","last_became_active":"2023-06-10T20:59:26.106011+0000","last_became_peered":"2023-06-10T20:59:26.106011+0000","last_unstale":"2023-06-10T20:59:26.655104+0000","last_undegraded":"2023-06-10T20:59:26.655104+0000","last_fullsized":"2023-06-10T20:59:26.655104+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655308+0000","last_change":"2023-06-10T20:59:25.409183+0000","last_active":"2023-06-10T20:59:26.655308+0000","last_peered":"2023-06-10T20:59:26.655308+0000","last_clean":"2023-06-10T20:59:26.655308+0000","last_became_active":"2023-06-10T20:59:25.408876+0000","last_became_peered":"2023-06-10T20:59:25.408876+0000","last_unstale":"2023-06-10T20:59:26.655308+0000","last_undegraded":"2023-06-10T20:59:26.655308+0000","last_fullsized":"2023-06-10T20:59:26.655308+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655561+0000","last_change":"2023-06-10T20:59:25.431005+0000","last_active":"2023-06-10T20:59:26.655561+0000","last_peered":"2023-06-10T20:59:26.655561+0000","last_clean":"2023-06-10T20:59:26.655561+0000","last_became_active":"2023-06-10T20:59:25.430701+0000","last_became_peered":"2023-06-10T20:59:25.430701+0000","last_unstale":"2023-06-10T20:59:26.655561+0000","last_undegraded":"2023-06-10T20:59:26.655561+0000","last_fullsized":"2023-06-10T20:59:26.655561+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.421606+0000","last_change":"2023-06-10T20:59:25.408524+0000","last_active":"2023-06-10T20:59:26.421606+0000","last_peered":"2023-06-10T20:59:26.421606+0000","last_clean":"2023-06-10T20:59:26.421606+0000","last_became_active":"2023-06-10T20:59:25.408052+0000","last_became_peered":"2023-06-10T20:59:25.408052+0000","last_unstale":"2023-06-10T20:59:26.421606+0000","last_undegraded":"2023-06-10T20:59:26.421606+0000","last_fullsized":"2023-06-10T20:59:26.421606+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654835+0000","last_change":"2023-06-10T20:59:26.107188+0000","last_active":"2023-06-10T20:59:26.654835+0000","last_peered":"2023-06-10T20:59:26.654835+0000","last_clean":"2023-06-10T20:59:26.654835+0000","last_became_active":"2023-06-10T20:59:26.106822+0000","last_became_peered":"2023-06-10T20:59:26.106822+0000","last_unstale":"2023-06-10T20:59:26.654835+0000","last_undegraded":"2023-06-10T20:59:26.654835+0000","last_fullsized":"2023-06-10T20:59:26.654835+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.026573+0000","last_change":"2023-06-10T20:59:25.411558+0000","last_active":"2023-06-10T20:59:38.026573+0000","last_peered":"2023-06-10T20:59:38.026573+0000","last_clean":"2023-06-10T20:59:38.026573+0000","last_became_active":"2023-06-10T20:59:25.411214+0000","last_became_peered":"2023-06-10T20:59:25.411214+0000","last_unstale":"2023-06-10T20:59:38.026573+0000","last_undegraded":"2023-06-10T20:59:38.026573+0000","last_fullsized":"2023-06-10T20:59:38.026573+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653540+0000","last_change":"2023-06-10T20:59:25.434771+0000","last_active":"2023-06-10T20:59:26.653540+0000","last_peered":"2023-06-10T20:59:26.653540+0000","last_clean":"2023-06-10T20:59:26.653540+0000","last_became_active":"2023-06-10T20:59:25.432415+0000","last_became_peered":"2023-06-10T20:59:25.432415+0000","last_unstale":"2023-06-10T20:59:26.653540+0000","last_undegraded":"2023-06-10T20:59:26.653540+0000","last_fullsized":"2023-06-10T20:59:26.653540+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654511+0000","last_change":"2023-06-10T20:59:25.408811+0000","last_active":"2023-06-10T20:59:26.654511+0000","last_peered":"2023-06-10T20:59:26.654511+0000","last_clean":"2023-06-10T20:59:26.654511+0000","last_became_active":"2023-06-10T20:59:25.408478+0000","last_became_peered":"2023-06-10T20:59:25.408478+0000","last_unstale":"2023-06-10T20:59:26.654511+0000","last_undegraded":"2023-06-10T20:59:26.654511+0000","last_fullsized":"2023-06-10T20:59:26.654511+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.657590+0000","last_change":"2023-06-10T20:59:26.656990+0000","last_active":"2023-06-10T20:59:26.657590+0000","last_peered":"2023-06-10T20:59:26.657590+0000","last_clean":"2023-06-10T20:59:26.657590+0000","last_became_active":"2023-06-10T20:59:26.656676+0000","last_became_peered":"2023-06-10T20:59:26.656676+0000","last_unstale":"2023-06-10T20:59:26.657590+0000","last_undegraded":"2023-06-10T20:59:26.657590+0000","last_fullsized":"2023-06-10T20:59:26.657590+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808402+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T20:59:28.808402+0000","last_peered":"2023-06-10T20:59:28.808402+0000","last_clean":"2023-06-10T20:59:28.808402+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T20:59:28.808402+0000","last_undegraded":"2023-06-10T20:59:28.808402+0000","last_fullsized":"2023-06-10T20:59:28.808402+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655172+0000","last_change":"2023-06-10T20:59:25.431587+0000","last_active":"2023-06-10T20:59:26.655172+0000","last_peered":"2023-06-10T20:59:26.655172+0000","last_clean":"2023-06-10T20:59:26.655172+0000","last_became_active":"2023-06-10T20:59:25.431318+0000","last_became_peered":"2023-06-10T20:59:25.431318+0000","last_unstale":"2023-06-10T20:59:26.655172+0000","last_undegraded":"2023-06-10T20:59:26.655172+0000","last_fullsized":"2023-06-10T20:59:26.655172+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028700+0000","last_change":"2023-06-10T20:59:25.412893+0000","last_active":"2023-06-10T20:59:38.028700+0000","last_peered":"2023-06-10T20:59:38.028700+0000","last_clean":"2023-06-10T20:59:38.028700+0000","last_became_active":"2023-06-10T20:59:25.412557+0000","last_became_peered":"2023-06-10T20:59:25.412557+0000","last_unstale":"2023-06-10T20:59:38.028700+0000","last_undegraded":"2023-06-10T20:59:38.028700+0000","last_fullsized":"2023-06-10T20:59:38.028700+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654786+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T20:59:26.654786+0000","last_peered":"2023-06-10T20:59:26.654786+0000","last_clean":"2023-06-10T20:59:26.654786+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T20:59:26.654786+0000","last_undegraded":"2023-06-10T20:59:26.654786+0000","last_fullsized":"2023-06-10T20:59:26.654786+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:28.808422+0000","last_change":"2023-06-10T20:59:25.415629+0000","last_active":"2023-06-10T20:59:28.808422+0000","last_peered":"2023-06-10T20:59:28.808422+0000","last_clean":"2023-06-10T20:59:28.808422+0000","last_became_active":"2023-06-10T20:59:25.415333+0000","last_became_peered":"2023-06-10T20:59:25.415333+0000","last_unstale":"2023-06-10T20:59:28.808422+0000","last_undegraded":"2023-06-10T20:59:28.808422+0000","last_fullsized":"2023-06-10T20:59:28.808422+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.424182+0000","last_change":"2023-06-10T20:59:25.930923+0000","last_active":"2023-06-10T20:59:26.424182+0000","last_peered":"2023-06-10T20:59:26.424182+0000","last_clean":"2023-06-10T20:59:26.424182+0000","last_became_active":"2023-06-10T20:59:25.930583+0000","last_became_peered":"2023-06-10T20:59:25.930583+0000","last_unstale":"2023-06-10T20:59:26.424182+0000","last_undegraded":"2023-06-10T20:59:26.424182+0000","last_fullsized":"2023-06-10T20:59:26.424182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":20,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028959+0000","last_change":"2023-06-10T20:59:25.404800+0000","last_active":"2023-06-10T20:59:38.028959+0000","last_peered":"2023-06-10T20:59:38.028959+0000","last_clean":"2023-06-10T20:59:38.028959+0000","last_became_active":"2023-06-10T20:59:25.404428+0000","last_became_peered":"2023-06-10T20:59:25.404428+0000","last_unstale":"2023-06-10T20:59:38.028959+0000","last_undegraded":"2023-06-10T20:59:38.028959+0000","last_fullsized":"2023-06-10T20:59:38.028959+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653896+0000","last_change":"2023-06-10T20:59:25.930888+0000","last_active":"2023-06-10T20:59:26.653896+0000","last_peered":"2023-06-10T20:59:26.653896+0000","last_clean":"2023-06-10T20:59:26.653896+0000","last_became_active":"2023-06-10T20:59:25.930539+0000","last_became_peered":"2023-06-10T20:59:25.930539+0000","last_unstale":"2023-06-10T20:59:26.653896+0000","last_undegraded":"2023-06-10T20:59:26.653896+0000","last_fullsized":"2023-06-10T20:59:26.653896+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.656007+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T20:59:26.656007+0000","last_peered":"2023-06-10T20:59:26.656007+0000","last_clean":"2023-06-10T20:59:26.656007+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T20:59:26.656007+0000","last_undegraded":"2023-06-10T20:59:26.656007+0000","last_fullsized":"2023-06-10T20:59:26.656007+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655260+0000","last_change":"2023-06-10T20:59:25.930888+0000","last_active":"2023-06-10T20:59:26.655260+0000","last_peered":"2023-06-10T20:59:26.655260+0000","last_clean":"2023-06-10T20:59:26.655260+0000","last_became_active":"2023-06-10T20:59:25.930503+0000","last_became_peered":"2023-06-10T20:59:25.930503+0000","last_unstale":"2023-06-10T20:59:26.655260+0000","last_undegraded":"2023-06-10T20:59:26.655260+0000","last_fullsized":"2023-06-10T20:59:26.655260+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653617+0000","last_change":"2023-06-10T20:59:25.404676+0000","last_active":"2023-06-10T20:59:26.653617+0000","last_peered":"2023-06-10T20:59:26.653617+0000","last_clean":"2023-06-10T20:59:26.653617+0000","last_became_active":"2023-06-10T20:59:25.404305+0000","last_became_peered":"2023-06-10T20:59:25.404305+0000","last_unstale":"2023-06-10T20:59:26.653617+0000","last_undegraded":"2023-06-10T20:59:26.653617+0000","last_fullsized":"2023-06-10T20:59:26.653617+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423873+0000","last_change":"2023-06-10T20:59:25.930933+0000","last_active":"2023-06-10T20:59:26.423873+0000","last_peered":"2023-06-10T20:59:26.423873+0000","last_clean":"2023-06-10T20:59:26.423873+0000","last_became_active":"2023-06-10T20:59:25.930600+0000","last_became_peered":"2023-06-10T20:59:25.930600+0000","last_unstale":"2023-06-10T20:59:26.423873+0000","last_undegraded":"2023-06-10T20:59:26.423873+0000","last_fullsized":"2023-06-10T20:59:26.423873+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":14,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.028696+0000","last_change":"2023-06-10T20:59:25.410958+0000","last_active":"2023-06-10T20:59:38.028696+0000","last_peered":"2023-06-10T20:59:38.028696+0000","last_clean":"2023-06-10T20:59:38.028696+0000","last_became_active":"2023-06-10T20:59:25.410668+0000","last_became_peered":"2023-06-10T20:59:25.410668+0000","last_unstale":"2023-06-10T20:59:38.028696+0000","last_undegraded":"2023-06-10T20:59:38.028696+0000","last_fullsized":"2023-06-10T20:59:38.028696+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.655610+0000","last_change":"2023-06-10T20:59:26.106185+0000","last_active":"2023-06-10T20:59:26.655610+0000","last_peered":"2023-06-10T20:59:26.655610+0000","last_clean":"2023-06-10T20:59:26.655610+0000","last_became_active":"2023-06-10T20:59:26.105815+0000","last_became_peered":"2023-06-10T20:59:26.105815+0000","last_unstale":"2023-06-10T20:59:26.655610+0000","last_undegraded":"2023-06-10T20:59:26.655610+0000","last_fullsized":"2023-06-10T20:59:26.655610+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":18,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029994+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T20:59:38.029994+0000","last_peered":"2023-06-10T20:59:38.029994+0000","last_clean":"2023-06-10T20:59:38.029994+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T20:59:38.029994+0000","last_undegraded":"2023-06-10T20:59:38.029994+0000","last_fullsized":"2023-06-10T20:59:38.029994+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420999+0000","last_change":"2023-06-10T20:59:26.105976+0000","last_active":"2023-06-10T20:59:26.420999+0000","last_peered":"2023-06-10T20:59:26.420999+0000","last_clean":"2023-06-10T20:59:26.420999+0000","last_became_active":"2023-06-10T20:59:26.105591+0000","last_became_peered":"2023-06-10T20:59:26.105591+0000","last_unstale":"2023-06-10T20:59:26.420999+0000","last_undegraded":"2023-06-10T20:59:26.420999+0000","last_fullsized":"2023-06-10T20:59:26.420999+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":15,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:38.029396+0000","last_change":"2023-06-10T20:59:25.410259+0000","last_active":"2023-06-10T20:59:38.029396+0000","last_peered":"2023-06-10T20:59:38.029396+0000","last_clean":"2023-06-10T20:59:38.029396+0000","last_became_active":"2023-06-10T20:59:25.409862+0000","last_became_peered":"2023-06-10T20:59:25.409862+0000","last_unstale":"2023-06-10T20:59:38.029396+0000","last_undegraded":"2023-06-10T20:59:38.029396+0000","last_fullsized":"2023-06-10T20:59:38.029396+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423711+0000","last_change":"2023-06-10T20:59:26.106353+0000","last_active":"2023-06-10T20:59:26.423711+0000","last_peered":"2023-06-10T20:59:26.423711+0000","last_clean":"2023-06-10T20:59:26.423711+0000","last_became_active":"2023-06-10T20:59:26.105974+0000","last_became_peered":"2023-06-10T20:59:26.105974+0000","last_unstale":"2023-06-10T20:59:26.423711+0000","last_undegraded":"2023-06-10T20:59:26.423711+0000","last_fullsized":"2023-06-10T20:59:26.423711+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654434+0000","last_change":"2023-06-10T20:59:25.404046+0000","last_active":"2023-06-10T20:59:26.654434+0000","last_peered":"2023-06-10T20:59:26.654434+0000","last_clean":"2023-06-10T20:59:26.654434+0000","last_became_active":"2023-06-10T20:59:25.403707+0000","last_became_peered":"2023-06-10T20:59:25.403707+0000","last_unstale":"2023-06-10T20:59:26.654434+0000","last_undegraded":"2023-06-10T20:59:26.654434+0000","last_fullsized":"2023-06-10T20:59:26.654434+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423560+0000","last_change":"2023-06-10T20:59:26.107602+0000","last_active":"2023-06-10T20:59:26.423560+0000","last_peered":"2023-06-10T20:59:26.423560+0000","last_clean":"2023-06-10T20:59:26.423560+0000","last_became_active":"2023-06-10T20:59:26.107262+0000","last_became_peered":"2023-06-10T20:59:26.107262+0000","last_unstale":"2023-06-10T20:59:26.423560+0000","last_undegraded":"2023-06-10T20:59:26.423560+0000","last_fullsized":"2023-06-10T20:59:26.423560+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.654058+0000","last_change":"2023-06-10T20:59:25.413422+0000","last_active":"2023-06-10T20:59:26.654058+0000","last_peered":"2023-06-10T20:59:26.654058+0000","last_clean":"2023-06-10T20:59:26.654058+0000","last_became_active":"2023-06-10T20:59:25.412850+0000","last_became_peered":"2023-06-10T20:59:25.412850+0000","last_unstale":"2023-06-10T20:59:26.654058+0000","last_undegraded":"2023-06-10T20:59:26.654058+0000","last_fullsized":"2023-06-10T20:59:26.654058+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.657444+0000","last_change":"2023-06-10T20:59:26.656575+0000","last_active":"2023-06-10T20:59:26.657444+0000","last_peered":"2023-06-10T20:59:26.657444+0000","last_clean":"2023-06-10T20:59:26.657444+0000","last_became_active":"2023-06-10T20:59:26.656167+0000","last_became_peered":"2023-06-10T20:59:26.656167+0000","last_unstale":"2023-06-10T20:59:26.657444+0000","last_undegraded":"2023-06-10T20:59:26.657444+0000","last_fullsized":"2023-06-10T20:59:26.657444+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.423589+0000","last_change":"2023-06-10T20:59:25.405537+0000","last_active":"2023-06-10T20:59:26.423589+0000","last_peered":"2023-06-10T20:59:26.423589+0000","last_clean":"2023-06-10T20:59:26.423589+0000","last_became_active":"2023-06-10T20:59:25.405233+0000","last_became_peered":"2023-06-10T20:59:25.405233+0000","last_unstale":"2023-06-10T20:59:26.423589+0000","last_undegraded":"2023-06-10T20:59:26.423589+0000","last_fullsized":"2023-06-10T20:59:26.423589+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.653984+0000","last_change":"2023-06-10T20:59:25.931230+0000","last_active":"2023-06-10T20:59:26.653984+0000","last_peered":"2023-06-10T20:59:26.653984+0000","last_clean":"2023-06-10T20:59:26.653984+0000","last_became_active":"2023-06-10T20:59:25.930826+0000","last_became_peered":"2023-06-10T20:59:25.930826+0000","last_unstale":"2023-06-10T20:59:26.653984+0000","last_undegraded":"2023-06-10T20:59:26.653984+0000","last_fullsized":"2023-06-10T20:59:26.653984+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.420912+0000","last_change":"2023-06-10T20:59:25.411727+0000","last_active":"2023-06-10T20:59:26.420912+0000","last_peered":"2023-06-10T20:59:26.420912+0000","last_clean":"2023-06-10T20:59:26.420912+0000","last_became_active":"2023-06-10T20:59:25.411353+0000","last_became_peered":"2023-06-10T20:59:25.411353+0000","last_unstale":"2023-06-10T20:59:26.420912+0000","last_undegraded":"2023-06-10T20:59:26.420912+0000","last_fullsized":"2023-06-10T20:59:26.420912+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":59,"state":"active+clean","last_fresh":"2023-06-10T20:59:26.424097+0000","last_change":"2023-06-10T20:59:25.435781+0000","last_active":"2023-06-10T20:59:26.424097+0000","last_peered":"2023-06-10T20:59:26.424097+0000","last_clean":"2023-06-10T20:59:26.424097+0000","last_became_active":"2023-06-10T20:59:25.435478+0000","last_became_peered":"2023-06-10T20:59:25.435478+0000","last_unstale":"2023-06-10T20:59:26.424097+0000","last_undegraded":"2023-06-10T20:59:26.424097+0000","last_fullsized":"2023-06-10T20:59:26.424097+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-06-10T20:59:27.427094+0000","last_change":"2023-06-10T20:59:25.405915+0000","last_active":"2023-06-10T20:59:27.427094+0000","last_peered":"2023-06-10T20:59:27.427094+0000","last_clean":"2023-06-10T20:59:27.427094+0000","last_became_active":"2023-06-10T20:59:25.405233+0000","last_became_peered":"2023-06-10T20:59:25.405233+0000","last_unstale":"2023-06-10T20:59:27.427094+0000","last_undegraded":"2023-06-10T20:59:27.427094+0000","last_fullsized":"2023-06-10T20:59:27.427094+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430238,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5616,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739536,"statfs":{"total":95995035648,"available":95989284864,"internally_reserved":0,"allocated":630784,"data_stored":257887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.58599999999999997,"15min":0.55100000000000005},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0140000000000002,"5min":6.0140000000000002,"15min":6.0140000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.504,"5min":0.52800000000000002,"15min":0.53200000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.129,"5min":1.129,"15min":1.129},"last":0.76800000000000002}]},{"osd":1,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.60199999999999998,"15min":0.56000000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.3019999999999996,"5min":7.3019999999999996,"15min":7.3019999999999996},"last":0.373},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.54600000000000004,"15min":0.52300000000000002},"min":{"1min":0.439,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":0.65900000000000003,"15min":0.58999999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":8.9949999999999992,"5min":8.9949999999999992,"15min":8.9949999999999992},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.63900000000000001,"15min":0.57499999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":8.8789999999999996,"5min":8.8789999999999996,"15min":8.8789999999999996},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":0.66400000000000003,"15min":0.58399999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.9489999999999998,"5min":7.9489999999999998,"15min":7.9489999999999998},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.61099999999999999,"15min":0.58499999999999996},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":5.016,"5min":5.016,"15min":5.016},"last":0.80000000000000004}]},{"osd":4,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.51100000000000001,"15min":0.51900000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.64500000000000002,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.52400000000000002,"15min":0.53300000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71899999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.51400000000000001,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.69499999999999995,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.56699999999999995,"15min":0.57899999999999996},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:59:56 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.56499999999999995,"15min":0.57499999999999996},"min":{"1min":0.42599999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.73599999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.60399999999999998,"15min":0.61599999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.71099999999999997,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.51800000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626466,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":257426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.57999999999999996,"15min":0.59099999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.57099999999999995,"15min":0.58099999999999996},"min":{"1min":0.39600000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67800000000000005,"5min":0.754,"15min":0.754},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69799999999999995,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.753,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.55800000000000005,"15min":0.56699999999999995},"min":{"1min":0.32900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.72399999999999998,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.55900000000000005,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.70799999999999996,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.501}]},{"osd":3,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59899999999999998,"15min":0.60699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.56699999999999995,"15min":0.57299999999999995},"min":{"1min":0.42399999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.68500000000000005,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.44,"15min":0.432},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.438},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.45400000000000001,"15min":0.44800000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.46300000000000002,"15min":0.45400000000000001},"min":{"1min":0.34899999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.376},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.498,"15min":0.49399999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.39100000000000001}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52100000000000002,"15min":0.51700000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79400000000000004,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.42399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5908,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739244,"statfs":{"total":95995035648,"available":95988985856,"internally_reserved":0,"allocated":602112,"data_stored":256135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:51 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.318},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.62,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.376,"15min":0.33400000000000002},"min":{"1min":0.26800000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.59199999999999997,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.43099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41899999999999998,"15min":0.42699999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.53700000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.308}]},{"osd":3,"last update":"Sat Jun 10 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.433,"15min":0.44700000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.49199999999999999,"15min":0.51000000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Jun 10 20:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.56599999999999995,"15min":0.57899999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.504},{"interface":"front","average":{"1min":0.73499999999999999,"5min":0.63,"15min":0.61399999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":4.593,"5min":4.593,"15min":4.593},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Jun 10 20:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42499999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57899999999999996,"15min":0.57399999999999995},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Jun 10 20:59:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.59799999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61899999999999999,"15min":0.621},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Jun 10 20:59:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.63200000000000001,"15min":0.64100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.86399999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.61599999999999999,"15min":0.623},"min":{"1min":0.439,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.56999999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738428,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5916,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739236,"statfs":{"total":95995035648,"available":95988977664,"internally_reserved":0,"allocated":610304,"data_stored":256711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 20:59:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":0.57799999999999996,"15min":0.434},"min":{"1min":0.26900000000000002,"5min":0.224,"15min":0.224},"max":{"1min":15.949999999999999,"5min":15.949999999999999,"15min":15.949999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.51600000000000001,"15min":0.42699999999999999},"min":{"1min":0.252,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.0500000000000007,"5min":9.0500000000000007,"15min":9.0500000000000007},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45700000000000002,"15min":0.46100000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.67700000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46400000000000002,"15min":0.46500000000000002},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.72999999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.40799999999999997}]},{"osd":3,"last update":"Sat Jun 10 20:59:39 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.58799999999999997,"15min":0.53400000000000003},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":11.128,"5min":11.128,"15min":11.128},"last":0.379},{"interface":"front","average":{"1min":1.3180000000000001,"5min":0.70699999999999996,"15min":0.61899999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":15.895,"5min":15.895,"15min":15.895},"last":0.434}]},{"osd":4,"last update":"Sat Jun 10 21:00:03 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.72599999999999998,"15min":0.69999999999999996},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":5.1900000000000004,"5min":5.1900000000000004,"15min":5.1900000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.879,"5min":0.71799999999999997,"15min":0.68999999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.9729999999999999,"5min":4.9729999999999999,"15min":4.9729999999999999},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Jun 10 20:59:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.621,"15min":0.624},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.76200000000000001,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.85199999999999998,"5min":1.393,"15min":1.393},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:59:39 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.69499999999999995,"15min":0.66200000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0179999999999998,"5min":5.0179999999999998,"15min":5.0179999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.212,"5min":0.751,"15min":0.67400000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":11.167,"5min":11.167,"15min":11.167},"last":0.67800000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:00:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.72799999999999998,"15min":0.70499999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":5.1520000000000001,"5min":5.1520000000000001,"15min":5.1520000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.74299999999999999,"15min":0.72199999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":5.0490000000000004},"last":0.69199999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":256045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.39200000000000002,"15min":0.373},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.315},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Jun 10 20:59:11 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.41499999999999998,"15min":0.40300000000000002},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Jun 10 20:59:44 2023","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.48599999999999999,"15min":0.48999999999999999},"min":{"1min":0.29199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.069,"5min":1.161,"15min":1.161},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.49099999999999999,"15min":0.505},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56299999999999994,"15min":0.56100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.71399999999999997,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60399999999999998,"15min":0.60499999999999998},"min":{"1min":0.41699999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80500000000000005,"5min":1.27,"15min":1.27},"last":0.49399999999999999}]},{"osd":5,"last update":"Sat Jun 10 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.60199999999999998,"15min":0.59199999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.2330000000000001,"5min":1.2330000000000001,"15min":1.2330000000000001},"last":0.433},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.61599999999999999,"15min":0.61299999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.68100000000000005,"15min":0.69899999999999995},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72799999999999998,"5min":1.254,"15min":1.254},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.66100000000000003,"15min":0.67000000000000004},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.84099999999999997,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:00:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.622,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.63200000000000001,"15min":0.64200000000000002},"min":{"1min":0.46400000000000002,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.78000000000000003,"5min":1.075,"15min":1.075},"last":0.45500000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182447,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":256225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.38500000000000001,"15min":0.39500000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.60099999999999998,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.377,"5min":0.42299999999999999,"15min":0.443},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.64200000000000002,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.59599999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.46400000000000002,"15min":0.47799999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.63,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Jun 10 20:59:36 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.495,"15min":0.51100000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66400000000000003,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.25900000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:00:05 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82799999999999996,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.503}]},{"osd":5,"last update":"Sat Jun 10 20:59:17 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.56200000000000006},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.77000000000000002,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.62,"15min":0.622},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81299999999999994,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Jun 10 20:59:40 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.65600000000000003,"15min":0.66600000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.61399999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.60699999999999998,"15min":0.61499999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.497},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.64900000000000002,"15min":0.65900000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.56799999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018922,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5800,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739352,"statfs":{"total":95995035648,"available":95989096448,"internally_reserved":0,"allocated":622592,"data_stored":257502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.53200000000000003,"15min":0.54600000000000004},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69799999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.52500000000000002,"15min":0.52900000000000003},"min":{"1min":0.38200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70599999999999996,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.73499999999999999}]},{"osd":1,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.52900000000000003,"15min":0.54300000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.64800000000000002,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.54300000000000004,"15min":0.55300000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68799999999999994,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.57699999999999996,"15min":0.55900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.4550000000000001,"5min":4.4550000000000001,"15min":4.4550000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.73799999999999999,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.55900000000000005,"15min":0.57099999999999995},"min":{"1min":0.39500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.67300000000000004,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.60599999999999998,"15min":0.58999999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.4649999999999999,"5min":4.4649999999999999,"15min":4.4649999999999999},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.38700000000000001,"15min":0.377},"min":{"1min":0.28699999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.443,"5min":0.40600000000000003,"15min":0.40000000000000002},"min":{"1min":0.33400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.78300000000000003,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Jun 10 20:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46800000000000003,"15min":0.46300000000000002},"min":{"1min":0.308,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.78900000000000003,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.443,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.746,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Jun 10 20:59:57 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.77500000000000002,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.56799999999999995,"15min":0.57099999999999995},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77000000000000002,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.60199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822694,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":256444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52000000000000002,"15min":0.51900000000000002},"min":{"1min":0.35799999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.53700000000000003,"15min":0.54100000000000004},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77700000000000002,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.377},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.441}]},{"osd":2,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.617,"5min":1.617,"15min":1.617},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.752,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.58499999999999996,"15min":0.58699999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Jun 10 20:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.45000000000000001,"15min":0.45100000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.502},{"interface":"front","average":{"1min":0.44,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.31,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Jun 10 20:59:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66800000000000004,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.51600000000000001,"15min":0.51300000000000001},"min":{"1min":0.36699999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68600000000000005,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:00:02 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77200000000000002,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:00:12.582 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-10T21:00:12.582 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-10T21:00:12.583 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-10T21:00:12.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph health --format=json 2023-06-10T21:00:12.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPING 2023-06-10T21:00:12.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:00:12.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STOPPED 2023-06-10T21:00:12.612 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTING 2023-06-10T21:00:12.613 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Serving on http://:::9283 2023-06-10T21:00:12.613 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:12 smithi064 conmon[97677]: [10/Jun/2023:21:00:12] ENGINE Bus STARTED 2023-06-10T21:00:13.242 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:13.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:11.972634+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:13.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.073628+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.077213+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.077613+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.078494+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: -06-10T21:00:12.078786+0000 mgr.y (mgr.14427) 89 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.082478+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.086972+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.087547+0000 mgr.y (mgr.14427) 90 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.206237+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.206643+0000 mgr.y (mgr.14427) 91 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.212927+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.217628+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.218084+0000 mgr.y (mgr.14427) 92 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: cluster 2023-06-10T21:00:12.319610+0000 mgr.y (mgr.14427) 93 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:13.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.336470+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00: 2023-06-10T21:00:13.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: 12.336839+0000 mgr.y (mgr.14427) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:11.972634+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.073628+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.077213+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.077613+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.078494+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.078786+0000 mgr.y (mgr.14427) 89 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.082478+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.086972+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.087547+0000 mgr.y (mgr.14427) 90 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.206237+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.206643+0000 mgr.y (mgr.14427) 91 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.212927+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.217628+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.218084+0000 mgr.y (mgr.14427) 92 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: 10T21:00:12.319610+0000 mgr.y (mgr.14427) 93 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12.336470+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:12. 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: 336839+0000 mgr.y (mgr.14427) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:13.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00:12.343044+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: audit 2023-06-10T21:00: 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[107122]: 12.345942+0000 mon.a (mon.0) 702 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: 2023-06-10T21:00:12 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: .343044+0000 mon.a (mon.0 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: ) 701 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: audit 2023-06-10T21:00:13.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: 2023-06-10T21:00 2023-06-10T21:00:13.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: :12.345942+0000 mon.a ( 2023-06-10T21:00:13.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: mon.0) 702 : audit [DBG] 2023-06-10T21:00:13.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:13 smithi064 conmon[97224]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:11.972634+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:00:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.073628+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.077213+0000 mon.a (mon.0) 693 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.077613+0000 mgr.y (mgr.14427) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.078494+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 10T21:00:12.078786+0000 mgr.y (mgr.14427) 89 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi064.front.sepia.ceph.com:9093"}]: dispatch 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.082478+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.086972+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.087547+0000 mgr.y (mgr.14427) 90 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.206237+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.206643+0000 mgr.y (mgr.14427) 91 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:12.212927+0000 mon.a (mon.0) 698 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10T21:00:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 2023-06-10T21:00:12.217628+0000 mon.a (mon.0) 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 699 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: T21:00:12.218084+0000 mgr.y (mgr.14427 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: ) 92 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: cluster 2023-06- 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 10T21:00:12.319610+0000 mgr.y (mgr 2023-06-10T21:00:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: .14427) 93 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 06-10T21:00:12.336470+0000 mon.a 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: (mon.0) 700 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: -10T21:00:12.336839+0000 mgr.y (mgr 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: .14427) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:00:13.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 06-10T21:00:12.343044+0000 mon.a (mon. 2023-06-10T21:00:13.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: 0) 701 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:13.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:00:13.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: -10T21:00:12.345942+0000 mon.a (mon.0) 702 : 2023-06-10T21:00:13.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:13 smithi152 conmon[97365]: audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: audit 2023-06-10T21:00:14. 2023-06-10T21:00:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: 312963+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:14.474 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: -06-10T21:00:14.313223+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: -06-10T21:00:14.313403+0000 mon.a (mon 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[107122]: .0) 705 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[97224]: audit 2023-06-10T21:00: 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[97224]: 14.312963+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.313223+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.476 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:14 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.313403+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-06-10T21:00:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:14 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.312963+0000 mon.a (mon.0 2023-06-10T21:00:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:14 smithi152 conmon[97365]: ) 703 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:14 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.313223+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:00:14.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:14 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.313403+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2023-06-10T21:00:14.897 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:00:15.495 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-10T21:00:15.496 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-10T21:00:15.496 INFO:teuthology.run_tasks:Running task print... 2023-06-10T21:00:15.505 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-06-10T21:00:15.506 INFO:teuthology.run_tasks:Running task parallel... 2023-06-10T21:00:15.528 INFO:teuthology.task.parallel:starting parallel... 2023-06-10T21:00:15.528 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-10T21:00:15.529 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-06-10T21:00:15.530 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:00:15.531 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:00:15.532 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-06-10T21:00:15.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_0 16 2023-06-10T21:00:15.533 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-10T21:00:15.533 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-06-10T21:00:15.534 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi064.front.sepia.ceph.com 2023-06-10T21:00:15.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-06-10T21:00:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: cluster 2023-06-10T21:00:14.319946+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.372602+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-06-10T21:00:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.372676+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:00:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.372745+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-06-10T21:00:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: cluster 2023-06-10T21:00:14.372794+0000 mon.a 2023-06-10T21:00:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: (mon.0) 709 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.894159+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.64:0/81202658' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: audit 2023-06-10T21:00:14.953893+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[97224]: cluster 2023-06-10T21:00:15.370105+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: cluster 2023-06-10T21:00:14.319946+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: audit 2023-06-10T21:00:14.372602+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-06-10T21:00:15.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: audit 2023-06-10T21:00:14.372676+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: audit 2023-06-10T21:00:14.372745+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: cluster 2023-06-10T21:00:14.372794+0000 mon.a (mon 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: .0) 709 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: audit 2023-06-10T21:00:14.894159+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.64:0/81202658' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: audit 2023-06-10T21:00:14.953893+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:15.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:15 smithi064 conmon[107122]: cluster 2023-06-10T21:00:15.370105+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-10T21:00:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: cluster 2023-06-10T21:00:14.319946+0000 mgr.y (mgr.14427) 95 2023-06-10T21:00:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.372602+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-06-10T21:00:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.372676+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:00:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.372745+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2023-06-10T21:00:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: cluster 2023-06-10T21:00:14.372794+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: audit 2023-06-10T21 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: :00:14.894159+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.64:0/81202658' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: audit 2023-06-10T21:00:14.953893+0000 mon.a 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: cluster 2023-06-10T21: 2023-06-10T21:00:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: 00:15.370105+0000 mon.a (mon.0) 711 : 2023-06-10T21:00:15.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:15 smithi152 conmon[97365]: cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-10T21:00:16.055 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:17.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: audit 2023-06-10T21:00: 2023-06-10T21:00:17.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: 15.973627+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: audit 2023-06-10T21:00:15.974774+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:17.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: audit 2023-06-10T21:00:15.975598+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: audit 2023-06-10T21:00:15.983379+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: audit 2023-06-10T21:00:15.989308+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[97224]: cluster 2023-06-10T21:00:16.320262+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit 2023-06-10T21:00:15.973627+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit 2023-06-10T21:00:17.231 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: 2023-06-10T21:00:15.974774+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: -10T21:00:15.975598+0000 mon.a (mon.0 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: ) 714 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: -06-10T21:00:15.983379 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: +0000 mon.a (mon.0) 715 : audit 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: 2023-06-10T21:00:15. 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: 989308+0000 mon.a (mon.0) 716 : 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: -10T21:00:16.320262+0000 mgr.y (mgr.14427) 2023-06-10T21:00:17.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:16 smithi064 conmon[107122]: 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: audit 2023-06-10T21:00:15.973627+0000 mon.a (mon.0) 712 : audit [INF] 2023-06-10T21:00:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: audit 2023-06-10T21:00:15.974774+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:17.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: audit 2023-06-10T21:00:15.975598+0000 mon.a (mon.0) 714 : audit [INF] 2023-06-10T21:00:17.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:17.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: audit 2023-06-10T21:00:15.983379+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: audit 2023-06-10T21:00:15.989308+0000 mon.a (mon.0) 716 : audit [DBG] 2023-06-10T21:00:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: cluster 2023-06-10T21 2023-06-10T21:00:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:16 smithi152 conmon[97365]: :00:16.320262+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v75: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:17.648 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:17 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:17] "GET /metrics HTTP/1.1" 200 187826 "" "Prometheus/2.33.4" 2023-06-10T21:00:17.908 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:00:17 smithi064 conmon[158459]: level=info ts=2023-06-10T21:00:17.891Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002500856s 2023-06-10T21:00:18.313 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE Bus STOPPING 2023-06-10T21:00:18.313 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:00:18.314 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE Bus STOPPED 2023-06-10T21:00:18.314 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE Bus STARTING 2023-06-10T21:00:18.314 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE Serving on http://:::9283 2023-06-10T21:00:18.314 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:18 smithi064 conmon[97677]: [10/Jun/2023:21:00:18] ENGINE Bus STARTED 2023-06-10T21:00:18.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph config set global log_to_journald false --force' 2023-06-10T21:00:19.154 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' created 2023-06-10T21:00:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.132821+0000 mon.a (mon.0) 717 : audit [INF] from='client.? 172.21.15.64:0/1028059335' entity='client.admin' 2023-06-10T21:00:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: cluster 2023-06-10T21:00:18.320617+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.382738+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.826643+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.827610+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.828359+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.832682+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.838201+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.839675+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18.840383+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: audit 2023-06-10T21:00:18 2023-06-10T21:00:19.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[107122]: .844816+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.132821+0000 mon.a (mon.0) 717 : audit [INF] from='client.? 172.21.15.64:0/1028059335' entity='client.admin' 2023-06-10T21:00:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: cluster 2023-06-10T21:00:18.320617+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.382738+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-10T21:00:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.826643+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.827610+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.828359+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.832682+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.838201+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.839675+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.840383+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: audit 2023-06-10T21:00:18.844816+0000 mon.a (mon.0) 726 : audit [INF] 2023-06-10T21:00:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:19 smithi064 conmon[97224]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.132821+0000 mon.a (mon 2023-06-10T21:00:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: .0) 717 : audit [INF] from='client.? 172.21.15.64:0/1028059335' entity='client.admin' 2023-06-10T21:00:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: cluster 2023-06-10T21:00:18.320617+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v76: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:19.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.382738+0000 mon.a (mon.0) 718 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-10T21:00:19.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.826643+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.827610+0000 mon.a (mon.0) 720 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.828359+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.832682+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:19.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.838201+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:19.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.839675+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.840383+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:19 smithi152 conmon[97365]: audit 2023-06-10T21:00:18.844816+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:20.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-06-10T21:00:20.296 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97224]: audit 2023-06-10T21:00:19.145976+0000 mon.a (mon.0) 727 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-10T21:00:20.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97224]: cluster 2023-06-10T21:00:19.146047+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-10T21:00:20.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: audit 2023-06-10T21:00:19.145976+0000 2023-06-10T21:00:20.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: mon.a (mon.0) 727 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-10T21:00:20.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: cluster 2023-06-10T21:00:20.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: 2023-06-10T21:00:19.146047+0000 2023-06-10T21:00:20.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: mon.a (mon.0) 728 : cluster 2023-06-10T21:00:20.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[107122]: [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-10T21:00:20.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:20 smithi152 conmon[97365]: audit 2023-06-10T21:00:19.145976+0000 mon.a (mon.0) 727 : audit [INF] from='client.? 172.21.15.64:0/553383480' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-10T21:00:20.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:20 smithi152 conmon[97365]: cluster 2023-06-10T21:00:19.146047+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-10T21:00:20.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:00:21.209 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97677]: [10/Jun/2023:21:00:20] ENGINE Bus STOPPING 2023-06-10T21:00:21.210 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97677]: [10/Jun/2023:21:00:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:00:21.210 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97677]: [10/Jun/2023:21:00:20] ENGINE Bus STOPPED 2023-06-10T21:00:21.210 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:20 smithi064 conmon[97677]: [10/Jun/2023:21:00:20] ENGINE Bus STARTING 2023-06-10T21:00:21.210 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97677]: [10/Jun/2023:21:00:21] ENGINE Serving on http://:::9283 2023-06-10T21:00:21.210 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97677]: [10/Jun/2023:21:00:21] ENGINE Bus STARTED 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: cluster 2023-06-10T21:00:20.148414+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: cluster 2023-06-10T21:00:20.321165+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: audit 2023-06-10T21:00:20.916490+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: audit 2023-06-10T21:00:20.967884+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: audit 2023-06-10T21:00:20.969382+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: audit 2023-06-10T21:00:20.970017+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:21.211 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[97224]: audit 2023-06-10T21:00:20.975536+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: cluster 2023-06-10T21:00:20.148414+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: cluster 2023-06-10T21:00:20.321165+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: audit 2023-06-10T21:00:20.916490+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: audit 2023-06-10T21:00:20.967884+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: audit 2023-06-10T21:00:20.969382+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:21.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: audit 2023-06-10T21:00:20.970017+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:21.213 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:21 smithi064 conmon[107122]: audit 2023-06-10T21:00:20.975536+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: cluster 2023-06-10T21:00:20.148414+0000 mon.a (mon. 2023-06-10T21:00:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: 0) 729 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-10T21:00:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: cluster 2023-06-10T21:00:20.321165+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: audit 2023 2023-06-10T21:00:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: -06-10T21:00:20.916490+0000 mon.a (mon.0) 730 : audit [INF] from='client.? ' entity='client.admin' 2023-06-10T21:00:21.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: audit 2023-06-10T21:00:21.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: 2023-06-10T21:00:20.967884+0000 mon.a ( 2023-06-10T21:00:21.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:21.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:00:21.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: 06-10T21:00:20.969382+0000 mon.a (mon.0) 2023-06-10T21:00:21.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: 732 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:21.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: audit 2023-06-10T21:00:20 2023-06-10T21:00:21.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: .970017+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:21.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: audit 2023-06-10T21:00 2023-06-10T21:00:21.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:21 smithi152 conmon[97365]: :20.975536+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:21.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-06-10T21:00:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:22 smithi064 conmon[97224]: cluster 2023-06-10T21:00:21.154361+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-10T21:00:22.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:22 smithi064 conmon[107122]: cluster 2023-06-10T21:00:21.154361+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-10T21:00:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:22 smithi152 conmon[97365]: cluster 2023-06-10T21:00:21.154361+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-10T21:00:23.161 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:00:22 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:00:23.166 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-06-10T21:00:23.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T21:00:23.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[97224]: T21:00:22.321624+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:23.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[97224]: audit 2023-06-10T21:00:22.914662+0000 mon.a (mon.0) 736 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:00:23.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[107122]: cluster 2023-06-10T21:00:22.321624+0000 mgr.y (mgr 2023-06-10T21:00:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[107122]: .14427) 99 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[107122]: audit 2023-06-10T21:00:22.914662+0000 mon.a (mon.0) 2023-06-10T21:00:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:23 smithi064 conmon[107122]: 736 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:00:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:23 smithi152 conmon[97365]: cluster 2023-06-10T21:00:22.321624+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:23 smithi152 conmon[97365]: 2023-06-10T21:00:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:23 smithi152 conmon[97365]: audit 2023-06-10T21:00:22.914662+0000 mon.a (mon.0) 736 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:00:24.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:24 smithi064 conmon[107122]: audit 2023-06-10T21:00:23.159555+0000 mon.a (mon.0) 737 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:00:24.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:24 smithi064 conmon[107122]: cluster 2023-06-10T21:00:23.159622+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-10T21:00:24.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:24 smithi064 conmon[97224]: audit 2023-06-10T21:00:23.159555+0000 mon.a (mon.0) 737 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:00:24.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:24 smithi064 conmon[97224]: cluster 2023-06-10T21:00:23.159622+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-10T21:00:24.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:24 smithi152 conmon[97365]: audit 2023-06-10T21:00:23.159555+0000 mon.a (mon.0) 737 : audit [INF] from='client.? 172.21.15.64:0/2559868509' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:00:24.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:24 smithi152 conmon[97365]: 2023-06-10T21:00:24.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:24 smithi152 conmon[97365]: cluster 2023-06-10T21:00:23.159622+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-10T21:00:24.512 INFO:teuthology.orchestra.run.smithi064.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:24.956 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-06-10T21:00:24.959 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:00:24.960 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:00:25.006 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:00:25.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:00:25.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_162320 2023-06-10T21:00:25.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:00:25.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:00:25.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:00:25.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: cluster 2023-06-10T21:00:24.321958 2023-06-10T21:00:25.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 100 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:25.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.505324+0000 mgr.y (mgr.14427) 101 : audit [DBG] from='client.24698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:25.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:24.506050+0000 mgr.y (mgr.14427) 102 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:25.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.509585+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.550626+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.552151+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.552828+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: audit 2023-06-10T21:00:24.557240+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[97224]: cephadm 2023-06-10T21:00:24.562209+0000 mgr.y (mgr.14427) 103 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: cluster 2023-06-10T21:00: 2023-06-10T21:00:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 24.321958+0000 mgr.y (mgr.14427) 100 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023-06-10T21:00:24. 2023-06-10T21:00:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 505324+0000 mgr.y (mgr.14427) 101 : audit [DBG] 2023-06-10T21:00:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: from='client.24698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: cephadm 2023-06-10T21 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: :00:24.506050+0000 mgr.y (mgr.14427 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: ) 102 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: -06-10T21:00:24.509585 2023-06-10T21:00:25.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: +0000 mon.a (mon.0) 739 : 2023-06-10T21:00:25.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:00:25.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 06-10T21:00:24.550626+0000 2023-06-10T21:00:25.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: mon.a (mon.0) 740 : audit [DBG] 2023-06-10T21:00:25.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:25.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:00:25.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 21:00:24.552151+0000 mon.a ( 2023-06-10T21:00:25.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:25.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 2023-06-10T21:00:25.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023-06-10T21:00 2023-06-10T21:00:25.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: :24.552828+0000 mon.a (mon.0) 2023-06-10T21:00:25.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 742 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:25.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit 2023 2023-06-10T21:00:25.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: -06-10T21:00:24. 2023-06-10T21:00:25.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 557240+0000 mon.a (mon.0) 743 : 2023-06-10T21:00:25.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: cephadm 2023-06 2023-06-10T21:00:25.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: -10T21:00:24.562209+0000 2023-06-10T21:00:25.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: mgr.y (mgr.14427) 2023-06-10T21:00:25.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:25 smithi064 conmon[107122]: 103 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: cluster 2023-06-10T21:00:24.321958+0000 mgr.y ( 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: mgr.14427) 100 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06-10T21:00:24.505324+0000 mgr.y (mgr.14427) 101 : audit [DBG] from='client.24698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: 2023-06-10T21:00:24.506050+0000 mgr.y (mgr.14427) 102 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06-10T21:00:24 2023-06-10T21:00:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: .509585+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: 00:24.550626+0000 mon.a (mon.0) 740 : audit 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: 2023-06-10T21:00:24.552151+0000 mon.a (mon.0) 741 : 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06-10T21:00:24 2023-06-10T21:00:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: .552828+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:00:25.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: 2023-06-10T21:00:25.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:00:25.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: -10T21 2023-06-10T21:00:25.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: :00:24.557240+0000 mon.a (mon 2023-06-10T21:00:25.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: .0) 743 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:00:25.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:25 smithi152 conmon[97365]: cephadm 2023-06-10T21:00:24.562209+0000 mgr.y (mgr.14427) 103 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5937c68aed42cde57f938ad8f8bb264f13176009 2023-06-10T21:00:26.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3076 3060 191.239 191.25 0.00248947 0.00520842 2023-06-10T21:00:27.007 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5542 5526 172.675 154.125 0.00443844 0.0057753 2023-06-10T21:00:27.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:27 smithi064 conmon[97224]: cluster 2023-06-10T21:00:26.322676+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s 2023-06-10T21:00:27.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:27 smithi064 conmon[107122]: cluster 2023-06-10T21:00:26.322676+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s 2023-06-10T21:00:27.300 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:27 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:27] "GET /metrics HTTP/1.1" 200 187826 "" "Prometheus/2.33.4" 2023-06-10T21:00:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:27 smithi152 conmon[97365]: cluster 2023-06-10T21:00:26.322676+0000 mgr.y (mgr.14427 2023-06-10T21:00:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:27 smithi152 conmon[97365]: ) 104 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s wr, 177 op/s 2023-06-10T21:00:28.008 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 7906 7890 164.272 147.75 0.0198027 0.0060712 2023-06-10T21:00:29.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 10094 10078 157.37 136.75 0.0050809 0.00634502 2023-06-10T21:00:29.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:29 smithi064 conmon[97224]: cluster 2023-06-10T21:00:28.323107+0000 mgr.y (mgr.14427) 105 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s 2023-06-10T21:00:29.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:29 smithi064 conmon[107122]: cluster 2023-06-10T21:00:28. 2023-06-10T21:00:29.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:29 smithi064 conmon[107122]: 323107+0000 mgr.y (mgr.14427) 105 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s 2023-06-10T21:00:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:29 smithi152 conmon[97365]: cluster 2023-06-10T21:00:28.323107+0000 mgr.y (mgr.14427) 2023-06-10T21:00:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:29 smithi152 conmon[97365]: 105 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 68 MiB data, 114 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s wr, 136 op/s 2023-06-10T21:00:29.815 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:00:29.815 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (22s) 13s ago 98s 17.2M - 0.23.0 ba2b418f427c 163fca1ec9a5 2023-06-10T21:00:29.815 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (60s) 11s ago 59s 49.4M - 8.3.5 dad864ee21e9 9249018f5990 2023-06-10T21:00:29.815 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (52s) 13s ago 52s 17.5M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (50s) 11s ago 49s 13.0M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443 running (5m) 11s ago 5m 392M - 16.2.13-257-gd8c5d349 b644dd95899d 9977c174f9de 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:9283 running (7m) 13s ago 7m 451M - 16.2.13-257-gd8c5d349 b644dd95899d 081c9920c771 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (7m) 13s ago 7m 89.7M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 27c52af4dbab 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (5m) 11s ago 5m 69.2M 2048M 16.2.13-257-gd8c5d349 b644dd95899d a46036733d6f 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (5m) 13s ago 5m 66.0M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 22815652c7a9 2023-06-10T21:00:29.816 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (114s) 13s ago 114s 11.5M - 1.3.1 1dbe0e931976 bf0e1033c7ed 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (110s) 11s ago 109s 13.6M - 1.3.1 1dbe0e931976 afdc33042e26 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (5m) 13s ago 5m 49.5M 4036M 16.2.13-257-gd8c5d349 b644dd95899d ef1eb4e15c36 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (4m) 13s ago 4m 48.2M 4036M 16.2.13-257-gd8c5d349 b644dd95899d 601741de864d 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (4m) 13s ago 4m 43.4M 4036M 16.2.13-257-gd8c5d349 b644dd95899d 8cca00e2828d 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (4m) 13s ago 4m 48.3M 4036M 16.2.13-257-gd8c5d349 b644dd95899d c9c0acd987a8 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (3m) 11s ago 3m 50.6M 4292M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:00:29.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (3m) 11s ago 3m 45.2M 4292M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:00:29.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (3m) 11s ago 3m 48.2M 4292M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:00:29.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (2m) 11s ago 2m 47.5M 4292M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:00:29.818 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (18s) 11s ago 118s 38.6M - 2.33.4 514e6a882f6e a4c5e6078a15 2023-06-10T21:00:30.009 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 12453 12437 155.381 147.438 0.00821366 0.00642777 2023-06-10T21:00:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:30 smithi064 conmon[97224]: audit 2023-06-10T21:00:29.791327+0000 mgr.y (mgr.14427) 106 : audit [DBG] from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:30 smithi064 conmon[97224]: 2023-06-10T21:00:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:30 smithi064 conmon[107122]: audit 2023-06-10T21:00:29.791327 2023-06-10T21:00:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:30 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 106 : audit [DBG] from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:30 smithi152 conmon[97365]: audit 2023-06-10T21:00:29.791327+0000 mgr.y (mgr.14427) 2023-06-10T21:00:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:30 smithi152 conmon[97365]: 106 : audit [DBG] from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:00:31.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 15 15040 15025 156.382 161.75 0.00823791 0.00638734 2023-06-10T21:00:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:31 smithi152 conmon[97365]: cluster 2023-06-10T21:00:30.324335+0000 mgr.y ( 2023-06-10T21:00:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:31 smithi152 conmon[97365]: mgr.14427) 107 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-06-10T21:00:31.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:31 smithi064 conmon[97224]: cluster 2023-06-10T21:00:30.324335+0000 mgr.y (mgr.14427) 107 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-06-10T21:00:31.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:31 smithi064 conmon[107122]: cluster 2023-06-10T21:00:30.324335+0000 mgr.y (mgr.14427) 107 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-06-10T21:00:32.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 17638 17622 157.227 162.312 0.00290449 0.00635315 2023-06-10T21:00:32.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'echo wait for minority of mons to upgrade' 2023-06-10T21:00:33.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 19875 19859 155.05 139.812 0.00684167 0.00644454 2023-06-10T21:00:33.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:00:32 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:00:33.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:33 smithi064 conmon[107122]: cluster 2023-06-10T21:00:32.324791+0000 2023-06-10T21:00:33.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:33 smithi064 conmon[107122]: mgr.y (mgr.14427) 108 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-06-10T21:00:33.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:33 smithi064 conmon[97224]: cluster 2023-06-10T21:00:32 2023-06-10T21:00:33.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:33 smithi064 conmon[97224]: .324791+0000 mgr.y (mgr.14427) 108 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-06-10T21:00:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:33 smithi152 conmon[97365]: cluster 2023-06-10T21:00:32.324791+0000 mgr.y (mgr.14427) 108 : cluster [DBG] 2023-06-10T21:00:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:33 smithi152 conmon[97365]: pgmap v87: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-06-10T21:00:34.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 22042 22026 152.866 135.438 0.00683983 0.00653467 2023-06-10T21:00:35.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 24148 24132 150.741 131.625 0.0102363 0.00662948 2023-06-10T21:00:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:35 smithi064 conmon[97224]: cluster 2023-06-10T21:00:34.326716+0000 mgr.y (mgr. 2023-06-10T21:00:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:35 smithi064 conmon[97224]: 14427) 109 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:00:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:35 smithi064 conmon[107122]: cluster 2023-06-10T21:00:34.326716+0000 mgr.y (mgr 2023-06-10T21:00:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:35 smithi064 conmon[107122]: .14427) 109 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:00:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:35 smithi152 conmon[97365]: cluster 2023-06-10T21:00:34.326716+0000 2023-06-10T21:00:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:35 smithi152 conmon[97365]: mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 825 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:00:35.869 INFO:teuthology.orchestra.run.smithi064.stdout:wait for minority of mons to upgrade 2023-06-10T21:00:36.012 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 26180 26164 148.58 127 0.00493777 0.00672463 2023-06-10T21:00:37.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 27690 27674 144.065 94.375 0.00591135 0.00693575 2023-06-10T21:00:37.299 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:37 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:37] "GET /metrics HTTP/1.1" 200 190635 "" "Prometheus/2.33.4" 2023-06-10T21:00:37.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:37 smithi064 conmon[107122]: cluster 2023-06-10T21:00:36.327418+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:37.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:37 smithi064 conmon[97224]: cluster 2023-06-10T21:00:36.327418+0000 mgr.y 2023-06-10T21:00:37.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:37 smithi064 conmon[97224]: (mgr.14427) 110 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:37.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-06-10T21:00:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:37 smithi152 conmon[97365]: cluster 2023-06-10T21:00:36.327418+0000 2023-06-10T21:00:37.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:37 smithi152 conmon[97365]: mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:38.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 29568 29552 142.01 117.375 0.00855491 0.00703702 2023-06-10T21:00:39.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 31436 31420 140.206 116.75 0.00788239 0.00712832 2023-06-10T21:00:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:39 smithi064 conmon[97224]: cluster 2023-06-10T21:00:38.328719+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-06-10T21:00:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:39 smithi064 conmon[107122]: cluster 2023-06-10T21:00:38.328719+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-06-10T21:00:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:39 smithi152 conmon[97365]: cluster 2023-06-10T21:00:38.328719+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-06-10T21:00:40.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 32946 32930 137.151 94.375 0.00330151 0.00728693 2023-06-10T21:00:41.013 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 34669 34653 135.309 107.688 0.0104859 0.00738452 2023-06-10T21:00:41.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:41 smithi064 conmon[97224]: cluster 2023-06-10T21:00:40.329505+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-06-10T21:00:41.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:41 smithi064 conmon[107122]: cluster 2023-06-10T21:00:40.329505+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-06-10T21:00:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:41 smithi152 conmon[97365]: cluster 2023-06-10T21:00:40.329505+0000 mgr.y (mgr 2023-06-10T21:00:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:41 smithi152 conmon[97365]: .14427) 112 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 169 MiB/s wr, 2.70k op/s 2023-06-10T21:00:42.015 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 36393 36377 133.676 107.75 0.0127811 0.00747621 2023-06-10T21:00:43.020 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 38117 38101 132.201 107.75 0.0114674 0.00755944 2023-06-10T21:00:43.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:00:42 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:00:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:43 smithi152 conmon[97365]: cluster 2023-06-10T21:00:42.329993+0000 mgr.y (mgr.14427) 113 : cluster 2023-06-10T21:00:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:43 smithi152 conmon[97365]: [DBG] pgmap v92: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:43 smithi064 conmon[97224]: cluster 2023-06-10T21:00:42.329993+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:43 smithi064 conmon[107122]: cluster 2023-06-10T21:00:42.329993+0000 mgr.y 2023-06-10T21:00:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:43 smithi064 conmon[107122]: (mgr.14427) 113 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:44.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 39751 39735 130.612 102.125 0.0112818 0.00765124 2023-06-10T21:00:45.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:00:45.018631+0000 min lat: 0.00115805 max lat: 0.322156 avg lat: 0.00773094 2023-06-10T21:00:45.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:00:45.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 41405 41389 129.249 103.375 0.0117678 0.00773094 2023-06-10T21:00:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:45 smithi152 conmon[97365]: cluster 2023-06-10T21:00:44.330424+0000 mgr.y (mgr.14427) 114 : cluster 2023-06-10T21:00:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:45 smithi152 conmon[97365]: [DBG] pgmap v93: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:45 smithi064 conmon[107122]: cluster 2023-06-10T21:00:44.330424+0000 2023-06-10T21:00:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:45 smithi064 conmon[107122]: mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:45 smithi064 conmon[97224]: cluster 2023-06-10T21:00: 2023-06-10T21:00:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:45 smithi064 conmon[97224]: 44.330424+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-06-10T21:00:46.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 43020 43004 127.901 100.938 0.00392537 0.00781203 2023-06-10T21:00:47.021 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 44541 44525 126.409 95.0625 0.00418909 0.0079061 2023-06-10T21:00:47.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:47 smithi064 conmon[97224]: cluster 2023-06-10T21:00:46.331121+0000 mgr.y (mgr 2023-06-10T21:00:47.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:47 smithi064 conmon[97224]: .14427) 115 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-10T21:00:47.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:47 smithi064 conmon[107122]: cluster 2023-06-10T21:00:46 2023-06-10T21:00:47.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:47 smithi064 conmon[107122]: .331121+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-10T21:00:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:47 smithi152 conmon[97365]: cluster 2023-06-10T21:00:46.331121+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-10T21:00:48.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 46038 46022 124.977 93.5625 0.00579896 0.00799652 2023-06-10T21:00:49.022 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 47572 47556 123.764 95.875 0.00943838 0.00807435 2023-06-10T21:00:49.662 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:47 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:47] "GET /metrics HTTP/1.1" 200 190972 "" "Prometheus/2.33.4" 2023-06-10T21:00:49.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:49 smithi064 conmon[97224]: cluster 2023-06-10T21:00:48.331648+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:00:49.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:49 smithi064 conmon[107122]: cluster 2023-06-10T21:00:48.331648+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:00:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:49 smithi152 conmon[97365]: cluster 2023-06-10T21:00:48.331648+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 2.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:00:50.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 49100 49084 122.614 95.5 0.00448335 0.00815002 2023-06-10T21:00:51.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 50532 50516 121.341 89.5 0.0149904 0.00823764 2023-06-10T21:00:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:51 smithi064 conmon[107122]: cluster 2023-06-10T21:00:50.332331+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:51 smithi064 conmon[97224]: cluster 2023-06-10T21:00:50.332331+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:51 smithi152 conmon[97365]: cluster 2023-06-10T21:00:50.332331+0000 mgr.y (mgr.14427) 117 2023-06-10T21:00:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:51 smithi152 conmon[97365]: : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:00:52.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 51879 51863 119.965 84.1875 0.0200889 0.00833111 2023-06-10T21:00:53.026 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 53310 53294 118.876 89.4375 0.0083561 0.00840696 2023-06-10T21:00:53.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:00:52 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:00:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:53 smithi064 conmon[97224]: cluster 2023-06-10T21:00:52.332753+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:53 smithi064 conmon[107122]: cluster 2023-06-10T21:00:52.332753+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:53.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:53 smithi152 conmon[97365]: cluster 2023-06-10T21:00:52.332753+0000 mgr.y (mgr. 2023-06-10T21:00:53.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:53 smithi152 conmon[97365]: 14427) 118 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:54.029 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 54668 54652 117.703 84.875 0.00804484 0.0084902 2023-06-10T21:00:55.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 56171 56155 116.911 93.9375 0.0136143 0.00854793 2023-06-10T21:00:55.683 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:55 smithi064 conmon[97224]: cluster 2023-06-10T21:00:54.333170+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:55.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:55 smithi064 conmon[107122]: cluster 2023-06-10T21:00:54. 2023-06-10T21:00:55.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:55 smithi064 conmon[107122]: 333170+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:55 smithi152 conmon[97365]: cluster 2023-06-10T21:00:54.333170+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-10T21:00:56.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 57558 57542 115.937 86.6875 0.00810459 0.00861902 2023-06-10T21:00:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:56 smithi152 conmon[97365]: audit 2023-06-10T21:00:55.592112+0000 mon.c (mon.2) 26 2023-06-10T21:00:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:56 smithi152 conmon[97365]: : audit [DBG] from='client.? 172.21.15.64:0/242516415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:56.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:56 smithi064 conmon[97224]: audit 2023-06-10T21:00:55.592112+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.64:0/242516415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:56.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:56 smithi064 conmon[107122]: audit 2023-06-10T21:00:55.592112+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.64:0/242516415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:57.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 59118 59102 115.361 97.5 0.0122645 0.00866362 2023-06-10T21:00:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:57 smithi064 conmon[97224]: cluster 2023-06-10T21:00:56.333805+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-06-10T21:00:57.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:57 smithi064 conmon[107122]: cluster 2023-06-10T21:00:56.333805+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-06-10T21:00:57.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:00:57 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:00:57] "GET /metrics HTTP/1.1" 200 190972 "" "Prometheus/2.33.4" 2023-06-10T21:00:57.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:57 smithi152 conmon[97365]: cluster 2023-06-10T21:00:56.333805+0000 mgr.y ( 2023-06-10T21:00:57.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:57 smithi152 conmon[97365]: mgr.14427) 120 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-06-10T21:00:58.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 60607 60591 114.685 93.0625 0.00843269 0.00871467 2023-06-10T21:00:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:58 smithi064 conmon[107122]: audit 2023-06-10T21:00:58 2023-06-10T21:00:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:58 smithi064 conmon[107122]: .237724+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.64:0/85740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:58 smithi064 conmon[97224]: audit 2023-06-10T21:00:58.237724+0000 mon.c (mon. 2023-06-10T21:00:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:58 smithi064 conmon[97224]: 2) 27 : audit [DBG] from='client.? 172.21.15.64:0/85740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:58 smithi152 conmon[97365]: audit 2023-06-10T21:00:58.237724+0000 mon.c (mon. 2023-06-10T21:00:58.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:58 smithi152 conmon[97365]: 2) 27 : audit [DBG] from='client.? 172.21.15.64:0/85740088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:00:59.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 62021 62005 113.911 88.375 0.00849507 0.00877388 2023-06-10T21:00:59.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:00:59 smithi064 conmon[107122]: cluster 2023-06-10T21:00:58.334199+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-10T21:00:59.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:59 smithi064 conmon[97224]: cluster 2023-06-10T21:00:58.334199 2023-06-10T21:00:59.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:00:59 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-10T21:00:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:59 smithi152 conmon[97365]: cluster 2023-06-10T21:00:58.334199 2023-06-10T21:00:59.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:00:59 smithi152 conmon[97365]: +0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-10T21:01:00.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 63440 63424 113.191 88.6875 0.0108496 0.00882963 2023-06-10T21:01:01.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 64829 64813 112.458 86.8125 0.0103855 0.00888757 2023-06-10T21:01:01.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:01 smithi152 conmon[97365]: cluster 2023-06-10T21:01:00.334944+0000 mgr.y (mgr.14427) 122 : cluster 2023-06-10T21:01:01.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:01 smithi152 conmon[97365]: [DBG] pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-10T21:01:01.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:01 smithi152 conmon[97365]: audit 2023-06-10T21:01:00.860666+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.64:0/2668222719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:01 smithi064 conmon[97224]: cluster 2023-06-10T21:01:00.334944+0000 mgr.y (mgr.14427) 122 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-10T21:01:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:01 smithi064 conmon[97224]: audit 2023-06-10T21:01:00.860666+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.64:0/2668222719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:01 smithi064 conmon[107122]: cluster 2023-06-10T21:01:00.334944+0000 mgr.y (mgr.14427) 122 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-10T21:01:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:01 smithi064 conmon[107122]: audit 2023-06-10T21:01:00.860666+0000 mon.a (mon.0) 744 : audit [DBG] from='client.? 172.21.15.64:0/2668222719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:02.027 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 66221 66205 111.77 87 0.00919785 0.00894224 2023-06-10T21:01:03.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 67629 67613 111.145 88 0.00921168 0.00899267 2023-06-10T21:01:03.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:02 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:03 smithi064 conmon[97224]: cluster 2023-06-10T21:01:02.337587+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:03 smithi064 conmon[107122]: cluster 2023-06-10T21:01:02. 2023-06-10T21:01:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:03 smithi064 conmon[107122]: 337587+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:03 smithi152 conmon[97365]: cluster 2023-06-10T21:01:02.337587+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:04.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 68994 68978 110.483 85.3125 0.0153203 0.00904654 2023-06-10T21:01:04.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:04 smithi064 conmon[107122]: audit 2023-06-10T21:01:03 2023-06-10T21:01:04.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:04 smithi064 conmon[107122]: .492489+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.64:0/3074952955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:04 smithi064 conmon[97224]: audit 2023-06-10T21:01:03.492489+0000 mon.c (mon.2 2023-06-10T21:01:04.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:04 smithi064 conmon[97224]: ) 28 : audit [DBG] from='client.? 172.21.15.64:0/3074952955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:04 smithi152 conmon[97365]: audit 2023-06-10T21:01:03.492489+0000 mon.c (mon.2) 28 : 2023-06-10T21:01:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:04 smithi152 conmon[97365]: audit [DBG] from='client.? 172.21.15.64:0/3074952955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:05.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:01:05.025463+0000 min lat: 0.00115805 max lat: 0.322156 avg lat: 0.00909582 2023-06-10T21:01:05.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:01:05.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 70383 70367 109.891 86.8125 0.0127822 0.00909582 2023-06-10T21:01:05.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:05 smithi064 conmon[97224]: cluster 2023-06-10T21:01:04.337996+0000 2023-06-10T21:01:05.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:05 smithi064 conmon[97224]: mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:05 smithi064 conmon[107122]: cluster 2023-06-10T21:01:04.337996+0000 2023-06-10T21:01:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:05 smithi064 conmon[107122]: mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:05 smithi152 conmon[97365]: cluster 2023-06-10T21:01:04.337996+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:06.028 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 71683 71667 109.192 81.25 0.0158513 0.0091537 2023-06-10T21:01:07.030 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 73025 73009 108.583 83.875 0.00927849 0.00920506 2023-06-10T21:01:07.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:07] "GET /metrics HTTP/1.1" 200 191018 "" "Prometheus/2.33.4" 2023-06-10T21:01:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[97224]: audit 2023-06-10T21:01:06.109521+0000 2023-06-10T21:01:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[97224]: mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.64:0/3993563665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[97224]: cluster 2023-06-10T21:01:06.338711+0000 mgr.y (mgr.14427) 125 : cluster [DBG] 2023-06-10T21:01:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[97224]: pgmap v104: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-10T21:01:07.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[107122]: audit 2023-06-10T21:01:06 2023-06-10T21:01:07.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[107122]: .109521+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.64:0/3993563665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:07.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[107122]: cluster 2023-06-10T21:01:06.338711+0000 2023-06-10T21:01:07.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:07 smithi064 conmon[107122]: mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v104: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-10T21:01:07.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:07 smithi152 conmon[97365]: audit 2023-06-10T21:01:06.109521+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.64:0/3993563665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:07.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:07 smithi152 conmon[97365]: cluster 2023-06-10T21:01:06.338711+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v104: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-10T21:01:08.032 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 74357 74341 107.991 83.25 0.00930806 0.00925561 2023-06-10T21:01:09.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 75614 75598 107.318 78.5625 0.0132661 0.00931368 2023-06-10T21:01:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:09 smithi064 conmon[97224]: cluster 2023-06-10T21:01:08.339137+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:01:09.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:09 smithi064 conmon[97224]: audit 2023-06-10T21:01:08.712870+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.64:0/1615996384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:09 smithi064 conmon[107122]: cluster 2023-06-10T21:01:08.339137+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:01:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:09 smithi064 conmon[107122]: audit 2023-06-10T21:01:08.712870+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.64:0/1615996384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:09.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:09 smithi152 conmon[97365]: cluster 2023-06-10T21:01:08.339137+0000 mgr.y (mgr.14427) 126 : 2023-06-10T21:01:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:09 smithi152 conmon[97365]: cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:01:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:09 smithi152 conmon[97365]: audit 2023-06-10T21:01:08.712870+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.64:0/1615996384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:10.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 76912 76896 106.736 81.125 0.0143665 0.00936474 2023-06-10T21:01:11.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 78183 78167 106.143 79.4375 0.0127081 0.0094172 2023-06-10T21:01:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:11 smithi152 conmon[97365]: cluster 2023-06-10T21:01:10.339846+0000 mgr.y (mgr.14427 2023-06-10T21:01:11.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:11 smithi152 conmon[97365]: ) 127 : cluster [DBG] pgmap v106: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-06-10T21:01:11.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:11 smithi152 conmon[97365]: audit 2023-06-10T21:01:11.298953+0000 mon.b (mon.1) 25 : audit [DBG] from='client.? 172.21.15.64:0/525817419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:11.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[97224]: cluster 2023-06-10T21:01:10.339846+0000 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[97224]: mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v106: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[97224]: audit 2023-06-10T21:01:11.298953+0000 mon.b (mon.1) 25 : audit [DBG] from='client.? 172.21.15.64:0/525817419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[107122]: cluster 2023-06-10T21:01:10.339846+0000 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[107122]: mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v106: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.66k op/s 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[107122]: audit 2023-06-10T21:01:11.298953+0000 mon.b (mon.1) 25 : audit [DBG] 2023-06-10T21:01:11.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:11 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/525817419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:12.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 79325 79309 105.403 71.375 0.0128002 0.00948202 2023-06-10T21:01:13.034 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 80498 80482 104.735 73.3125 0.0144609 0.00954315 2023-06-10T21:01:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: audit 2023-06-10T21:01:11.740207+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: cephadm 2023-06-10T21:01:11.740987+0000 mgr.y (mgr.14427) 128 : cephadm [INF] Upgrade: Target is version 18.0.0-3603-g5937c68a (unknown) 2023-06-10T21:01:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: cephadm 2023-06-10T21:01:11.741063+0000 mgr.y (mgr.14427) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143'] 2023-06-10T21:01:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: audit 2023-06-10T21:01:11.742385+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:01:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: audit 2023-06-10T21:01:11.759064+0000 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: mon.a (mon.0) 747 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[97224]: cephadm 2023-06-10T21:01:11.760420+0000 mgr.y (mgr.14427) 130 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: audit 2023-06-10T21:01:11. 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 740207+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: cephadm 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 2023-06-10T21:01:11.740987+0000 mgr.y (mgr.14427) 2023-06-10T21:01:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 128 : cephadm [INF] Upgrade: Target is version 18.0.0-3603-g5937c68a (unknown) 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: T21:01:11.741063+0000 mgr.y ( 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: mgr.14427) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143'] 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: audit 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 2023-06-10T 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 21:01:11.742385+0000 mon.a (mon 2023-06-10T21:01:13.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: .0) 746 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: 06-10T21:01:11.759064+0000 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: mon.a (mon.0) 747 : audit [INF] 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: T21:01:11.760420+0000 mgr.y ( 2023-06-10T21:01:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:12 smithi064 conmon[107122]: mgr.14427) 130 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:01:13.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:01:13.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: -10T21:01:11.740207+0000 mon.a (mon.0) 745 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: cephadm 2023-06-10T21:01:11.740987+0000 mgr.y (mgr.14427) 128 : cephadm [INF] Upgrade: Target is version 18.0.0-3603-g5937c68a (unknown) 2023-06-10T21:01:13.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: cephadm 2023-06-10T21:01:11.741063+0000 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: mgr.y (mgr.14427) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143'] 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: audit 2023-06-10T21:01:11 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: .742385+0000 mon.a (mon.0) 746 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: audit 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: 2023-06-10T21:01:11.759064+0000 mon.a (mon.0) 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: 747 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:01:13.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: cephadm 2023-06-10T21:01 2023-06-10T21:01:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: :11.760420+0000 mgr.y (mgr.14427) 130 : 2023-06-10T21:01:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[97365]: cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:01:13.200 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:12 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:14.035 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 81580 81564 103.975 67.625 0.0172926 0.00961248 2023-06-10T21:01:14.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[97224]: cephadm 2023-06-10T21:01:12.309320+0000 mgr.y (mgr.14427) 131 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 on smithi152 2023-06-10T21:01:14.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[97224]: cluster 2023-06-10T21:01:12.340850+0000 mgr.y (mgr.14427) 132 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[107122]: cephadm 2023-06-10T21:01:12.309320 2023-06-10T21:01:14.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 131 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 on smithi152 2023-06-10T21:01:14.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[107122]: cluster 2023 2023-06-10T21:01:14.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[107122]: -06-10T21:01:12.340850+0000 mgr.y (mgr.14427) 132 : 2023-06-10T21:01:14.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:13 smithi064 conmon[107122]: cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:13 smithi152 conmon[97365]: cephadm 2023-06-10T21:01:12.309320+0000 mgr.y (mgr.14427) 131 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 on smithi152 2023-06-10T21:01:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:13 smithi152 conmon[97365]: cluster 2023-06-10T21:01:12.340850+0000 mgr.y (mgr.14427) 132 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:15.037 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 82652 82636 103.232 67 0.0223576 0.00968233 2023-06-10T21:01:15.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:14 smithi064 conmon[107122]: audit 2023-06-10T21:01:13 2023-06-10T21:01:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:14 smithi064 conmon[107122]: .884853+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.64:0/3097778969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:15.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:14 smithi064 conmon[97224]: audit 2023-06-10T21:01:13.884853 2023-06-10T21:01:15.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:14 smithi064 conmon[97224]: +0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.64:0/3097778969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:15.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:14 smithi152 conmon[97365]: audit 2023-06-10T21:01:13.884853+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.64:0/3097778969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:16.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 83738 83722 102.538 67.875 0.0285363 0.00974781 2023-06-10T21:01:16.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:15 smithi064 conmon[97224]: cluster 2023-06-10T21:01:14.341247+0000 mgr.y (mgr. 2023-06-10T21:01:16.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:15 smithi064 conmon[97224]: 14427) 133 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-10T21:01:16.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:15 smithi064 conmon[107122]: cluster 2023-06-10T21:01:14. 2023-06-10T21:01:16.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:15 smithi064 conmon[107122]: 341247+0000 mgr.y (mgr.14427) 133 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-10T21:01:16.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:15 smithi152 conmon[97365]: cluster 2023-06-10T21:01:14.341247+0000 mgr.y (mgr.14427) 133 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-10T21:01:17.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 84775 84759 101.813 64.8125 0.00762846 0.00981749 2023-06-10T21:01:17.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:17 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:17] "GET /metrics HTTP/1.1" 200 191042 "" "Prometheus/2.33.4" 2023-06-10T21:01:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:17 smithi064 conmon[107122]: cluster 2023-06-10T21:01:16.341945+0000 mgr.y (mgr.14427) 134 : cluster [DBG] pgmap v109: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-10T21:01:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:17 smithi064 conmon[107122]: audit 2023-06-10T21:01:16.478836+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.64:0/841991887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:17 smithi064 conmon[97224]: cluster 2023-06-10T21:01:16.341945+0000 mgr.y (mgr.14427) 134 : cluster [DBG] pgmap v109: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-10T21:01:17.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:17 smithi064 conmon[97224]: audit 2023-06-10T21:01:16.478836+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.64:0/841991887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:17 smithi152 conmon[97365]: cluster 2023-06-10T21:01:16.341945+0000 mgr.y 2023-06-10T21:01:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:17 smithi152 conmon[97365]: (mgr.14427) 134 : cluster [DBG] pgmap v109: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-10T21:01:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:17 smithi152 conmon[97365]: audit 2023-06-10T21:01:16.478836+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.64:0/841991887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:18.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 85813 85797 101.117 64.875 0.00722713 0.00988525 2023-06-10T21:01:19.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 86817 86801 100.406 62.75 0.0154924 0.0099546 2023-06-10T21:01:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:19 smithi152 conmon[97365]: cluster 2023-06-10T21:01:18.344282+0000 mgr.y (mgr.14427) 135 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-10T21:01:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:19 smithi152 conmon[97365]: audit 2023-06-10T21:01:19.061043+0000 mon.b (mon.1) 26 : audit [DBG] from='client.? 172.21.15.64:0/462874614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:19.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[107122]: cluster 2023-06-10T21:01:18.344282 2023-06-10T21:01:19.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 135 : cluster [DBG] pgmap v110: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-10T21:01:19.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:01:19.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[107122]: 01:19.061043+0000 mon.b (mon.1) 26 : audit [DBG] from='client.? 172.21.15.64:0/462874614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:19.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[97224]: cluster 2023-06-10T21:01:18.344282+0000 mgr.y (mgr.14427) 135 : cluster 2023-06-10T21:01:19.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[97224]: [DBG] pgmap v110: 89 pgs: 89 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-10T21:01:19.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:19 smithi064 conmon[97224]: audit 2023-06-10T21:01:19.061043+0000 mon.b (mon.1) 26 : audit [DBG] from='client.? 172.21.15.64:0/462874614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:20.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 87860 87844 99.7662 65.1875 0.015291 0.0100183 2023-06-10T21:01:21.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 88891 88875 99.1356 64.4375 0.0115768 0.0100827 2023-06-10T21:01:21.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:21 smithi152 conmon[97365]: cluster 2023-06-10T21:01:20.344790+0000 mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:01:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:21 smithi064 conmon[97224]: cluster 2023-06-10T21:01:20.344790+0000 2023-06-10T21:01:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:21 smithi064 conmon[97224]: mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:01:21.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:21 smithi064 conmon[107122]: cluster 2023-06-10T21:01:20.344790 2023-06-10T21:01:21.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:21 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:01:22.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 89868 89852 98.4675 61.0625 0.0117198 0.01015 2023-06-10T21:01:22.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:22 smithi152 conmon[97365]: audit 2023-06-10T21:01:21.621375+0000 mon.a (mon.0) 749 : audit [DBG] 2023-06-10T21:01:22.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:22 smithi152 conmon[97365]: from='client.? 172.21.15.64:0/2105647508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:22.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:22 smithi064 conmon[107122]: audit 2023-06-10T21:01:21.621375+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.64:0/2105647508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:22.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:22 smithi064 conmon[97224]: audit 2023-06-10T21:01:21.621375+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.64:0/2105647508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:23.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 90858 90842 97.8368 61.875 0.00971528 0.010217 2023-06-10T21:01:23.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:22 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:23.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:23 smithi152 conmon[97365]: cluster 2023-06-10T21:01:22.345261+0000 mgr.y (mgr.14427) 137 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-06-10T21:01:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:23 smithi064 conmon[107122]: cluster 2023-06-10T21:01: 2023-06-10T21:01:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:23 smithi064 conmon[107122]: 22.345261+0000 mgr.y (mgr.14427) 137 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-06-10T21:01:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:23 smithi064 conmon[97224]: cluster 2023-06-10T21:01:22.345261+0000 mgr.y (mgr 2023-06-10T21:01:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:23 smithi064 conmon[97224]: .14427) 137 : cluster [DBG] pgmap v112: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-06-10T21:01:24.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 91854 91838 97.2338 62.25 0.0144208 0.0102801 2023-06-10T21:01:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:24 smithi064 conmon[107122]: audit 2023-06-10T21:01:24 2023-06-10T21:01:24.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:24 smithi064 conmon[107122]: .184557+0000 mon.a (mon.0) 750 : audit [DBG] from='client.? 172.21.15.64:0/2216586601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:24.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:24 smithi064 conmon[97224]: audit 2023-06-10T21:01:24.184557+0000 mon.a (mon.0) 750 2023-06-10T21:01:24.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:24 smithi064 conmon[97224]: : audit [DBG] from='client.? 172.21.15.64:0/2216586601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:24 smithi152 conmon[97365]: audit 2023-06-10T21:01:24.184557+0000 mon.a (mon.0) 750 : audit [DBG] from='client.? 172.21.15.64:0/2216586601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:25.038 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:01:25.036305+0000 min lat: 0.00115805 max lat: 0.322156 avg lat: 0.0103421 2023-06-10T21:01:25.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:01:25.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 92843 92827 96.6435 61.8125 0.0144173 0.0103421 2023-06-10T21:01:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:25 smithi064 conmon[107122]: cluster 2023-06-10T21:01:24.345646+0000 2023-06-10T21:01:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:25 smithi064 conmon[107122]: mgr.y (mgr.14427) 138 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:01:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:25 smithi064 conmon[97224]: cluster 2023-06-10T21:01:24.345646+0000 mgr.y 2023-06-10T21:01:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:25 smithi064 conmon[97224]: (mgr.14427) 138 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:01:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:25 smithi152 conmon[97365]: cluster 2023-06-10T21:01: 2023-06-10T21:01:25.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:25 smithi152 conmon[97365]: 24.345646+0000 mgr.y (mgr.14427) 138 : cluster [DBG] pgmap v113: 89 pgs: 89 active+clean; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:01:26.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 93813 93797 96.0533 60.625 0.01495 0.0104056 2023-06-10T21:01:27.039 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 94748 94732 95.4468 58.4375 0.0182519 0.0104715 2023-06-10T21:01:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[107122]: cluster 2023-06-10T21:01:26.346426+0000 mgr.y 2023-06-10T21:01:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[107122]: (mgr.14427) 139 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-06-10T21:01:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[107122]: audit 2023-06-10T21:01:26.744629+0000 mon.a (mon.0) 751 : audit [DBG] from='client.? 172.21.15.64:0/3259059557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[97224]: cluster 2023-06-10T21:01:26.346426+0000 mgr.y (mgr.14427) 139 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-06-10T21:01:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[97224]: audit 2023-06-10T21:01:26.744629+0000 mon.a (mon. 2023-06-10T21:01:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[97224]: 0) 751 : audit [DBG] from='client.? 172.21.15.64:0/3259059557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:27.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:27 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:27] "GET /metrics HTTP/1.1" 200 191042 "" "Prometheus/2.33.4" 2023-06-10T21:01:27.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:27 smithi152 conmon[97365]: cluster 2023-06-10T21:01:26.346426+0000 mgr.y (mgr 2023-06-10T21:01:27.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:27 smithi152 conmon[97365]: .14427) 139 : cluster [DBG] pgmap v114: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2023-06-10T21:01:27.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:27 smithi152 conmon[97365]: audit 2023-06-10T21:01:26.744629+0000 mon.a (mon.0) 751 : audit [DBG] from='client.? 172.21.15.64:0/3259059557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:28.042 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 95711 95695 94.8829 60.1875 0.0191154 0.0105349 2023-06-10T21:01:29.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 96656 96640 94.3221 59.0625 0.0147732 0.0105971 2023-06-10T21:01:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[107122]: cluster 2023-06-10T21:01:28.346785+0000 2023-06-10T21:01:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[107122]: mgr.y (mgr.14427) 140 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-06-10T21:01:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[107122]: audit 2023-06-10T21:01:29.336318+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.64:0/3700731270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:29.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[97224]: cluster 2023-06-10T21:01:28.346785+0000 mgr.y ( 2023-06-10T21:01:29.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[97224]: mgr.14427) 140 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-06-10T21:01:29.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[97224]: audit 2023-06-10T21:01:29.336318+0000 mon.a ( 2023-06-10T21:01:29.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:29 smithi064 conmon[97224]: mon.0) 752 : audit [DBG] from='client.? 172.21.15.64:0/3700731270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:29.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:29 smithi152 conmon[97365]: cluster 2023-06-10T21:01:28.346785+0000 mgr.y (mgr.14427) 140 : cluster [DBG] pgmap v115: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-06-10T21:01:29.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:29 smithi152 conmon[97365]: audit 2023-06-10T21:01:29.336318+0000 mon.a (mon.0) 752 : audit [DBG] from='client.? 172.21.15.64:0/3700731270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:30.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 97592 97576 93.7711 58.5 0.0228627 0.0106598 2023-06-10T21:01:31.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 98521 98505 93.2303 58.0625 0.0116869 0.0107218 2023-06-10T21:01:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:31 smithi152 conmon[97365]: cluster 2023-06-10T21:01:30.347383+0000 mgr.y (mgr 2023-06-10T21:01:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:31 smithi152 conmon[97365]: .14427) 141 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:31.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:31 smithi064 conmon[97224]: cluster 2023-06-10T21:01:31.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:31 smithi064 conmon[97224]: 2023-06-10T21:01:30.347383+0000 mgr.y (mgr.14427) 141 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:31.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:31 smithi064 conmon[107122]: cluster 2023-06-10T21:01:30.347383+0000 mgr.y (mgr.14427) 141 : cluster [DBG] pgmap v116: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:01:32.043 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 99509 99493 92.7605 61.75 0.0119993 0.0107758 2023-06-10T21:01:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:32 smithi152 conmon[97365]: audit 2023-06-10T21:01:31.935590+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.64:0/1495754027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:32.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:32 smithi064 conmon[97224]: audit 2023-06-10T21:01:31.935590+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.64:0/1495754027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:32.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:32 smithi064 conmon[107122]: audit 2023-06-10T21:01:31.935590+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.64:0/1495754027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:33.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 100563 100547 92.3603 65.875 0.0124286 0.0108225 2023-06-10T21:01:33.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:32 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:33 smithi152 conmon[97365]: cluster 2023-06-10T21:01:32.347817+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:01:33.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:33 smithi064 conmon[107122]: cluster 2023-06-10T21:01:32.347817+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:01:33.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:33 smithi064 conmon[97224]: cluster 2023-06-10T21:01 2023-06-10T21:01:33.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:33 smithi064 conmon[97224]: :32.347817+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v117: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:01:34.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 101438 101422 91.8146 54.6875 0.00612413 0.0108866 2023-06-10T21:01:35.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 102263 102247 91.2397 51.5625 0.0266263 0.0109551 2023-06-10T21:01:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[97224]: cluster 2023-06-10T21:01:34.348169+0000 mgr.y (mgr.14427) 143 : cluster [DBG] pgmap v118: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-06-10T21:01:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[97224]: audit 2023-06-10T21:01:34.503549+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.64:0/1030251797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[107122]: cluster 2023-06-10T21:01:34.348169+0000 mgr.y (mgr.14427) 143 : cluster [DBG] 2023-06-10T21:01:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[107122]: pgmap v118: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-06-10T21:01:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[107122]: audit 2023-06-10T21:01:34.503549+0000 mon.c (mon.2) 33 : audit 2023-06-10T21:01:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:35 smithi064 conmon[107122]: [DBG] from='client.? 172.21.15.64:0/1030251797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:35 smithi152 conmon[97365]: cluster 2023-06-10T21:01:34.348169+0000 mgr.y (mgr.14427) 143 : cluster [DBG] pgmap v118: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2023-06-10T21:01:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:35 smithi152 conmon[97365]: audit 2023-06-10T21:01:34.503549+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.64:0/1030251797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:36.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 103107 103091 90.6978 52.75 0.0125283 0.0110198 2023-06-10T21:01:37.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 103987 103971 90.2022 55 0.0127659 0.0110811 2023-06-10T21:01:37.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:37 smithi064 conmon[97224]: cluster 2023-06-10T21:01:36.348857+0000 mgr.y (mgr.14427) 144 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:37.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:37 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:37] "GET /metrics HTTP/1.1" 200 191083 "" "Prometheus/2.33.4" 2023-06-10T21:01:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:37 smithi064 conmon[107122]: cluster 2023-06-10T21:01:36.348857+0000 mgr.y (mgr. 2023-06-10T21:01:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:37 smithi064 conmon[107122]: 14427) 144 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:37.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:37 smithi152 conmon[97365]: cluster 2023-06-10T21:01:36.348857+0000 mgr.y (mgr 2023-06-10T21:01:37.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:37 smithi152 conmon[97365]: .14427) 144 : cluster [DBG] pgmap v119: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:01:38.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 104829 104813 89.6876 52.625 0.0180374 0.011144 2023-06-10T21:01:38.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:38 smithi064 conmon[97224]: audit 2023-06-10T21:01:37.074488+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.64:0/3539998007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:38 smithi064 conmon[107122]: audit 2023-06-10T21:01:37.074488+0000 mon.c (mon 2023-06-10T21:01:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:38 smithi064 conmon[107122]: .2) 34 : audit [DBG] from='client.? 172.21.15.64:0/3539998007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:38 smithi152 conmon[97365]: audit 2023-06-10T21:01:37.074488+0000 mon.c 2023-06-10T21:01:38.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:38 smithi152 conmon[97365]: (mon.2) 34 : audit [DBG] from='client.? 172.21.15.64:0/3539998007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:39.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 105675 105659 89.1903 52.875 0.0192907 0.0112062 2023-06-10T21:01:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:39 smithi064 conmon[97224]: cluster 2023-06-10T21:01:38. 2023-06-10T21:01:39.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:39 smithi064 conmon[97224]: 349227+0000 mgr.y (mgr.14427) 145 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:39.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:39 smithi064 conmon[107122]: cluster 2023-06-10T21:01: 2023-06-10T21:01:39.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:39 smithi064 conmon[107122]: 38.349227+0000 mgr.y (mgr.14427) 145 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:39 smithi152 conmon[97365]: cluster 2023-06-10T21:01:38.349227+0000 mgr.y (mgr. 2023-06-10T21:01:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:39 smithi152 conmon[97365]: 14427) 145 : cluster [DBG] pgmap v120: 89 pgs: 89 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:40.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 106510 106494 88.6971 52.1875 0.0142581 0.01127 2023-06-10T21:01:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:40 smithi064 conmon[97224]: audit 2023-06-10T21:01:39.646145+0000 mon.c 2023-06-10T21:01:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:40 smithi064 conmon[97224]: (mon.2) 35 : audit [DBG] from='client.? 172.21.15.64:0/3537949786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:40 smithi064 conmon[107122]: audit 2023-06-10T21:01 2023-06-10T21:01:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:40 smithi064 conmon[107122]: :39.646145+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.64:0/3537949786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:40 smithi152 conmon[97365]: audit 2023-06-10T21:01:39.646145+0000 mon.c (mon. 2023-06-10T21:01:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:40 smithi152 conmon[97365]: 2) 35 : audit [DBG] from='client.? 172.21.15.64:0/3537949786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:41.047 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 107360 107344 88.2292 53.125 0.0249547 0.0113295 2023-06-10T21:01:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:41 smithi064 conmon[107122]: cluster 2023-06-10T21:01 2023-06-10T21:01:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:41 smithi064 conmon[107122]: :40.349761+0000 mgr.y (mgr.14427) 146 : cluster [DBG] pgmap v121: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:01:41.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:41 smithi064 conmon[97224]: cluster 2023-06-10T21:01:40.349761+0000 mgr.y (mgr.14427) 146 : cluster 2023-06-10T21:01:41.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:41 smithi064 conmon[97224]: [DBG] pgmap v121: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:01:41.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:41 smithi152 conmon[97365]: cluster 2023-06-10T21:01:40.349761+0000 mgr.y (mgr.14427) 146 : cluster [DBG] pgmap v121: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:01:42.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 108164 108148 87.7359 50.25 0.0158821 0.0113928 2023-06-10T21:01:42.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:42 smithi064 conmon[107122]: audit 2023-06-10T21:01:42.212942+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.64:0/1787035895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:42.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:42 smithi064 conmon[97224]: audit 2023-06-10T21:01:42.212942+0000 2023-06-10T21:01:42.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:42 smithi064 conmon[97224]: mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.64:0/1787035895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:42 smithi152 conmon[97365]: audit 2023-06-10T21:01:42.212942+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.64:0/1787035895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:43.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 108999 108983 87.2803 52.1875 0.014237 0.0114521 2023-06-10T21:01:43.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:42 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:43.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:43 smithi064 conmon[107122]: cluster 2023-06-10T21:01:42.350188+0000 mgr.y (mgr.14427) 147 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:01:43.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:43 smithi064 conmon[97224]: cluster 2023-06-10T21:01:42.350188+0000 2023-06-10T21:01:43.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:43 smithi064 conmon[97224]: mgr.y (mgr.14427) 147 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:01:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:43 smithi152 conmon[97365]: cluster 2023-06-10T21:01:42.350188+0000 mgr.y (mgr 2023-06-10T21:01:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:43 smithi152 conmon[97365]: .14427) 147 : cluster [DBG] pgmap v122: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:01:44.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 109862 109846 86.8584 53.9375 0.00980726 0.0115084 2023-06-10T21:01:45.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:01:45.045604+0000 min lat: 0.00115805 max lat: 0.322156 avg lat: 0.0115671 2023-06-10T21:01:45.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:01:45.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 110679 110663 86.4111 51.0625 0.0140609 0.0115671 2023-06-10T21:01:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:45 smithi152 conmon[97365]: cluster 2023-06-10T21:01:44.350603+0000 mgr.y (mgr.14427) 148 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:01:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:45 smithi152 conmon[97365]: audit 2023-06-10T21:01:44.777692+0000 mon.b (mon.1) 27 : audit [DBG] from='client.? 172.21.15.64:0/3645433268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:45.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[107122]: cluster 2023-06-10T21:01:44.350603+0000 mgr.y ( 2023-06-10T21:01:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[107122]: mgr.14427) 148 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:01:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[107122]: audit 2023-06-10T21:01:44.777692+0000 mon.b (mon.1) 27 : audit [DBG] 2023-06-10T21:01:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/3645433268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[97224]: cluster 2023-06-10T21:01:44.350603+0000 2023-06-10T21:01:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[97224]: mgr.y (mgr.14427) 148 : cluster [DBG] pgmap v123: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:01:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[97224]: audit 2023-06-10T21:01:44.777692+0000 mon.b (mon.1) 27 : audit [DBG] 2023-06-10T21:01:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:45 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/3645433268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:46.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 111454 111438 85.9424 48.4375 0.00484561 0.0116301 2023-06-10T21:01:47.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 112254 112238 85.5042 50 0.0346226 0.0116898 2023-06-10T21:01:47.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:47 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:47] "GET /metrics HTTP/1.1" 200 191101 "" "Prometheus/2.33.4" 2023-06-10T21:01:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:47 smithi064 conmon[107122]: cluster 2023-06-10T21:01:46.351161+0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:01:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:47 smithi064 conmon[97224]: cluster 2023-06-10T21:01:46.351161 2023-06-10T21:01:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:47 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:01:47.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:47 smithi152 conmon[97365]: cluster 2023-06-10T21:01:46.351161+0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v124: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:01:48.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 113098 113082 85.1097 52.75 0.0277951 0.0117439 2023-06-10T21:01:48.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:48 smithi064 conmon[107122]: audit 2023-06-10T21:01:47.319694+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.64:0/3346931582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:48.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:48 smithi064 conmon[97224]: audit 2023-06-10T21:01:47.319694+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.64:0/3346931582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:48 smithi152 conmon[97365]: audit 2023-06-10T21:01:47.319694+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.64:0/3346931582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:49.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 113914 113898 84.7037 51 0.00465608 0.0117999 2023-06-10T21:01:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:49 smithi064 conmon[97224]: cluster 2023-06-10T21:01:48.351572 2023-06-10T21:01:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:49 smithi064 conmon[97224]: +0000 mgr.y (mgr.14427) 150 : cluster [DBG] pgmap v125: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-06-10T21:01:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:49 smithi064 conmon[107122]: cluster 2023-06-10T21:01:48.351572+0000 mgr.y (mgr. 2023-06-10T21:01:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:49 smithi064 conmon[107122]: 14427) 150 : cluster [DBG] pgmap v125: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-06-10T21:01:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:49 smithi152 conmon[97365]: cluster 2023-06-10T21:01:48.351572+0000 mgr.y (mgr.14427) 150 : cluster [DBG] 2023-06-10T21:01:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:49 smithi152 conmon[97365]: pgmap v125: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 699 op/s 2023-06-10T21:01:50.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 114791 114775 84.3521 54.8125 0.0233087 0.0118499 2023-06-10T21:01:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:50 smithi064 conmon[97224]: audit 2023-06-10T21:01:49.901466+0000 mon.c (mon. 2023-06-10T21:01:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:50 smithi064 conmon[97224]: 2) 36 : audit [DBG] from='client.? 172.21.15.64:0/3119923772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:50 smithi064 conmon[107122]: audit 2023-06-10T21:01: 2023-06-10T21:01:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:50 smithi064 conmon[107122]: 49.901466+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.64:0/3119923772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:50 smithi152 conmon[97365]: audit 2023-06-10T21:01:49.901466+0000 mon.c (mon. 2023-06-10T21:01:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:50 smithi152 conmon[97365]: 2) 36 : audit [DBG] from='client.? 172.21.15.64:0/3119923772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:51.049 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 115686 115670 84.0218 55.9375 0.0110594 0.011896 2023-06-10T21:01:51.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:51 smithi064 conmon[97224]: cluster 2023-06-10T21:01:50.352020+0000 mgr.y 2023-06-10T21:01:51.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:51 smithi064 conmon[97224]: (mgr.14427) 151 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-06-10T21:01:51.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:51 smithi064 conmon[107122]: cluster 2023-06-10T21:01:50.352020+0000 mgr.y (mgr.14427) 151 : cluster [DBG] pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-06-10T21:01:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:51 smithi152 conmon[97365]: cluster 2023-06-10T21:01:50.352020+0000 mgr.y (mgr.14427) 151 : cluster [DBG] 2023-06-10T21:01:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:51 smithi152 conmon[97365]: pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-06-10T21:01:52.048 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 116625 116609 83.7307 58.6875 0.0252796 0.0119383 2023-06-10T21:01:53.052 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 117542 117526 83.4272 57.3125 0.0237955 0.0119815 2023-06-10T21:01:53.205 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:01:52 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:01:53.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:53 smithi152 conmon[97365]: cluster 2023-06-10T21:01:52.354669+0000 mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:53.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:53 smithi152 conmon[97365]: audit 2023-06-10T21:01:52.481782+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.64:0/3766579193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:53.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[97224]: cluster 2023-06-10T21:01:52.354669+0000 2023-06-10T21:01:53.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[97224]: mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:53.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[97224]: audit 2023-06-10T21:01:52.481782+0000 mon.c (mon.2) 37 2023-06-10T21:01:53.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[97224]: : audit [DBG] from='client.? 172.21.15.64:0/3766579193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:53.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[107122]: cluster 2023-06-10T21:01:52.354669+0000 mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v127: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s 2023-06-10T21:01:53.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:53 smithi064 conmon[107122]: audit 2023-06-10T21:01:52.481782+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.64:0/3766579193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:54.056 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 118472 118456 83.1397 58.125 0.0219344 0.0120236 2023-06-10T21:01:55.056 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0078 2023-06-10T21:01:55.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 119324 2023-06-10T21:01:55.057 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:01:55.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:01:55.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 82.8567 2023-06-10T21:01:55.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 31.0613 2023-06-10T21:01:55.058 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 191.25 2023-06-10T21:01:55.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 48.4375 2023-06-10T21:01:55.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1325 2023-06-10T21:01:55.059 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 496.98 2023-06-10T21:01:55.060 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3060 2023-06-10T21:01:55.060 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 775 2023-06-10T21:01:55.060 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0120654 2023-06-10T21:01:55.060 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.00806299 2023-06-10T21:01:55.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.322156 2023-06-10T21:01:55.061 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00115805 2023-06-10T21:01:55.064 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:01:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:55 smithi152 conmon[97365]: cluster 2023-06-10T21:01:54.355062+0000 mgr.y ( 2023-06-10T21:01:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:55 smithi152 conmon[97365]: mgr.14427) 153 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-06-10T21:01:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:55 smithi152 conmon[97365]: audit 2023-06-10T21:01:55.064597+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.64:0/4004636277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:55.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[107122]: cluster 2023-06-10T21:01:54. 2023-06-10T21:01:55.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[107122]: 355062+0000 mgr.y (mgr.14427) 153 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-06-10T21:01:55.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:01:55.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[107122]: -10T21:01:55.064597+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.64:0/4004636277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:55.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[107122]: 2023-06-10T21:01:55.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[97224]: cluster 2023-06-10T21:01:54.355062+0000 mgr.y (mgr 2023-06-10T21:01:55.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[97224]: .14427) 153 : cluster [DBG] pgmap v128: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 692 op/s 2023-06-10T21:01:55.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:55 smithi064 conmon[97224]: audit 2023-06-10T21:01:55.064597+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.64:0/4004636277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:57.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:57 smithi064 conmon[97224]: cluster 2023-06-10T21:01:56.355776+0000 mgr.y ( 2023-06-10T21:01:57.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:57 smithi064 conmon[97224]: mgr.14427) 154 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.07k op/s 2023-06-10T21:01:57.412 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:01:57 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:01:57] "GET /metrics HTTP/1.1" 200 191101 "" "Prometheus/2.33.4" 2023-06-10T21:01:57.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:57 smithi064 conmon[107122]: cluster 2023-06-10T21:01:56.355776+0000 2023-06-10T21:01:57.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:57 smithi064 conmon[107122]: mgr.y (mgr.14427) 154 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.07k op/s 2023-06-10T21:01:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:57 smithi152 conmon[97365]: cluster 2023-06-10T21:01:56.355776+0000 mgr.y ( 2023-06-10T21:01:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:57 smithi152 conmon[97365]: mgr.14427) 154 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.07k op/s 2023-06-10T21:01:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:58 smithi064 conmon[107122]: audit 2023-06-10T21:01:57. 2023-06-10T21:01:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:58 smithi064 conmon[107122]: 655558+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.64:0/2129216706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:58 smithi064 conmon[97224]: audit 2023-06-10T21:01:57.655558+0000 mon.c (mon. 2023-06-10T21:01:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:58 smithi064 conmon[97224]: 2) 38 : audit [DBG] from='client.? 172.21.15.64:0/2129216706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:58.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:58 smithi152 conmon[97365]: audit 2023-06-10T21:01:57.655558+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.64:0/2129216706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:01:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:59 smithi064 conmon[107122]: cluster 2023-06-10T21:01:58. 2023-06-10T21:01:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:01:59 smithi064 conmon[107122]: 356149+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 726 op/s 2023-06-10T21:01:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:01:59 smithi064 conmon[97224]: cluster 2023-06-10T21:01:58.356149+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 726 op/s 2023-06-10T21:01:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:01:59 smithi152 conmon[97365]: cluster 2023-06-10T21:01:58.356149+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v130: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 726 op/s 2023-06-10T21:02:00.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:00 smithi064 conmon[97224]: audit 2023-06-10T21:02:00.261960+0000 2023-06-10T21:02:00.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:00 smithi064 conmon[97224]: mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.64:0/3072485359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:00.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:00 smithi064 conmon[107122]: audit 2023-06-10T21:02:00.261960+0000 mon.a (mon.0 2023-06-10T21:02:00.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:00 smithi064 conmon[107122]: ) 756 : audit [DBG] from='client.? 172.21.15.64:0/3072485359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:00 smithi152 conmon[97365]: audit 2023-06-10T21:02:00.261960+0000 mon.a 2023-06-10T21:02:00.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:00 smithi152 conmon[97365]: (mon.0) 756 : audit [DBG] from='client.? 172.21.15.64:0/3072485359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:01.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:01 smithi064 conmon[97224]: cluster 2023-06-10T21:02:00.356710+0000 mgr.y (mgr.14427) 156 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 917 op/s 2023-06-10T21:02:01.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:01 smithi064 conmon[107122]: cluster 2023-06-10T21:02:00.356710+0000 mgr.y (mgr.14427) 156 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 917 op/s 2023-06-10T21:02:01.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:01 smithi152 conmon[97365]: cluster 2023-06-10T21:02:00.356710+0000 mgr.y (mgr. 2023-06-10T21:02:01.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:01 smithi152 conmon[97365]: 14427) 156 : cluster [DBG] pgmap v131: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 917 op/s 2023-06-10T21:02:03.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:02 smithi152 conmon[99677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:02:03.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:03 smithi152 conmon[97365]: cluster 2023-06-10T21:02:02.357302+0000 mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 903 op/s 2023-06-10T21:02:03.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:03 smithi152 conmon[97365]: audit 2023-06-10T21:02:02.852592+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.64:0/1322703025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[97224]: cluster 2023-06-10T21:02:02.357302+0000 2023-06-10T21:02:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[97224]: mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 903 op/s 2023-06-10T21:02:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[97224]: audit 2023-06-10T21:02:02.852592+0000 mon.a (mon.0) 757 : audit [DBG] 2023-06-10T21:02:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/1322703025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:03.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[107122]: cluster 2023-06-10T21:02:02.357302 2023-06-10T21:02:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[107122]: +0000 mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 903 op/s 2023-06-10T21:02:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[107122]: audit 2023-06-10T21:02:02.852592+0000 mon.a (mon.0) 757 : 2023-06-10T21:02:03.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:03 smithi064 conmon[107122]: audit [DBG] from='client.? 172.21.15.64:0/1322703025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:05.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: cluster 2023-06-10T21:02:04.357716+0000 mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 770 op/s 2023-06-10T21:02:05.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:04.568479+0000 mgr.y (mgr.14427) 159 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-10T21:02:05.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: audit 2023-06-10T21:02:04.586060+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: audit 2023-06-10T21:02:04.587588+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: audit 2023-06-10T21:02:04.588411+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: audit 2023-06-10T21:02:04.589430+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:04 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: .590467+0000 mgr.y (mgr.14427) 160 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:02:05.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:05 smithi152 conmon[97365]: audit 2023-06-10T21:02:05.420729+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.64:0/4110944111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: cluster 2023-06-10T21:02:04 2023-06-10T21:02:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: .357716+0000 mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 770 op/s 2023-06-10T21:02:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:04.568479+0000 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: mgr.y (mgr.14427) 159 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: T21:02:04.586060+0000 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: mon.a (mon.0) 758 : 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:02:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: -10T21:02:04.587588+0000 mon.a 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit 2023-06-10T21:02 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: :04.588411+0000 mon.a (mon.0) 760 : 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit 2023-06-10T21:02:05.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: 2023-06-10T21:02:04.589430+0000 mon.a (mon.0) 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: 761 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: cephadm 2023-06-10T21 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: :02:04.590467+0000 mgr.y ( 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: mgr.14427) 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: 160 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:02:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:05.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: 02:05.420729+0000 mon.c (mon.2) 2023-06-10T21:02:05.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[107122]: 39 : audit [DBG] from='client.? 172.21.15.64:0/4110944111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:05.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: cluster 2023-06-10T21:02:04.357716+0000 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 770 op/s 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:04.568479+0000 mgr.y (mgr.14427) 159 : cephadm 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: [INF] Upgrade: Updating mgr.x 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: 02:04.586060+0000 mon.a (mon.0) 758 : audit 2023-06-10T21:02:05.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:05.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: audit 2023-06-10T21:02:05.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: 2023-06-10T21:02:04.587588+0000 2023-06-10T21:02:05.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: mon.a (mon.0) 759 : audit [INF] 2023-06-10T21:02:05.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:02:05.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:05.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: -06-10T21:02:04.588411+0000 mon.a (mon. 2023-06-10T21:02:05.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: 0) 760 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:02:05.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: audit 2023-06-10T21 2023-06-10T21:02:05.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: :02:04.589430+0000 mon.a (mon.0 2023-06-10T21:02:05.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: ) 761 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:05.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: cephadm 2023-06-10T21 2023-06-10T21:02:05.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: :02:04.590467+0000 mgr.y (mgr. 2023-06-10T21:02:05.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: 14427) 160 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:02:05.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:02:05.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: T21:02:05.420729+0000 mon.c ( 2023-06-10T21:02:05.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:05 smithi064 conmon[97224]: mon.2) 39 : audit [DBG] from='client.? 172.21.15.64:0/4110944111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:06.850 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:06 smithi152 systemd[1]: Stopping Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:07.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:06 smithi152 bash[134583]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-mgr.x found: no such container 2023-06-10T21:02:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:07 smithi152 conmon[97365]: cluster 2023-06-10T21:02:06.358311+0000 mgr.y 2023-06-10T21:02:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:07 smithi152 conmon[97365]: (mgr.14427) 161 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.14k op/s 2023-06-10T21:02:07.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:07 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:07] "GET /metrics HTTP/1.1" 200 191145 "" "Prometheus/2.33.4" 2023-06-10T21:02:07.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:07 smithi064 conmon[97224]: cluster 2023-06-10T21:02:06.358311+0000 2023-06-10T21:02:07.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:07 smithi064 conmon[97224]: mgr.y (mgr.14427) 161 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.14k op/s 2023-06-10T21:02:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:07 smithi064 conmon[107122]: cluster 2023-06-10T21:02:06. 2023-06-10T21:02:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:07 smithi064 conmon[107122]: 358311+0000 mgr.y (mgr.14427) 161 : cluster [DBG] pgmap v134: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.14k op/s 2023-06-10T21:02:07.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:07 smithi152 bash[134689]: 9977c174f9de461041ad64dd7e78064f725235659cf465e801bdb717916c0917 2023-06-10T21:02:08.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:08 smithi064 conmon[107122]: audit 2023-06-10T21:02:08.011289 2023-06-10T21:02:08.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:08 smithi064 conmon[107122]: +0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.64:0/3794180539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:08 smithi064 conmon[97224]: audit 2023-06-10T21:02:08.011289+0000 mon.b (mon. 2023-06-10T21:02:08.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:08 smithi064 conmon[97224]: 1) 28 : audit [DBG] from='client.? 172.21.15.64:0/3794180539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:08.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:08 smithi152 conmon[97365]: audit 2023-06-10T21:02:08.011289+0000 2023-06-10T21:02:08.415 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:08 smithi152 conmon[97365]: mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.64:0/3794180539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:08.415 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:08 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:02:08.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:08 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2023-06-10T21:02:08.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:08 smithi152 systemd[1]: Stopped Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:08.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:08 smithi152 systemd[1]: Starting Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:09 smithi064 conmon[97224]: cluster 2023-06-10T21:02:08.358726+0000 mgr.y (mgr 2023-06-10T21:02:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:09 smithi064 conmon[97224]: .14427) 162 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-06-10T21:02:09.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:09 smithi064 conmon[107122]: cluster 2023-06-10T21:02:08.358726+0000 mgr.y (mgr.14427) 162 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-06-10T21:02:09.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:09 smithi152 conmon[97365]: cluster 2023-06-10T21:02:08.358726+0000 mgr.y (mgr.14427 2023-06-10T21:02:09.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:09 smithi152 conmon[97365]: ) 162 : cluster [DBG] pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2023-06-10T21:02:09.920 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:09 smithi152 bash[135022]: b20b3f89b2be3e06d3d44bb6701a50abab2868e77b4e449c37fe1d8535f68f39 2023-06-10T21:02:09.920 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:09 smithi152 systemd[1]: Started Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:09.920 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:09 smithi152 conmon[135060]: debug 2023-06-10T21:02:09.910+0000 7f35d1cdf200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:02:10.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[135060]: debug 2023-06-10T21:02:10.059+0000 7f35d1cdf200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:02:10.657 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[135060]: debug 2023-06-10T21:02:10.208+0000 7f35d1cdf200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:02:10.658 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[135060]: debug 2023-06-10T21:02:10.379+0000 7f35d1cdf200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:02:10.658 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[135060]: debug 2023-06-10T21:02:10.653+0000 7f35d1cdf200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:02:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[97365]: audit 2023-06-10T21:02:09.741708+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:10.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[97365]: audit 2023-06-10T21:02:09.743053+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:10.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:10 smithi152 conmon[97365]: audit 2023-06-10T21:02:10.570266+0000 mon.a (mon.0) 764 : audit [DBG] from='client.? 172.21.15.64:0/739425544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[97224]: audit 2023-06-10T21:02:09.741708+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[97224]: audit 2023-06-10T21:02:09.743053+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[97224]: audit 2023-06-10T21:02:10.570266+0000 mon.a (mon.0) 764 : audit [DBG] from='client.? 172.21.15.64:0/739425544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[107122]: audit 2023-06-10T21:02:09.741708+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[107122]: audit 2023-06-10T21:02:09.743053+0000 mon.a (mon.0) 763 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:10 smithi064 conmon[107122]: audit 2023-06-10T21:02:10.570266+0000 mon.a (mon.0) 764 : audit [DBG] from='client.? 172.21.15.64:0/739425544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:11.328 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:11 smithi152 conmon[135060]: debug 2023-06-10T21:02:11.180+0000 7f35d1cdf200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:02:11.328 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:11 smithi152 conmon[135060]: debug 2023-06-10T21:02:11.324+0000 7f35d1cdf200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:02:11.684 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:11 smithi152 conmon[135060]: debug 2023-06-10T21:02:11.524+0000 7f35d1cdf200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:02:11.684 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:11 smithi152 conmon[135060]: debug 2023-06-10T21:02:11.679+0000 7f35d1cdf200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:02:11.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:11 smithi152 conmon[97365]: cluster 2023-06-10T21:02:10.359246+0000 mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-06-10T21:02:12.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:11 smithi064 conmon[97224]: cluster 2023-06-10T21:02:10.359246+0000 2023-06-10T21:02:12.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:11 smithi064 conmon[97224]: mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-06-10T21:02:12.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:11 smithi064 conmon[107122]: cluster 2023-06-10T21:02: 2023-06-10T21:02:12.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:11 smithi064 conmon[107122]: 10.359246+0000 mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 825 op/s 2023-06-10T21:02:12.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:12 smithi152 conmon[135060]: debug 2023-06-10T21:02:12.397+0000 7f35d1cdf200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:02:13.625 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[135060]: debug 2023-06-10T21:02:13.342+0000 7f35d1cdf200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:02:13.626 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[135060]: debug 2023-06-10T21:02:13.617+0000 7f35d1cdf200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:02:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[97365]: cluster 2023-06-10T21:02:12.359856+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-06-10T21:02:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[97365]: audit 2023-06-10T21:02:13.138029+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.64:0/2782409062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:02:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:13 smithi152 conmon[97365]: T21:02:13.224518+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:14.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[97224]: cluster 2023-06-10T21:02:12.359856+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-06-10T21:02:14.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[97224]: audit 2023-06-10T21:02:13.138029+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.64:0/2782409062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:14.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[97224]: audit 2023-06-10T21:02:13.224518+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:14.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: cluster 2023-06-10T21:02 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: :12.359856+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v137: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: audit 2023-06-10T21:02 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: :13.138029+0000 mon.a (mon.0) 765 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: : audit [DBG] from='client.? 172.21.15.64:0/2782409062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: audit 2023 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: -06-10T21:02:13.224518+0000 mon.a (mon 2023-06-10T21:02:14.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:13 smithi064 conmon[107122]: .0) 766 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:15.201 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:14 smithi152 conmon[135060]: debug 2023-06-10T21:02:14.883+0000 7f35d1cdf200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:02:15.201 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[135060]: debug 2023-06-10T21:02:15.060+0000 7f35d1cdf200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:02:15.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[135060]: debug 2023-06-10T21:02:15.371+0000 7f35d1cdf200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:02:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[107122]: cluster 2023-06-10T21:02:14. 2023-06-10T21:02:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[107122]: 360256+0000 mgr.y (mgr.14427) 165 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-10T21:02:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[107122]: audit 2023-06-10T21:02:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[107122]: 2023-06-10T21:02:15.676848+0000 mon.c (mon.2) 40 : audit 2023-06-10T21:02:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[107122]: [DBG] from='client.? 172.21.15.64:0/259216917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[97224]: cluster 2023-06-10T21:02:14.360256+0000 mgr.y 2023-06-10T21:02:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[97224]: (mgr.14427) 165 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-10T21:02:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[97224]: audit 2023-06-10T21:02:15.676848+0000 2023-06-10T21:02:16.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:15 smithi064 conmon[97224]: mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.64:0/259216917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:16.199 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[135060]: debug 2023-06-10T21:02:15.863+0000 7f35d1cdf200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:02:16.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[97365]: cluster 2023-06-10T21:02:14.360256+0000 mgr.y (mgr 2023-06-10T21:02:16.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[97365]: .14427) 165 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-10T21:02:16.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:15 smithi152 conmon[97365]: audit 2023-06-10T21:02:15.676848+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.64:0/259216917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:16.699 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[135060]: debug 2023-06-10T21:02:16.234+0000 7f35d1cdf200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:02:16.700 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[135060]: debug 2023-06-10T21:02:16.481+0000 7f35d1cdf200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:02:17.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023-06-10T21:02:15.866424+0000 2023-06-10T21:02:17.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023-06-10T21:02:16.510755+0000 mon.a (mon.0) 2023-06-10T21:02:17.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 768 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:17.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023-06-10T21:02:17.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 2023-06-10T21:02:16.511522+0000 mon.a (mon.0) 2023-06-10T21:02:17.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 769 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:02:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 06-10T21:02:16.527978+0000 mon.a ( 2023-06-10T21:02:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023-06-10T21:02:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 2023-06-10T21:02:16.539278+0000 2023-06-10T21:02:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: mon.a (mon.0) 771 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 2023-06-10T21:02:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: audit 2023-06-10T21:02:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 2023-06-10T21:02:16.546628+0000 mon.a ( 2023-06-10T21:02:17.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: mon.0) 772 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-10T21:02:17.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: cluster 2023 2023-06-10T21:02:17.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: -06-10T21:02:16.554820+0000 2023-06-10T21:02:17.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: mon.a (mon.0) 773 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-10T21:02:17.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[107122]: 2023-06-10T21:02:17.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06-10T21:02:15.866424+0000 2023-06-10T21:02:17.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06-10T21:02:16.510755+0000 mon.a (mon.0) 768 : audit [DBG] 2023-06-10T21:02:17.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:17.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: 02:16.511522+0000 mon.a (mon.0) 769 : audit 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: 2023-06-10T21:02:16.527978+0000 mon.a 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: (mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:17.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: 10T21:02:16.539278+0000 mon.a (mon.0 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: ) 771 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: audit 2023-06-10T21 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: :02:16.546628+0000 mon.a (mon. 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: 0) 772 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: T21:02:16.554820+0000 mon.a ( 2023-06-10T21:02:17.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:16 smithi064 conmon[97224]: mon.0) 773 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-10T21:02:17.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:15.866424+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:16.510755+0000 mon.a (mon.0) 768 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:17.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:16.511522+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:17.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:16.527978+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' 2023-06-10T21:02:17.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:16.539278+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:17.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: audit 2023-06-10T21:02:16.546628+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-10T21:02:17.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[97365]: cluster 2023-06-10T21:02:16.554820+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-10T21:02:17.197 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:16 smithi152 conmon[135060]: debug 2023-06-10T21:02:16.921+0000 7f35d1cdf200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:02:17.593 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:17] "GET /metrics HTTP/1.1" 200 191142 "" "Prometheus/2.33.4" 2023-06-10T21:02:17.901 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97677]: ignoring --setuser ceph since I am not root 2023-06-10T21:02:17.901 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97677]: ignoring --setgroup ceph since I am not root 2023-06-10T21:02:17.901 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97677]: debug 2023-06-10T21:02:17.815+0000 7fe60e695680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:02:17.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: cluster 2023-06-10T21:02:16.360930+0000 2023-06-10T21:02:17.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: mgr.y (mgr.14427) 166 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-06-10T21:02:17.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:17.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: 2023-06-10T21 2023-06-10T21:02:17.902 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: :02:16.543140+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: 06-10T21:02:16.546248+0000 mgr.y (mgr.14427) 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: 168 : cephadm [INF] Failing over to other MGR 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: -06-10T21:02:17.540233+0000 mon.a ( 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: mon.0) 774 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-10T21:02:17.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: cluster 2023 2023-06-10T21:02:17.904 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: -06-10T21:02:17.540314+0000 mon.a 2023-06-10T21:02:17.904 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97224]: (mon.0) 775 : cluster [DBG] mgrmap e20: x(active, starting, since 0.992974s) 2023-06-10T21:02:18.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[97677]: debug 2023-06-10T21:02:17.966+0000 7fe60e695680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:02:18.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[107122]: cluster 2023-06-10T21:02:16.360930+0000 mgr.y (mgr.14427) 166 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-06-10T21:02:18.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:16.543140+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:02:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:16.546248+0000 mgr.y (mgr.14427) 168 : cephadm [INF] Failing over to other MGR 2023-06-10T21:02:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[107122]: audit 2023-06-10T21:02:17.540233+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-10T21:02:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:17 smithi064 conmon[107122]: cluster 2023-06-10T21:02:17.540314+0000 mon.a (mon.0) 775 : cluster [DBG] mgrmap e20: x(active, starting, since 0.992974s) 2023-06-10T21:02:18.163 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[135060]: debug 2023-06-10T21:02:17.973+0000 7f35d1cdf200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:02:18.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: cluster 2023-06-10T21:02:16.360930+0000 mgr.y 2023-06-10T21:02:18.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: (mgr.14427) 166 : cluster [DBG] pgmap v139: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-06-10T21:02:18.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:16.543140+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-10T21:02:18.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:16.546248+0000 mgr.y (mgr.14427) 168 : cephadm [INF] Failing over to other MGR 2023-06-10T21:02:18.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: audit 2023-06-10T21:02:17.540233+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14427 172.21.15.64:0/3914044905' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-10T21:02:18.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:17 smithi152 conmon[97365]: cluster 2023-06-10T21:02:17.540314+0000 mon.a (mon.0) 775 : cluster [DBG] mgrmap e20: x(active, starting, since 0.992974s) 2023-06-10T21:02:18.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[135060]: debug 2023-06-10T21:02:18.159+0000 7f35d1cdf200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:02:18.908 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[135060]: debug 2023-06-10T21:02:18.460+0000 7f35d1cdf200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:02:18.908 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[135060]: debug 2023-06-10T21:02:18.637+0000 7f35d1cdf200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:02:18.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:18 smithi064 conmon[97224]: audit 2023-06-10T21:02:18.269092+0000 mon.a (mon. 2023-06-10T21:02:18.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:18 smithi064 conmon[97224]: 0) 776 : audit [DBG] from='client.? 172.21.15.64:0/881422229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:18.912 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:18 smithi064 conmon[97677]: debug 2023-06-10T21:02:18.612+0000 7fe60e695680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:02:18.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:18 smithi064 conmon[107122]: audit 2023-06-10T21:02: 2023-06-10T21:02:18.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:18 smithi064 conmon[107122]: 18.269092+0000 mon.a (mon.0) 776 : audit [DBG] from='client.? 172.21.15.64:0/881422229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:19.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[135060]: debug 2023-06-10T21:02:18.993+0000 7f35d1cdf200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:02:19.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[97365]: audit 2023-06-10T21:02:18.269092+0000 mon.a (mon.0) 776 : audit 2023-06-10T21:02:19.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:18 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/881422229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:20.411 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:19 smithi064 conmon[97677]: debug 2023-06-10T21:02:19.907+0000 7fe60e695680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:02:20.411 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97677]: debug 2023-06-10T21:02:20.050+0000 7fe60e695680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:02:20.646 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.316+0000 7f35d1cdf200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:02:20.646 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STARTING 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: CherryPy Checker: 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: The Application mounted at '' has an empty config. 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Serving on http://:::9283 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STARTED 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STOPPING 2023-06-10T21:02:20.647 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:02:20.648 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STOPPED 2023-06-10T21:02:20.648 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STARTING 2023-06-10T21:02:20.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: cluster 2023-06-10T21:02:20.335531+0000 mon.a (mon 2023-06-10T21:02:20.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: .0) 777 : cluster [INF] Active manager daemon x restarted 2023-06-10T21:02:20.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: cluster 2023-06-10T21:02:20.337322+0000 mon.a (mon.0) 778 : cluster [INF] Activating manager daemon x 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 21:02:20.344733+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: audit 2023-06-10T21:02:20. 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 346018+0000 mon.c (mon.2) 41 : audit 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.649 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 2023-06-10T21:02:20.354474+0000 mon.c ( 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: mon.2) 42 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 21:02:20.356932+0000 mon.c (mon. 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 2) 43 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: 02:20.357593+0000 mon.c (mon.2) 44 2023-06-10T21:02:20.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[97365]: : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:20.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: cluster 2023-06-10T21:02:20.335531+0000 mon.a (mon.0) 777 : cluster [INF] Active manager daemon x restarted 2023-06-10T21:02:20.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: cluster 2023-06-10T21:02:20.337322+0000 mon.a (mon.0) 778 : cluster [INF] Activating manager daemon x 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: cluster 2023-06-10T21:02:20.344733+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.346018+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.354474+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.356932+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.357593+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:20.733 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[97677]: debug 2023-06-10T21:02:20.409+0000 7fe60e695680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: cluster 2023-06-10T21:02:20.335531+0000 mon.a (mon.0) 777 : cluster [INF] Active manager daemon x restarted 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: cluster 2023-06-10T21:02:20.337322+0000 mon.a (mon.0) 778 : cluster [INF] Activating manager daemon x 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: cluster 2023-06-10T21:02:20.344733+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.346018+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.354474+0000 mon.c (mon.2) 42 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:20.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.356932+0000 mon.c (mon.2) 43 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:02:20.735 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:20 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.357593+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.? 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:20.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Serving on http://:::9283 2023-06-10T21:02:20.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: [10/Jun/2023:21:02:20] ENGINE Bus STARTED 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.820+0000 7f352ebfa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.820+0000 7f352ebfa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.820+0000 7f352ebfa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.820+0000 7f352ebfa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.820+0000 7f352ebfa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.838+0000 7f3531c00700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.838+0000 7f3531c00700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.948 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.838+0000 7f3531c00700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.948 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.838+0000 7f3531c00700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:20.948 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:20 smithi152 conmon[135060]: debug 2023-06-10T21:02:20.838+0000 7f3531c00700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:02:21.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: cluster 2023-06-10T21:02:20.393601+0000 mon.a (mon.0) 780 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0566425s) 2023-06-10T21:02:21.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.401950+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:21.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.402172+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:21.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.402282+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:21.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.402824+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.403120+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.404191+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.404411+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.404683+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.404930+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.405235+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:02:21.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.405442+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.405648+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.405841+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.406049+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 02:20.412328+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.412475+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.412589+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:02:21.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: cluster 2023-06-10T21:02:20.542650+0000 mon.a (mon.0) 781 : cluster [INF] Manager daemon x is now available 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.568403+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.598381+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.614161+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:20.622101+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: :02:20.663787+0000 mon.b (mon.1) 49 : audit [INF] 2023-06-10T21:02:21.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 06-10T21:02:20.667957+0000 mon.a (mon.0) 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 783 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: :20.679861+0000 mon.b (mon.1) 50 : audit 2023-06-10T21:02:21.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 2023-06-10T21:02:20.683271+0000 mon.a 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: (mon.0) 784 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: T21:02:20.707181+0000 mon.a (mon. 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 0) 785 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 06-10T21:02:20.712529+0000 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: mon.b (mon.1) 51 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: -06-10T21:02:20.716844+0000 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: mon.a (mon.0) 786 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 06-10T21:02:20.870714+0000 mon.c (mon.2) 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 45 : audit [DBG] from='client.? 172.21.15.64:0/454796269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: :21.180854+0000 mon.a (mon.0) 787 : 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: audit 2023-06-10T21:02:21. 2023-06-10T21:02:21.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: 203434+0000 mon.a (mon.0) 788 : audit [INF] 2023-06-10T21:02:21.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:21 smithi152 conmon[97365]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: cluster 2023-06-10T21:02:20.393601+0000 2023-06-10T21:02:21.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.a (mon.0) 780 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0566425s) 2023-06-10T21:02:21.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.401950+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:21.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.402172+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:21.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.402282+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:21.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.402824+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.403120+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.404191+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.404411+0000 mon.b ( 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.1) 35 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:02:21.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: T21:02:20.404683+0000 mon.b (mon. 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 1) 36 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: -06-10T21:02:20.404930+0000 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.b (mon.1) 37 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: -10T21:02:20.405235+0000 mon.b ( 2023-06-10T21:02:21.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.1) 38 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 10T21:02:20.405442+0000 mon.b (mon 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: .1) 39 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.405648+0000 mon.b 2023-06-10T21:02:21.920 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: (mon.1) 40 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.405841+0000 mon.b 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: (mon.1) 41 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: -06-10T21:02:20.406049+0000 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.b (mon.1) 42 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:02:21.921 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:20.412328+0000 mon.b (mon.1) 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 43 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 02:20.412475+0000 mon.b (mon.1) 44 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:02:21.922 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: :20.412589+0000 mon.b (mon.1) 45 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: cluster 2023- 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.542650+0000 mon.a 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: (mon.0) 781 : cluster [INF] Manager daemon x is now available 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:21.923 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:20.568403+0000 mon.a (mon 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: .0) 782 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 10T21:02:20.598381+0000 mon.b (mon 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: .1) 46 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:21.924 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: -10T21:02:20.614161+0000 mon.b ( 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.1) 47 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.622101+0000 mon.b 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: (mon.1) 48 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20 2023-06-10T21:02:21.925 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: .663787+0000 mon.b (mon.1) 49 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.667957+0000 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.a (mon.0) 783 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02: 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 20.679861+0000 mon.b (mon.1) 50 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:20.683271+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.926 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.707181 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: +0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 06-10T21:02:20.712529+0000 mon.b (mon.1) 51 : audit [INF] 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:20.716844 2023-06-10T21:02:21.927 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: +0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: T21:02:20.870714+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.64:0/454796269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:21.180854+0000 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: mon.a (mon.0) 787 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: audit 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: 2023-06-10T21:02:21.203434+0000 mon.a (mon.0) 788 : audit [INF] 2023-06-10T21:02:21.928 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: cluster 2023-06-10T21:02:20.393601+0000 mon.a (mon.0) 780 : cluster [DBG] mgrmap e21: x(active, starting, since 0.0566425s) 2023-06-10T21:02:21.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.401950+0000 mon.b (mon.1) 29 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:21.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.402172+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:21.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.402282+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:21.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.402824+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.403120+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.404191+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.404411+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.404683+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.404930+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.405235+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:02:21.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.405442+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.405648+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.405841+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.406049+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.412328+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.412475+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.412589+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:02:21.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: cluster 2023-06-10T21:02:20.542650+0000 mon.a (mon.0) 781 : cluster [INF] Manager daemon x is now available 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.568403+0000 mon.a (mon.0) 782 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.598381+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.614161+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.622101+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.663787+0000 mon.b (mon.1) 49 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.667957+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.679861+0000 mon.b (mon.1) 50 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.683271+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.707181+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.712529+0000 mon.b (mon.1) 51 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.716844+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:20.870714+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.64:0/454796269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:21.180854+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:21.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:21 smithi064 conmon[107122]: audit 2023-06-10T21:02:21.203434+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:22.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97677]: debug 2023-06-10T21:02:22.029+0000 7fe60e695680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:02:22.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97677]: debug 2023-06-10T21:02:22.258+0000 7fe60e695680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:02:22.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97677]: debug 2023-06-10T21:02:22.401+0000 7fe60e695680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:02:22.665 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97677]: debug 2023-06-10T21:02:22.556+0000 7fe60e695680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:02:22.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:21.267416+0000 mgr.x (mgr. 2023-06-10T21:02:22.670 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 24857) 1 : cephadm [INF] Deploying cephadm binary to smithi064 2023-06-10T21:02:22.670 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:21.273016+0000 mgr.x (mgr.24857) 2 : cephadm [INF] Deploying cephadm binary to smithi152 2023-06-10T21:02:22.670 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T21:02:22.670 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: T21:02:21.438718+0000 mon.a (mon.0 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: ) 789 : cluster [DBG] mgrmap e22: x(active, since 1.10176s) 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: cluster 2023-06- 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 10T21:02:21.474358+0000 mgr.x (mgr 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: .24857) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 10T21:02:21.578901+0000 mon.a ( 2023-06-10T21:02:22.671 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: mon.0) 790 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: cluster 2023- 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 06-10T21:02:21.578995 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: +0000 mon.a (mon.0) 791 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 06-10T21:02:21.592230+0000 2023-06-10T21:02:22.672 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: mon.b (mon.1) 52 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:22.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: 06-10T21:02:21.592864+0000 2023-06-10T21:02:22.673 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97224]: mon.a (mon.0) 792 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:21.267416+0000 2023-06-10T21:02:22.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: mgr.x (mgr.24857) 1 : cephadm [INF] Deploying cephadm binary to smithi064 2023-06-10T21:02:22.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:21.273016 2023-06-10T21:02:22.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 2 : cephadm [INF] Deploying cephadm binary to smithi152 2023-06-10T21:02:22.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: cluster 2023-06-10T21: 2023-06-10T21:02:22.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 02:21.438718+0000 mon.a (mon.0) 789 2023-06-10T21:02:22.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: : cluster [DBG] mgrmap e22: x(active, since 1.10176s) 2023-06-10T21:02:22.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: cluster 2023 2023-06-10T21:02:22.675 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: -06-10T21:02:21.474358+0000 2023-06-10T21:02:22.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: mgr.x (mgr.24857) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:22.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 2023-06-10T21:02:22.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: audit 2023 2023-06-10T21:02:22.676 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: -06-10T21:02:21.578901+0000 mon.a (mon. 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 0) 790 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: T21:02:21.578995+0000 mon.a (mon. 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 0) 791 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 21:02:21.592230+0000 mon.b (mon.1) 2023-06-10T21:02:22.677 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: 52 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: audit 2023-06-10T21:02 2023-06-10T21:02:22.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: :21.592864+0000 mon.a (mon.0) 792 : 2023-06-10T21:02:22.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[107122]: audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.918 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:21.267416 2023-06-10T21:02:22.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: +0000 mgr.x (mgr.24857) 1 : cephadm [INF] Deploying cephadm binary to smithi064 2023-06-10T21:02:22.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:21.273016+0000 mgr.x (mgr.24857) 2 : cephadm [INF] Deploying cephadm binary to smithi152 2023-06-10T21:02:22.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: cluster 2023-06-10T21:02:21.438718+0000 mon.a (mon.0) 789 : cluster [DBG] mgrmap e22: x(active, since 1.10176s) 2023-06-10T21:02:22.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: cluster 2023-06-10T21:02:21.474358+0000 mgr.x (mgr.24857) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:22.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: audit 2023-06-10T21:02:21.578901+0000 mon.a (mon 2023-06-10T21:02:22.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: .0) 790 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:22.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: cluster 2023 2023-06-10T21:02:22.920 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: -06-10T21:02:21 2023-06-10T21:02:22.921 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: .578995+0000 mon.a (mon.0) 791 2023-06-10T21:02:22.922 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-10T21:02:22.922 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: audit 2023 2023-06-10T21:02:22.922 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: -06-10T21:02:21. 2023-06-10T21:02:22.923 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: 592230+0000 mon.b (mon.1) 52 : 2023-06-10T21:02:22.923 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.923 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:02:22.923 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: -10T21:02:21.592864+0000 2023-06-10T21:02:22.924 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: mon.a (mon.0) 792 : audit [INF] 2023-06-10T21:02:22.924 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[97365]: from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:02:22.930 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:22 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:22] "GET /metrics HTTP/1.1" 200 33316 "" "Prometheus/2.33.4" 2023-06-10T21:02:23.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:22 smithi064 conmon[97677]: debug 2023-06-10T21:02:22.865+0000 7fe60e695680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:02:23.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:21.912764+0000 mgr.x (mgr.24857) 4 : cephadm [INF] [10/Jun/2023:21:02:21] ENGINE Bus STARTING 2023-06-10T21:02:23.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:22.096471+0000 mgr.x (mgr.24857) 5 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Serving on http://172.21.15.152:8765 2023-06-10T21:02:23.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:22.331781+0000 mgr.x (mgr.24857) 6 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Serving on https://172.21.15.152:7150 2023-06-10T21:02:23.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:22.331905+0000 mgr.x (mgr.24857) 7 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Bus STARTED 2023-06-10T21:02:23.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cluster 2023-06-10T21:02:22.413537+0000 mgr.x (mgr.24857) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:23.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: audit 2023-06-10T21:02:22.580020+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:23.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cluster 2023-06-10T21:02:22.580092+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-10T21:02:23.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: cluster 2023-06-10T21:02:22.601433+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-06-10T21:02:23.919 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97224]: audit 2023-06-10T21:02:23.529751+0000 mon.a (mon.0) 796 : audit [DBG] from='client.? 172.21.15.64:0/3970618756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:23.919 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97677]: debug 2023-06-10T21:02:23.440+0000 7fe60e695680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:02:23.919 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[97677]: debug 2023-06-10T21:02:23.652+0000 7fe60e695680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:02:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:21.912764+0000 2023-06-10T21:02:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: mgr.x (mgr.24857) 4 : cephadm [INF] [10/Jun/2023:21:02:21] ENGINE Bus STARTING 2023-06-10T21:02:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:22.096471+0000 mgr.x (mgr.24857) 5 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Serving on http://172.21.15.152:8765 2023-06-10T21:02:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:22.331781+0000 mgr.x (mgr.24857) 6 : cephadm [INF] 2023-06-10T21:02:23.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: [10/Jun/2023:21:02:22] ENGINE Serving on https://172.21.15.152:7150 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 2023-06-10T21:02:22.331905+0000 mgr.x (mgr. 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 24857) 7 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Bus STARTED 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 10T21:02:22.413537+0000 mgr.x (mgr 2023-06-10T21:02:23.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: .24857) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 21:02:22.580020+0000 mon.a (mon.0 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: ) 793 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 21:02:22.580092+0000 mon.a (mon.0 2023-06-10T21:02:23.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: ) 794 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 21:02:22.601433+0000 mon.a (mon.0 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: ) 795 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: T21:02:23.529751+0000 mon.a (mon. 2023-06-10T21:02:23.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:23 smithi064 conmon[107122]: 0) 796 : audit [DBG] from='client.? 172.21.15.64:0/3970618756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:23.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:21.912764+0000 mgr.x (mgr. 2023-06-10T21:02:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 24857) 4 : cephadm [INF] [10/Jun/2023:21:02:21] ENGINE Bus STARTING 2023-06-10T21:02:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:22.096471+0000 mgr.x (mgr.24857) 5 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Serving on http://172.21.15.152:8765 2023-06-10T21:02:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:22.331781+0000 mgr.x (mgr.24857) 6 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Serving on https://172.21.15.152:7150 2023-06-10T21:02:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cephadm 2023-06- 2023-06-10T21:02:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 10T21:02:22.331905+0000 mgr.x (mgr. 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 24857) 7 : cephadm [INF] [10/Jun/2023:21:02:22] ENGINE Bus STARTED 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cluster 2023-06 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: -10T21:02:22.413537+0000 mgr.x ( 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: mgr.24857) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: audit 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 2023-06-10T21:02:22.580020+0000 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: mon.a (mon.0) 793 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:02:23.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cluster 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 2023-06-10T21:02:22.580092+0000 mon.a (mon. 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 0) 794 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: cluster 2023-06 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: -10T21:02:22.601433+0000 mon.a ( 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: mon.0) 795 : cluster [DBG] mgrmap e23: x(active, since 2s) 2023-06-10T21:02:23.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: audit 2023-06-10T21:02:23.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 2023-06-10T21:02:23.529751+0000 2023-06-10T21:02:23.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: mon.a (mon.0) 796 : audit [DBG] from='client.? 172.21.15.64:0/3970618756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:23.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:23 smithi152 conmon[97365]: 2023-06-10T21:02:25.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97677]: debug 2023-06-10T21:02:25.005+0000 7fe60e695680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:02:25.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97677]: debug 2023-06-10T21:02:25.157+0000 7fe60e695680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:02:25.776 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97677]: debug 2023-06-10T21:02:25.426+0000 7fe60e695680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:02:25.777 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97677]: debug 2023-06-10T21:02:25.771+0000 7fe60e695680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:02:25.777 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[107122]: cluster 2023-06-10T21:02:24.413913+0000 mgr.x (mgr.24857) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:25.777 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[107122]: cluster 2023-06-10T21:02:24.601815+0000 mon.a (mon.0) 797 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-06-10T21:02:25.777 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[107122]: audit 2023-06-10T21:02:24.840002+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:25.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: cluster 2023-06-10T21:02:24.413913+0000 2023-06-10T21:02:25.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: mgr.x (mgr.24857) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:25.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: cluster 2023-06-10T21:02:24.601815+0000 mon.a (mon.0) 2023-06-10T21:02:25.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: 797 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-06-10T21:02:25.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: audit 2023-06-10T21:02:24.840002 2023-06-10T21:02:25.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:25 smithi064 conmon[97224]: +0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:25 smithi152 conmon[97365]: cluster 2023-06-10T21:02:24.413913+0000 mgr.x (mgr.24857) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:25 smithi152 conmon[97365]: cluster 2023-06-10T21:02:24.601815+0000 mon.a (mon.0) 797 : cluster [DBG] mgrmap e24: x(active, since 4s) 2023-06-10T21:02:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:25 smithi152 conmon[97365]: audit 2023-06-10T21:02:24.840002+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:26.478 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:26 smithi064 conmon[97677]: debug 2023-06-10T21:02:26.211+0000 7fe60e695680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:02:26.478 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:26 smithi064 conmon[97677]: debug 2023-06-10T21:02:26.472+0000 7fe60e695680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:02:26.784 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:26 smithi064 conmon[97677]: debug 2023-06-10T21:02:26.778+0000 7fe60e695680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:02:27.119 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: debug 2023-06-10T21:02:27.113+0000 7fe60e695680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:02:27.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: debug 2023-06-10T21:02:27.273+0000 7fe60e695680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:02:27.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: [10/Jun/2023:21:02:27] ENGINE Bus STARTING 2023-06-10T21:02:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[107122]: cluster 2023-06-10T21:02:26.414246+0000 mgr.x (mgr.24857) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[107122]: audit 2023-06-10T21:02:26.419967+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.64:0/2570533216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97224]: cluster 2023-06-10T21:02:26.414246+0000 mgr.x (mgr.24857 2023-06-10T21:02:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97224]: ) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97224]: audit 2023-06-10T21:02:26.419967+0000 mon.c (mon.2) 46 : audit [DBG] 2023-06-10T21:02:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/2570533216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:27 smithi152 conmon[97365]: cluster 2023-06-10T21:02:26.414246+0000 mgr.x (mgr.24857) 10 : cluster 2023-06-10T21:02:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:27 smithi152 conmon[97365]: [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail 2023-06-10T21:02:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:27 smithi152 conmon[97365]: audit 2023-06-10T21:02:26.419967+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.64:0/2570533216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:27.911 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: CherryPy Checker: 2023-06-10T21:02:27.911 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: The Application mounted at '' has an empty config. 2023-06-10T21:02:27.911 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: 2023-06-10T21:02:27.912 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: [10/Jun/2023:21:02:27] ENGINE Serving on http://:::9283 2023-06-10T21:02:27.912 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:27 smithi064 conmon[97677]: [10/Jun/2023:21:02:27] ENGINE Bus STARTED 2023-06-10T21:02:28.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: audit 2023-06-10T21:02:27.292052+0000 mon.c (mon 2023-06-10T21:02:28.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: .2) 47 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:02:28.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: cluster 2023-06-10T21:02:27.293252+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2023-06-10T21:02:28.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: audit 2023-06-10T21:02:27.294579+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:28.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: audit 2023-06-10T21:02:27.297495+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:02:28.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:28 smithi152 conmon[97365]: audit 2023-06-10T21:02:27.303683+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:28.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[97224]: audit 2023-06-10T21:02:27.292052+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:02:28.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[97224]: cluster 2023-06-10T21:02:27.293252+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2023-06-10T21:02:28.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[97224]: audit 2023-06-10T21:02:27.294579+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:28.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[97224]: audit 2023-06-10T21:02:27.297495+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:02:28.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[97224]: audit 2023-06-10T21:02:27.303683+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:28.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[107122]: audit 2023-06-10T21:02:27.292052+0000 mon.c (mon.2) 47 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:02:28.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[107122]: cluster 2023-06-10T21:02:27.293252+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2023-06-10T21:02:28.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[107122]: audit 2023-06-10T21:02:27.294579+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:02:28.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[107122]: audit 2023-06-10T21:02:27.297495+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:02:28.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:28 smithi064 conmon[107122]: audit 2023-06-10T21:02:27.303683+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.? 172.21.15.64:0/1312458392' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:02:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: cluster 2023-06-10T21:02:28.153953+0000 mon.a 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: (mon.0) 800 : cluster [DBG] mgrmap e25: x(active, since 7s), standbys: y 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: 21:02:28.160576+0000 mon.b (mon.1) 53 : audit 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: cluster 2023-06-10T21:02: 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: 28.414899+0000 mgr.x (mgr.24857) 11 2023-06-10T21:02:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 491 op/s 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: :02:28.486009+0000 mon.a (mon. 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: 0) 801 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: -10T21:02:28.507210+0000 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: mon.a (mon.0) 802 : audit [INF] 2023-06-10T21:02:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: audit 2023-06-10T21:02:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: 2023-06-10T21:02:29.151639+0000 2023-06-10T21:02:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: mon.c (mon.2) 51 : audit [DBG] 2023-06-10T21:02:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/3126824244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: cluster 2023-06-10T21:02:28.153953+0000 mon.a (mon.0) 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: 800 : cluster [DBG] mgrmap e25: x(active, since 7s), standbys: y 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: audit 2023-06-10T21:02:28.160576+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: cluster 2023-06-10T21:02:28.414899+0000 mgr.x (mgr.24857) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 491 op/s 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: audit 2023-06-10T21:02:28.486009+0000 mon.a (mon.0) 801 : audit [INF] 2023-06-10T21:02:29.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: audit 2023-06-10T21:02:28.507210+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: audit 2023-06-10T21:02:29.151639+0000 2023-06-10T21:02:29.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: mon.c (mon.2) 51 : audit [DBG] 2023-06-10T21:02:29.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:29 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/3126824244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:29.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: cluster 2023-06-10T21:02:28.153953+0000 mon.a (mon.0) 800 : cluster [DBG] mgrmap e25: x(active, since 7s), standbys: y 2023-06-10T21:02:29.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: audit 2023-06-10T21:02:28.160576+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:02:29.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: cluster 2023-06-10T21:02:28.414899+0000 mgr.x (mgr.24857) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 12 KiB/s rd, 235 KiB/s wr, 491 op/s 2023-06-10T21:02:29.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: audit 2023-06-10T21:02:28.486009+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: audit 2023-06-10T21:02:28.507210+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:29.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:29 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.151639+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.64:0/3126824244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: audit 2023-06-10T21:02:29.214803+0000 2023-06-10T21:02:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: mon.a (mon.0) 803 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: audit 2023-06-10T21:02:29.224950+0000 mon.a (mon. 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: 0) 804 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: audit 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: 2023-06-10T21:02:29.700536+0000 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: mon.a (mon.0) 805 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: audit 2023-06-10T21:02:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: 2023-06-10T21:02:29 2023-06-10T21:02:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: .913621+0000 mon.a (mon.0) 806 2023-06-10T21:02:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:02:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: T21:02:29.927766+0000 mon.a ( 2023-06-10T21:02:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[107122]: mon.0) 807 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: audit 2023-06-10T21:02:29.214803 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: +0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: audit 2023-06-10T21:02:29.224950+0000 mon.a (mon 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: .0) 804 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: audit 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: 2023-06-10T21:02:29.700536+0000 mon.a (mon.0 2023-06-10T21:02:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: ) 805 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: audit 2023-06-10T21 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: :02:29.913621+0000 mon.a (mon.0 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: ) 806 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: 10T21:02:29.927766+0000 mon.a 2023-06-10T21:02:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:30 smithi064 conmon[97224]: (mon.0) 807 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:30 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.214803+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:30 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.224950+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:30 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.700536+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:30 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.913621+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:30 smithi152 conmon[97365]: audit 2023-06-10T21:02:29.927766+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:31 smithi152 conmon[97365]: audit 2023-06-10T21:02:30.376113+0000 mon.a (mon 2023-06-10T21:02:31.390 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:31 smithi152 conmon[97365]: .0) 808 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.390 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:31 smithi152 conmon[97365]: audit 2023-06-10T21:02:30.397801+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.390 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:31 smithi152 conmon[97365]: cluster 2023-06-10T21:02:30.415318+0000 mgr.x (mgr.24857) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 386 op/s 2023-06-10T21:02:31.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[107122]: audit 2023-06-10T21:02:30.376113+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[107122]: audit 2023-06-10T21:02:30.397801+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[107122]: cluster 2023-06-10T21:02:30.415318+0000 mgr.x (mgr.24857) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 386 op/s 2023-06-10T21:02:31.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[97224]: audit 2023-06-10T21:02:30.376113+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[97224]: audit 2023-06-10T21:02:30.397801+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:31.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:31 smithi064 conmon[97224]: cluster 2023-06-10T21:02:30.415318+0000 mgr.x (mgr.24857) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 9.3 KiB/s rd, 185 KiB/s wr, 386 op/s 2023-06-10T21:02:33.196 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:32 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:32] "GET /metrics HTTP/1.1" 200 33316 "" "Prometheus/2.33.4" 2023-06-10T21:02:33.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:31.761153+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.64:0/2569238779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:33.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:32.060044+0000 mon.a (mon.0) 810 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:33.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:32.061125+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:33.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:32.061278+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:33.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:32.061386+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:33.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: cluster 2023-06-10T21:02:32.416023+0000 mgr.x (mgr.24857) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 673 op/s 2023-06-10T21:02:33.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.127175+0000 mon.a (mon.0) 811 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:33.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:31.761153+0000 2023-06-10T21:02:33.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.64:0/2569238779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:33.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:32.060044+0000 mon.a (mon.0) 810 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:33.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:32.061125+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:33.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:32.061278+0000 mon.b (mon.1) 55 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:33.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:32.061386+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:33.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: cluster 2023-06-10T21:02:32.416023+0000 mgr.x (mgr.24857) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 673 op/s 2023-06-10T21:02:33.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:33 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.127175+0000 mon.a (mon.0) 811 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:34.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02:31.761153+0000 2023-06-10T21:02:34.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.64:0/2569238779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:34.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02:32.060044+0000 mon.a (mon.0) 810 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02:32.061125+0000 mon.b (mon.1) 54 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02:32 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: .061278+0000 mon.b (mon.1) 55 : audit 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02:32 2023-06-10T21:02:34.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: .061386+0000 mon.b (mon.1) 56 : audit 2023-06-10T21:02:34.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: cluster 2023-06-10T21:02:32 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: .416023+0000 mgr.x (mgr.24857) 13 : cluster 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 673 op/s 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit 2023-06-10T21:02 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: :33.127175+0000 mon.a (mon.0) 811 : 2023-06-10T21:02:34.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:33 smithi152 conmon[97365]: audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:35.015 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.128932+0000 mon.b (mon.1) 57 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:35.015 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.812388+0000 mon.b (mon.1) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:35.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.816170+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.816373+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.816546+0000 mon.b (mon.1) 61 2023-06-10T21:02:35.016 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.829082+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.829335+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.829541+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.017 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[97224]: audit 2023-06-10T21:02:34.465516+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.64:0/1185144871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.128932+0000 mon.b (mon.1) 57 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.812388+0000 mon.b (mon.1) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.816170+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.816373+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.816546+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.829082+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.829335+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.829541+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:35 smithi064 conmon[107122]: audit 2023-06-10T21:02:34.465516+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.64:0/1185144871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.128932+0000 mon.b (mon.1) 57 : audit 2023-06-10T21:02:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.812388+0000 mon.b (mon.1) 58 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.816170+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.816373+0000 mon.b (mon.1) 60 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.816546+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.829082+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.829335+0000 mon.b (mon.1) 63 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.829541+0000 mon.b (mon.1) 64 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:02:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:35 smithi152 conmon[97365]: audit 2023-06-10T21:02:34.465516+0000 mon.a (mon.0) 812 : audit [DBG] from='client.? 172.21.15.64:0/1185144871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:36.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:33.828685+0000 2023-06-10T21:02:36.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: cluster 2023-06-10T21:02:34.418842+0000 mgr.x (mgr.24857) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 569 op/s 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:34.992097+0000 mon.c ( 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:35. 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 598608+0000 mon.b (mon.1) 65 : audit 2023-06-10T21:02:36.026 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:35 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: .771266+0000 mon.a (mon.0) 813 : 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 10T21:02:35.784320+0000 mon.a 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: (mon.0) 814 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.027 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:35.796440+0000 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: mon.b (mon.1) 66 : audit [INF] 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02 2023-06-10T21:02:36.028 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: :35.796988+0000 mon.a (mon.0) 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 815 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:35 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: .797964+0000 mon.b (mon.1) 67 : audit 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.029 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:02:36.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:33.828685+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:36.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: cluster 2023-06-10T21:02:34.418842+0000 mgr.x (mgr.24857) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 569 op/s 2023-06-10T21:02:36.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:34.992097+0000 mon.c (mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:36.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.598608+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:36.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.771266+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.784320+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.796440+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.796988+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.797964+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.798446+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: 21:02:35.799513+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.800001+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.810503+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.811112+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:36 smithi152 conmon[97365]: audit 2023-06-10T21:02:35.817993+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: -10T21:02:35.798446+0000 2023-06-10T21:02:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: mon.a (mon.0) 816 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:35. 2023-06-10T21:02:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 799513+0000 mon.b (mon.1) 68 : audit 2023-06-10T21:02:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02:35.800001+0000 mon.a 2023-06-10T21:02:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: (mon.0) 817 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: -10T21:02:35.810503+0000 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: mon.b (mon.1) 69 : audit [INF] 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: T21:02:35.811112+0000 mon.a 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: (mon.0) 818 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 2023-06-10T21:02 2023-06-10T21:02:36.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: :35.817993+0000 mon.a (mon.0) 2023-06-10T21:02:36.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[97224]: 819 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:33.828685+0000 mon.c (mon.2) 53 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-10T21:02:36.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: cluster 2023-06-10T21:02:34.418842+0000 mgr.x (mgr.24857) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.7 KiB/s rd, 182 KiB/s wr, 569 op/s 2023-06-10T21:02:36.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:34.992097+0000 mon.c (mon.2) 54 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-10T21:02:36.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.598608+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:36.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.771266+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.784320+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.796440+0000 mon.b (mon.1) 66 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.796988+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.797964+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.798446+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.799513+0000 mon.b (mon.1) 68 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.800001+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.810503+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.811112+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:36.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:36 smithi064 conmon[107122]: audit 2023-06-10T21:02:35.817993+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:35.812721+0000 mgr.x ( 2023-06-10T21:02:37.115 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: mgr.24857) 15 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 2500M 2023-06-10T21:02:37.115 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: cluster 2023-06-10T21:02:36.419236+0000 mgr.x (mgr.24857) 16 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 561 op/s 2023-06-10T21:02:37.115 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.844522+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.115 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.867411+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.880131+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.880750+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.881736+0000 mon.b (mon.1) 71 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.116 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.882206+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.883032+0000 mon.b (mon.1) 72 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.883492+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.118 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.884287+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.118 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.884984+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.118 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:36.896365+0000 mon.a ( 2023-06-10T21:02:37.118 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: mon.0) 826 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21:02:37.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: 2023-06-10T21:02:36.897898+0000 mon.b (mon 2023-06-10T21:02:37.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: .1) 74 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:37.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:02:37.120 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: 02:36.898708+0000 mon.b (mon.1) 75 : 2023-06-10T21:02:37.120 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:37 smithi152 conmon[97365]: audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:37.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:02:37.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:35. 2023-06-10T21:02:37.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 812721+0000 mgr.x (mgr.24857) 15 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 2500M 2023-06-10T21:02:37.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: cluster 2023- 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 06-10T21:02:36.419236+0000 mgr.x (mgr.24857) 16 : cluster 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 561 op/s 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 2023-06-10T21:02:36.844522+0000 mon.a (mon. 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 0) 820 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:37.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 02:36.867411+0000 mon.a (mon.0) 821 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 21:02:36.880131+0000 mon.b (mon.1 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: ) 70 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :02:36.880750+0000 mon.a (mon.0 2023-06-10T21:02:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: ) 822 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:37.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 02:36.881736+0000 mon.b (mon.1) 2023-06-10T21:02:37.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 71 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:02:37.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :02:36.882206+0000 mon.a (mon.0) 2023-06-10T21:02:37.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 823 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21:02 2023-06-10T21:02:37.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :36.883032+0000 mon.b (mon.1) 2023-06-10T21:02:37.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 72 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21:02 2023-06-10T21:02:37.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :36.883492+0000 mon.a (mon.0) 824 : 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21:02: 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 36.884287+0000 mon.b (mon.1) 73 : audit 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21:02:36.884984 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: +0000 mon.a (mon.0) 825 : audit [INF] 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:02:37.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 06-10T21:02:36.896365+0000 mon.a (mon.0) 826 2023-06-10T21:02:37.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:02:37.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :02:36.897898+0000 mon.b (mon.1) 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:35.812721+0000 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: mgr.x (mgr.24857) 15 : cephadm [INF] Adjusting osd_memory_target on smithi064 to 2500M 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: cluster 2023-06-10T21:02:36.419236+0000 mgr.x (mgr.24857) 16 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 561 op/s 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.844522+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.867411+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:02:37.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: T21:02:36.880131+0000 mon.b (mon.1) 70 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:36.880750 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: +0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:36. 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 881736+0000 mon.b (mon.1) 71 : audit 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:37.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: -06-10T21:02:36.882206+0000 mon.a (mon.0) 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 823 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:36.883032 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: +0000 mon.b (mon.1) 72 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 06-10T21:02:36 2023-06-10T21:02:37.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: .883492+0000 mon.a (mon. 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 0) 824 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02: 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 36.884287+0000 mon.b (mon. 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 1) 73 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02: 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 36.884984+0000 mon.a (mon 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: .0) 825 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 06-10T21:02:36 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: .896365+0000 mon.a (mon. 2023-06-10T21:02:37.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 0) 826 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:37.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:37.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:37.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02: 2023-06-10T21:02:37.422 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 36.897898+0000 mon.b (mon. 2023-06-10T21:02:37.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: 74 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: :02:36.898708+0000 mon.b (mon.1 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[107122]: ) 75 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 1) 74 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:37.423 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: audit 2023-06-10T21:02:37.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02: 2023-06-10T21:02:37.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 36.898708+0000 mon.b (mon. 2023-06-10T21:02:37.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 1) 75 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:02:37.424 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:37 smithi064 conmon[97224]: 2023-06-10T21:02:38.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:36. 2023-06-10T21:02:38.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: 888647+0000 mgr.x (mgr.24857) 17 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 2500M 2023-06-10T21:02:38.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:02:38.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: 06-10T21:02:36.899893+0000 mgr.x (mgr.24857) 18 : cephadm 2023-06-10T21:02:38.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: T21:02:36.900054+0000 mgr.x ( 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: mgr.24857) 19 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: 06-10T21:02:37.133660+0000 mon.c (mon.2) 55 : audit 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: [DBG] from='client.? 172.21.15.64:0/14818184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:38.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm 2023-06 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: -10T21:02:37.372702+0000 mgr.x (mgr.24857) 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: 20 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm 2023-06-10T21 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: :02:37.478520+0000 mgr.x (mgr 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: .24857) 21 : 2023-06-10T21:02:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[107122]: cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:38.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:36.888647+0000 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: mgr.x (mgr.24857) 17 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 2500M 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:36.899893 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 18 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm 2023-06-10 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: T21:02:36.900054+0000 2023-06-10T21:02:38.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: mgr.x (mgr.24857) 19 : 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: audit 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: 2023-06-10T21:02 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: :37.133660+0000 mon.c ( 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: mon.2) 55 : audit [DBG] 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/14818184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T21:02:38.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: 06-10T21:02:37 2023-06-10T21:02:38.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: .372702+0000 mgr.x (mgr.24857 2023-06-10T21:02:38.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: ) 20 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:38.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:38 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:37.478520+0000 mgr.x (mgr.24857) 21 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:38.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:36.888647+0000 mgr.x (mgr.24857) 17 : cephadm [INF] Adjusting osd_memory_target on smithi152 to 2500M 2023-06-10T21:02:38.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:36.899893+0000 mgr.x (mgr.24857) 18 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:02:38.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:36.900054+0000 mgr.x (mgr.24857) 19 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:02:38.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: audit 2023-06-10T21:02:37.133660+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.64:0/14818184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:38.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:37.372702+0000 mgr.x (mgr.24857) 20 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:38.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:38 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:37.478520+0000 mgr.x (mgr.24857) 21 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:02:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:37.940771+0000 mgr.x (mgr.24857) 22 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:37.957267+0000 mgr.x (mgr.24857) 23 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: cluster 2023-06-10T21:02:38.419867+0000 mgr.x (mgr.24857) 24 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 836 op/s 2023-06-10T21:02:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:38.443791+0000 mgr.x (mgr.24857) 25 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:38.531382+0000 mgr.x (mgr.24857) 26 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: audit 2023-06-10T21:02:38.983078+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[107122]: audit 2023-06-10T21:02:39.010928+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:39.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:37.940771+0000 2023-06-10T21:02:39.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: mgr.x (mgr.24857) 22 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:37.957267+0000 mgr.x (mgr.24857) 23 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: 2023-06-10T21:02:39.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cluster 2023-06-10T21:02:38 2023-06-10T21:02:39.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: .419867+0000 mgr.x (mgr.24857) 24 : 2023-06-10T21:02:39.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 836 op/s 2023-06-10T21:02:39.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cephadm 2023-06-10T21 2023-06-10T21:02:39.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: :02:38.443791+0000 mgr.x (mgr. 2023-06-10T21:02:39.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: 24857) 25 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: cephadm 2023-06-10T 2023-06-10T21:02:39.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: 21:02:38.531382+0000 mgr.x (mgr 2023-06-10T21:02:39.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: .24857) 26 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: -10T21:02:38.983078+0000 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: mon.a (mon.0) 827 : audit [INF] 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: audit 2023 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: -06-10T21:02:39.010928+0000 mon.a 2023-06-10T21:02:39.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:39 smithi064 conmon[97224]: (mon.0) 828 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:37.940771+0000 mgr.x (mgr.24857) 22 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:37.957267+0000 mgr.x (mgr.24857) 23 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: cluster 2023-06-10T21:02:38.419867+0000 mgr.x (mgr.24857) 24 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 8.3 KiB/s rd, 201 KiB/s wr, 836 op/s 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:38.443791+0000 mgr.x (mgr.24857) 25 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:38.531382+0000 mgr.x (mgr.24857) 26 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: audit 2023-06-10T21:02:38.983078+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:39.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:39 smithi152 conmon[97365]: audit 2023-06-10T21:02:39.010928+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: audit 2023-06-10T21:02:39. 2023-06-10T21:02:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: 139285+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: audit 2023-06-10T21:02:39.160189+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: audit 2023-06-10T21:02:39.191433+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:39.385651+0000 mgr.x (mgr.24857) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-10T21:02:40.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:39.386015+0000 mgr.x (mgr.24857) 28 : cephadm [INF] 2023-06-10T21:02:40.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: Reconfiguring daemon node-exporter.a on smithi064 2023-06-10T21:02:40.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:02:40.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: 06-10T21:02:39.819428+0000 mon.a (mon.0) 832 : audit [DBG] 2023-06-10T21:02:40.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[97224]: from='client.? 172.21.15.64:0/3705962090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:40.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:39 smithi064 systemd[1]: Stopping Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:40.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:40 smithi064 bash[168576]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-node-exporter.a found: no such container 2023-06-10T21:02:40.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: audit 2023-06-10T21:02:39.139285 2023-06-10T21:02:40.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: +0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: audit 2023-06-10T21:02:39.160189+0000 2023-06-10T21:02:40.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: mon.a (mon.0) 830 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:02:40.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: -10T21:02:39.191433+0000 mon.a (mon.0) 2023-06-10T21:02:40.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: 831 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: cephadm 2023 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: -06-10T21:02:39.385651+0000 mgr.x 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: (mgr.24857) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: 06-10T21:02:39.386015+0000 mgr.x 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: (mgr.24857) 28 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi064 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: audit 2023 2023-06-10T21:02:40.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: -06-10T21:02:39.819428+0000 mon.a (mon.0) 2023-06-10T21:02:40.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:40 smithi064 conmon[107122]: 832 : audit [DBG] from='client.? 172.21.15.64:0/3705962090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: audit 2023-06-10T21:02:39.139285+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: audit 2023-06-10T21:02:39.160189+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: audit 2023-06-10T21:02:39.191433+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:39.385651+0000 mgr.x (mgr.24857) 27 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-10T21:02:40.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:39.386015+0000 mgr.x (mgr.24857) 28 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi064 2023-06-10T21:02:40.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:40 smithi152 conmon[97365]: audit 2023-06-10T21:02:39.819428+0000 mon.a (mon.0) 832 : audit [DBG] from='client.? 172.21.15.64:0/3705962090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:41.155 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:40 smithi064 bash[168683]: bf0e1033c7ed03b77a062ad9ce4da94a1b033e72352315f9f9d2d6d51b4274f3 2023-06-10T21:02:41.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:41 smithi064 conmon[107122]: cluster 2023-06-10T21:02:40.423690+0000 mgr.x (mgr.24857) 29 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 552 op/s 2023-06-10T21:02:41.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:41 smithi064 conmon[97224]: cluster 2023-06-10T21:02:40.423690+0000 2023-06-10T21:02:41.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:41 smithi064 conmon[97224]: mgr.x (mgr.24857) 29 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 552 op/s 2023-06-10T21:02:41.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:41 smithi152 conmon[97365]: cluster 2023-06-10T21:02:40.423690+0000 mgr.x (mgr.24857) 29 : cluster [DBG] 2023-06-10T21:02:41.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:41 smithi152 conmon[97365]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 552 op/s 2023-06-10T21:02:42.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:42 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:02:42.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:42 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.a.service: Failed with result 'exit-code'. 2023-06-10T21:02:42.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:42 smithi064 systemd[1]: Stopped Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:42.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:42 smithi064 systemd[1]: Starting Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:42.697 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:02:42 smithi152 conmon[132019]: ts=2023-06-10T21:02:42.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.64:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:02:43.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:42 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:42] "GET /metrics HTTP/1.1" 200 189988 "" "Prometheus/2.33.4" 2023-06-10T21:02:43.409 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.097Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-10T21:02:43.409 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.097Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-10T21:02:43.410 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.097Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-10T21:02:43.410 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.097Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-10T21:02:43.410 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-10T21:02:43.410 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=arp 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-10T21:02:43.411 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-10T21:02:43.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-10T21:02:43.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=edac 2023-06-10T21:02:43.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-10T21:02:43.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-10T21:02:43.412 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.098Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-10T21:02:43.413 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-10T21:02:43.414 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-10T21:02:43.414 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-10T21:02:43.414 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-10T21:02:43.414 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=os 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-10T21:02:43.415 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-10T21:02:43.416 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-10T21:02:43.416 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-10T21:02:43.416 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-10T21:02:43.416 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=stat 2023-06-10T21:02:43.416 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=time 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=uname 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-10T21:02:43.417 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-10T21:02:43.418 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-10T21:02:43.418 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-10T21:02:43.418 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[169058]: ts=2023-06-10T21:02:43.099Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-10T21:02:43.418 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 bash[169021]: c7179c864e62f6a284404f0d439e2dd338b8ac2c74f4c122a3725dca11c7dc10 2023-06-10T21:02:43.418 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:02:43 smithi064 systemd[1]: Started Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[107122]: cluster 2023-06-10T21:02:42.424344+0000 mgr.x (mgr.24857) 30 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 862 op/s 2023-06-10T21:02:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[107122]: audit 2023-06-10T21:02:42.491776+0000 mon.b (mon.1) 76 : audit [DBG] from='client.? 172.21.15.64:0/759171528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[107122]: audit 2023-06-10T21:02:43.181641+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[107122]: audit 2023-06-10T21:02:43.197930+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: cluster 2023-06-10T21:02:42.424344+0000 mgr.x (mgr. 2023-06-10T21:02:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: 24857) 30 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 862 op/s 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: audit 2023-06-10T21:02:42.491776+0000 mon.b (mon.1) 76 : 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: audit [DBG] from='client.? 172.21.15.64:0/759171528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: audit 2023-06-10T21:02: 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: 43.181641+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: audit 2023-06-10T21 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: :02:43.197930+0000 mon.a (mon.0) 2023-06-10T21:02:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:43 smithi064 conmon[97224]: 834 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: cluster 2023-06-10T21:02:42.424344+0000 mgr.x (mgr.24857) 2023-06-10T21:02:43.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: 30 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 862 op/s 2023-06-10T21:02:43.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: audit 2023-06-10T21:02:42.491776+0000 mon.b (mon.1) 76 : audit [DBG] from='client.? 172.21.15.64:0/759171528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:43.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:02:43.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: 06-10T21:02:43.181641+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: audit 2023-06-10T21:02:43.197930 2023-06-10T21:02:43.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: +0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:43.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:43 smithi152 conmon[97365]: 2023-06-10T21:02:44.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:44 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:43.242082+0000 mgr.x (mgr.24857) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:02:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:44 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:43.256746+0000 mgr.x (mgr.24857) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:02:44.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:44 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:43.242082+0000 2023-06-10T21:02:44.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:44 smithi064 conmon[97224]: mgr.x (mgr.24857) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:02:44.918 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:44 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:43.256746+0000 mgr.x (mgr.24857) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:02:44.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:44 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:43.242082+0000 mgr.x (mgr.24857) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-10T21:02:44.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:44 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:43.256746+0000 mgr.x (mgr.24857) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi064 2023-06-10T21:02:45.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[97224]: cluster 2023-06-10T21:02:44.429246+0000 2023-06-10T21:02:45.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[97224]: mgr.x (mgr.24857) 33 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:45.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[97224]: audit 2023-06-10T21:02:45.173936+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.64:0/17998874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:45.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[97224]: 2023-06-10T21:02:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[107122]: cluster 2023-06-10T21:02: 2023-06-10T21:02:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[107122]: 44.429246+0000 mgr.x (mgr.24857) 33 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[107122]: 02:45.173936+0000 mon.c (mon.2) 56 : 2023-06-10T21:02:45.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:45 smithi064 conmon[107122]: audit [DBG] from='client.? 172.21.15.64:0/17998874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:45.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:45 smithi152 conmon[97365]: cluster 2023-06-10T21:02:44.429246+0000 mgr.x (mgr 2023-06-10T21:02:45.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:45 smithi152 conmon[97365]: .24857) 33 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:45.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:45 smithi152 conmon[97365]: audit 2023-06-10T21:02:45.173936+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.64:0/17998874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:47.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:47 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:02:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:47 smithi064 conmon[97224]: cluster 2023-06-10T21:02:46.429631+0000 mgr.x (mgr.24857) 34 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:47.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:47 smithi064 conmon[107122]: cluster 2023-06-10T21:02:46.429631 2023-06-10T21:02:47.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:47 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 34 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:47 smithi152 conmon[97365]: cluster 2023-06-10T21:02:46.429631+0000 mgr.x (mgr.24857) 34 : cluster 2023-06-10T21:02:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:47 smithi152 conmon[97365]: [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 585 op/s 2023-06-10T21:02:48.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:48 smithi064 conmon[97224]: audit 2023-06-10T21:02:47.855219+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.64:0/2796799661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:48.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:48 smithi064 conmon[107122]: audit 2023-06-10T21:02:47.855219+0000 2023-06-10T21:02:48.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:48 smithi064 conmon[107122]: mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.64:0/2796799661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:48 smithi152 conmon[97365]: audit 2023-06-10T21:02:47.855219+0000 mon.c ( 2023-06-10T21:02:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:48 smithi152 conmon[97365]: mon.2) 57 : audit [DBG] from='client.? 172.21.15.64:0/2796799661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:49.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:49 smithi064 conmon[97224]: cluster 2023-06-10T21:02: 2023-06-10T21:02:49.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:49 smithi064 conmon[97224]: 48.433525+0000 mgr.x (mgr.24857) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 900 op/s 2023-06-10T21:02:49.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:49 smithi064 conmon[107122]: cluster 2023-06-10T21:02:48.433525 2023-06-10T21:02:49.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:49 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 900 op/s 2023-06-10T21:02:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:49 smithi152 conmon[97365]: cluster 2023-06-10T21:02:48.433525+0000 mgr.x (mgr 2023-06-10T21:02:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:49 smithi152 conmon[97365]: .24857) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 900 op/s 2023-06-10T21:02:51.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[97224]: cluster 2023-06-10T21:02:50.433882+0000 mgr.x (mgr.24857) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-06-10T21:02:51.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[97224]: audit 2023-06-10T21:02:50.499370+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.64:0/2712621699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:51.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[97224]: audit 2023-06-10T21:02:50.598790+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: cluster 2023-06-10T21:02:50.433882+0000 mgr.x (mgr.24857) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: audit 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: 2023-06-10T21:02:50. 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: 499370+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.64:0/2712621699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:02:51.878 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: 21:02:50.598790+0000 mon.b (mon.1) 77 2023-06-10T21:02:51.879 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:51 smithi064 conmon[107122]: : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:51.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:51 smithi152 conmon[97365]: cluster 2023-06-10T21:02:50.433882+0000 mgr.x ( 2023-06-10T21:02:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:51 smithi152 conmon[97365]: mgr.24857) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 625 op/s 2023-06-10T21:02:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:51 smithi152 conmon[97365]: audit 2023-06-10T21:02:50.499370+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.64:0/2712621699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:51 smithi152 conmon[97365]: audit 2023-06-10T21:02:50.598790 2023-06-10T21:02:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:51 smithi152 conmon[97365]: +0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:02:52.697 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:02:52 smithi152 conmon[132019]: ts=2023-06-10T21:02:52.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.64:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:02:52.908 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:52 smithi064 systemd[1]: Stopping Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:52.908 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:52 smithi064 bash[169583]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-alertmanager.a found: no such container 2023-06-10T21:02:53.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:02:52 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:52] "GET /metrics HTTP/1.1" 200 189996 "" "Prometheus/2.33.4" 2023-06-10T21:02:53.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[158459]: level=info ts=2023-06-10T21:02:53.294Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:02:53.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:53 smithi064 bash[169711]: 163fca1ec9a54a7e49b48476f0249f1d643cefeac1e7019799d3f2af5fe46010 2023-06-10T21:02:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[107122]: cluster 2023-06-10T21:02:52 2023-06-10T21:02:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[107122]: .434463+0000 mgr.x (mgr.24857) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-10T21:02:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[107122]: 02:53.195322+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.64:0/2629930984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:53.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[97224]: cluster 2023-06-10T21:02:52.434463+0000 mgr.x (mgr.24857) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-10T21:02:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:53 smithi064 conmon[97224]: audit 2023-06-10T21:02:53.195322+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.64:0/2629930984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:53.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:53 smithi152 conmon[97365]: cluster 2023-06-10T21:02:52.434463+0000 mgr.x (mgr.24857) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-10T21:02:53.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:53 smithi152 conmon[97365]: audit 2023-06-10T21:02:53.195322+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.64:0/2629930984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:53.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:02:53 smithi152 conmon[132019]: ts=2023-06-10T21:02:53.506Z caller=notifier.go:526 level=error component=notifier alertmanager=http://smithi064.front.sepia.ceph.com:9093/api/v2/alerts count=4 msg="Error sending alert" err="Post \"http://smithi064.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.64:9093: connect: connection refused" 2023-06-10T21:02:54.661 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:54 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-10T21:02:54.661 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:54 smithi064 systemd[1]: Stopped Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:54.661 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:54 smithi064 systemd[1]: Starting Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:02:55.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.373Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-06-10T21:02:55.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.373Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-06-10T21:02:55.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.374Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.64 port=9094 2023-06-10T21:02:55.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.377Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-06-10T21:02:55.659 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 bash[170030]: 13fd7844ce612dcb5667df3fcfb587e62e7875ea5981408350c3fa7f477c606c 2023-06-10T21:02:55.660 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 systemd[1]: Started Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:02:55.660 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.446Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T21:02:55.660 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.447Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-10T21:02:55.660 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.454Z caller=main.go:518 msg=Listening address=:9093 2023-06-10T21:02:55.660 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:55.454Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-10T21:02:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: cluster 2023-06-10T21:02:54.434848 2023-06-10T21:02:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:55.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:02:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: -10T21:02:55.477929+0000 mon.a (mon.0) 837 : audit 2023-06-10T21:02:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:02:55.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: 02:55.498695+0000 mon.a (mon.0 2023-06-10T21:02:55.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[107122]: ) 838 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:55.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[97224]: cluster 2023-06-10T21:02:54.434848+0000 mgr.x ( 2023-06-10T21:02:55.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[97224]: mgr.24857) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[97224]: audit 2023-06-10T21:02:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[97224]: 2023-06-10T21:02:55.477929+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:55.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:55 smithi064 conmon[97224]: audit 2023-06-10T21:02:55.498695+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:55.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:55 smithi152 conmon[97365]: cluster 2023-06-10T21:02:54.434848+0000 mgr.x (mgr.24857) 38 : cluster [DBG] 2023-06-10T21:02:55.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:55 smithi152 conmon[97365]: pgmap v22: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:55.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:55 smithi152 conmon[97365]: audit 2023-06-10T21:02:55.477929+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:55.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:55 smithi152 conmon[97365]: audit 2023-06-10T21:02:55.498695+0000 mon.a (mon.0 2023-06-10T21:02:55.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:55 smithi152 conmon[97365]: ) 838 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:02:56.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:56 smithi064 conmon[107122]: cephadm 2023-06-10T21:02:55.533020+0000 mgr.x (mgr.24857) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:02:56.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:56 smithi064 conmon[107122]: audit 2023-06-10T21:02:55.856324+0000 mon.b (mon.1) 78 : audit [DBG] from='client.? 172.21.15.64:0/2962238528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:56 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:55.533020+0000 mgr.x (mgr.24857) 2023-06-10T21:02:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:56 smithi064 conmon[97224]: 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:02:56.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:56 smithi064 conmon[97224]: audit 2023-06-10T21:02:55.856324+0000 mon.b (mon.1) 78 : audit [DBG] from='client.? 172.21.15.64:0/2962238528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:56.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:56 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:55.533020+0000 mgr.x (mgr 2023-06-10T21:02:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:56 smithi152 conmon[97365]: .24857) 39 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:02:56.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:56 smithi152 conmon[97365]: audit 2023-06-10T21:02:55.856324+0000 mon.b (mon.1) 78 : audit [DBG] from='client.? 172.21.15.64:0/2962238528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:57.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:02:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:02:57.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[170068]: level=info ts=2023-06-10T21:02:57.381Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.003910613s 2023-06-10T21:02:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[97224]: cephadm 2023-06-10T21:02:55.782033+0000 2023-06-10T21:02:57.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[97224]: mgr.x (mgr.24857) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:02:57.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[97224]: cluster 2023-06-10T21:02:56.435282+0000 mgr.x ( 2023-06-10T21:02:57.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[97224]: mgr.24857) 41 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[107122]: cephadm 2023-06-10T21:02: 2023-06-10T21:02:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[107122]: 55.782033+0000 mgr.x (mgr.24857) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:02:57.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[107122]: cluster 2023- 2023-06-10T21:02:57.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[107122]: 06-10T21:02:56.435282+0000 mgr.x (mgr.24857) 41 2023-06-10T21:02:57.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:57 smithi064 conmon[107122]: : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:57 smithi152 conmon[97365]: cephadm 2023-06-10T21:02:55.782033+0000 mgr.x (mgr.24857) 40 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:02:57.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:57 smithi152 conmon[97365]: cluster 2023-06-10T21:02:56.435282+0000 mgr.x (mgr.24857) 41 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:02:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:58 smithi064 conmon[97224]: audit 2023-06-10T21:02:58.476742+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.64:0/2652628392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:58 smithi064 conmon[107122]: audit 2023-06-10T21:02:58.476742+0000 2023-06-10T21:02:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:58 smithi064 conmon[107122]: mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.64:0/2652628392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:58 smithi152 conmon[97365]: audit 2023-06-10T21:02:58.476742+0000 mon.a 2023-06-10T21:02:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:58 smithi152 conmon[97365]: (mon.0) 839 : audit [DBG] from='client.? 172.21.15.64:0/2652628392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:02:59.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:59 smithi064 conmon[107122]: cluster 2023-06-10T21:02:58.435914 2023-06-10T21:02:59.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:02:59 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 42 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-06-10T21:02:59.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:59 smithi064 conmon[97224]: cluster 2023-06-10T21:02:58.435914+0000 2023-06-10T21:02:59.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:02:59 smithi064 conmon[97224]: mgr.x (mgr.24857) 42 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-06-10T21:02:59.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:59 smithi152 conmon[97365]: cluster 2023-06-10T21:02:58.435914+0000 2023-06-10T21:02:59.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:02:59 smithi152 conmon[97365]: mgr.x (mgr.24857) 42 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 915 op/s 2023-06-10T21:03:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:01 smithi064 conmon[107122]: cluster 2023-06-10T21:03:00.436270+0000 mgr.x (mgr.24857) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-10T21:03:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:01 smithi064 conmon[107122]: audit 2023-06-10T21:03:01.105415+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.64:0/680107153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:01 smithi064 conmon[97224]: cluster 2023-06-10T21:03:00.436270+0000 2023-06-10T21:03:01.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:01 smithi064 conmon[97224]: mgr.x (mgr.24857) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-10T21:03:01.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:01 smithi064 conmon[97224]: audit 2023-06-10T21:03:01.105415+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.64:0/680107153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:01 smithi152 conmon[97365]: cluster 2023-06-10T21:03:00.436270 2023-06-10T21:03:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:01 smithi152 conmon[97365]: +0000 mgr.x (mgr.24857) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-10T21:03:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:01 smithi152 conmon[97365]: audit 2023-06-10T21:03:01.105415+0000 mon.a (mon.0) 840 : audit [DBG] from='client.? 172.21.15.64:0/680107153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:02.696 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:02 smithi152 conmon[132019]: ts=2023-06-10T21:03:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.64:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:03.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:02 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:02] "GET /metrics HTTP/1.1" 200 189996 "" "Prometheus/2.33.4" 2023-06-10T21:03:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:03 smithi064 conmon[97224]: cluster 2023-06-10T21:03:02.436893+0000 mgr.x (mgr.24857) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-06-10T21:03:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:03 smithi064 conmon[107122]: cluster 2023-06-10T21:03:02. 2023-06-10T21:03:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:03 smithi064 conmon[107122]: 436893+0000 mgr.x (mgr.24857) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-06-10T21:03:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:03 smithi152 conmon[97365]: cluster 2023-06-10T21:03:02.436893+0000 mgr.x (mgr.24857) 44 2023-06-10T21:03:03.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:03 smithi152 conmon[97365]: : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2023-06-10T21:03:04.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:04 smithi064 conmon[107122]: audit 2023-06-10T21:03:03.754610+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.64:0/3516406089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:04.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:04 smithi064 conmon[97224]: audit 2023-06-10T21:03:03.754610 2023-06-10T21:03:04.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:04 smithi064 conmon[97224]: +0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.64:0/3516406089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:04.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:04 smithi152 conmon[97365]: audit 2023-06-10T21:03:03.754610+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.64:0/3516406089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:05.656 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[170068]: level=info ts=2023-06-10T21:03:05.384Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.006768752s 2023-06-10T21:03:05.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[97224]: cluster 2023-06-10T21:03:04.437264+0000 2023-06-10T21:03:05.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[97224]: mgr.x (mgr.24857) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:05.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[97224]: audit 2023-06-10T21:03:05.599377+0000 mon.b 2023-06-10T21:03:05.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[97224]: (mon.1) 79 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[107122]: cluster 2023-06-10T21:03:04 2023-06-10T21:03:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[107122]: .437264+0000 mgr.x (mgr.24857) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:05.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[107122]: audit 2023 2023-06-10T21:03:05.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[107122]: -06-10T21:03:05.599377+0000 mon.b (mon.1 2023-06-10T21:03:05.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:05 smithi064 conmon[107122]: ) 79 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:05.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:05 smithi152 conmon[97365]: cluster 2023-06-10T21:03:04.437264+0000 mgr.x (mgr 2023-06-10T21:03:05.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:05 smithi152 conmon[97365]: .24857) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:05.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:05 smithi152 conmon[97365]: audit 2023-06-10T21:03:05.599377+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[97224]: audit 2023-06-10T21:03:06 2023-06-10T21:03:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[97224]: .394534+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.64:0/2649744752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[97224]: cluster 2023-06-10T21 2023-06-10T21:03:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[97224]: :03:06.439787+0000 mgr.x (mgr.24857) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:07.410 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[97677]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-10T21:03:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:03:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: 03:06.394534+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.64:0/2649744752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: cluster 2023-06-10T21:03:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: 2023-06-10T21: 2023-06-10T21:03:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: 03:06.439787+0000 mgr.x ( 2023-06-10T21:03:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: mgr.24857) 46 : cluster [DBG] 2023-06-10T21:03:07.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:07 smithi064 conmon[107122]: pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:07.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:07 smithi152 conmon[97365]: audit 2023-06-10T21:03:06.394534+0000 mon.b (mon.1) 80 : audit [DBG] from='client.? 172.21.15.64:0/2649744752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:07.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:07 smithi152 conmon[97365]: cluster 2023-06-10T21:03:06.439787+0000 mgr.x (mgr.24857) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-10T21:03:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:09 smithi064 conmon[107122]: cluster 2023-06-10T21:03:08.440397+0000 mgr.x (mgr.24857) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-10T21:03:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:09 smithi064 conmon[107122]: audit 2023-06-10T21:03:09.063618+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.64:0/1780099960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:09 smithi064 conmon[97224]: cluster 2023-06-10T21:03:08.440397+0000 2023-06-10T21:03:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:09 smithi064 conmon[97224]: mgr.x (mgr.24857) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-10T21:03:09.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:09 smithi064 conmon[97224]: audit 2023-06-10T21:03:09.063618+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.64:0/1780099960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:09.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:09 smithi152 conmon[97365]: cluster 2023-06-10T21:03:08.440397+0000 mgr.x ( 2023-06-10T21:03:09.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:09 smithi152 conmon[97365]: mgr.24857) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-10T21:03:09.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:09 smithi152 conmon[97365]: audit 2023-06-10T21:03:09.063618+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.64:0/1780099960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:11.670 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:11 smithi152 conmon[97365]: cluster 2023-06-10T21:03:10.440715+0000 mgr.x 2023-06-10T21:03:11.670 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:11 smithi152 conmon[97365]: (mgr.24857) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-10T21:03:11.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:11 smithi064 conmon[107122]: cluster 2023-06-10T21:03:10.440715+0000 mgr.x (mgr 2023-06-10T21:03:11.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:11 smithi064 conmon[107122]: .24857) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-10T21:03:11.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:11 smithi064 conmon[97224]: cluster 2023-06-10T21:03:10.440715+0000 mgr.x ( 2023-06-10T21:03:11.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:11 smithi064 conmon[97224]: mgr.24857) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-10T21:03:12.444 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:12 smithi152 conmon[132019]: ts=2023-06-10T21:03:12.313Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.64:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:12.830 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:12 smithi152 conmon[97365]: audit 2023-06-10T21:03:11.695541+0000 mon.c (mon.2 2023-06-10T21:03:12.831 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:12 smithi152 conmon[97365]: ) 60 : audit [DBG] from='client.? 172.21.15.64:0/264039401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:12.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:12 smithi064 conmon[97224]: audit 2023-06-10T21:03:11.695541 2023-06-10T21:03:12.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:12 smithi064 conmon[97224]: +0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.64:0/264039401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:12.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:12 smithi064 conmon[107122]: audit 2023-06-10T21:03:11.695541+0000 mon.c 2023-06-10T21:03:12.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:12 smithi064 conmon[107122]: (mon.2) 60 : audit [DBG] from='client.? 172.21.15.64:0/264039401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:13.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:12 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:12] "GET /metrics HTTP/1.1" 200 190019 "" "Prometheus/2.33.4" 2023-06-10T21:03:13.195 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:12 smithi152 systemd[1]: Stopping Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:03:13.196 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:12 smithi152 bash[140160]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-prometheus.a found: no such container 2023-06-10T21:03:13.759 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:03:13.759 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-10T21:03:13.759 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-10T21:03:13.759 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.480Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.482Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-10T21:03:13.760 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.482Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-10T21:03:13.761 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[132019]: ts=2023-06-10T21:03:13.482Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-10T21:03:13.761 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:13 smithi152 conmon[97365]: cluster 2023-06-10T21:03:12.441461+0000 mgr.x (mgr.24857) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-10T21:03:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:13 smithi064 conmon[97224]: cluster 2023-06-10T21:03:12.441461+0000 mgr.x (mgr. 2023-06-10T21:03:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:13 smithi064 conmon[97224]: 24857) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-10T21:03:13.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:13 smithi064 conmon[107122]: cluster 2023-06-10T21:03:12.441461+0000 2023-06-10T21:03:13.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:13 smithi064 conmon[107122]: mgr.x (mgr.24857) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-10T21:03:14.198 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:13 smithi152 bash[140266]: a4c5e6078a1577f28577b11f6a68a62df38b211ea20ef502995fae1c1a8ce5be 2023-06-10T21:03:14.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:14 smithi064 conmon[107122]: audit 2023-06-10T21:03:14.319796+0000 mon.a (mon.0 2023-06-10T21:03:14.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:14 smithi064 conmon[107122]: ) 841 : audit [DBG] from='client.? 172.21.15.64:0/1528585577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:14 smithi064 conmon[97224]: audit 2023-06-10T21:03:14.319796+0000 mon.a (mon.0) 841 : audit [DBG] from='client.? 172.21.15.64:0/1528585577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:14 smithi152 conmon[97365]: audit 2023-06-10T21:03 2023-06-10T21:03:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:14 smithi152 conmon[97365]: :14.319796+0000 mon.a (mon.0) 841 : audit [DBG] from='client.? 172.21.15.64:0/1528585577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:14.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:14 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-10T21:03:14.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:14 smithi152 systemd[1]: Stopped Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:03:14.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:14 smithi152 systemd[1]: Starting Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:03:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:15 smithi064 conmon[107122]: cluster 2023-06-10T21:03:14. 2023-06-10T21:03:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:15 smithi064 conmon[107122]: 442983+0000 mgr.x (mgr.24857) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:15.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:15 smithi064 conmon[97224]: cluster 2023-06-10T21:03:14.442983+0000 mgr.x 2023-06-10T21:03:15.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:15 smithi064 conmon[97224]: (mgr.24857) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[97365]: cluster 2023-06-10T21:03:14.442983+0000 mgr.x ( 2023-06-10T21:03:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[97365]: mgr.24857) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 bash[140578]: ae230a4afbd61866b53a0bebde758aa04e8130051a2dc0281dfecb6de30eb699 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.515Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.515Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.515Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi152 (none))" 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.515Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-10T21:03:15.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.515Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 systemd[1]: Started Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.524Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.525Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.528Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.529Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.529Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.531µs 2023-06-10T21:03:15.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.529Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-10T21:03:16.236 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:15 smithi152 conmon[140617]: ts=2023-06-10T21:03:15.949Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-06-10T21:03:16.238 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:16 smithi152 systemd[1]: Stopping Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:03:16.582 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:16 smithi152 bash[140762]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-node-exporter.b found: no such container 2023-06-10T21:03:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: audit 2023-06-10T21:03:15.579060 2023-06-10T21:03:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: +0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: audit 2023-06-10T21:03:15.589992+0000 mon.a 2023-06-10T21:03:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: (mon.0) 843 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: cephadm 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: 2023-06-10T21:03:15.635753+0000 mgr.x (mgr 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: .24857) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: 06-10T21:03:15.636392+0000 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: mgr.x (mgr.24857) 52 : cephadm [INF] 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[107122]: Reconfiguring daemon node-exporter.b on smithi152 2023-06-10T21:03:16.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: audit 2023-06-10T21:03:15.579060+0000 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: mon.a (mon.0) 842 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: audit 2023-06-10T21:03:15.589992+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: cephadm 2023-06-10T21:03:15 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: .635753+0000 mgr.x (mgr.24857) 51 : cephadm 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: cephadm 2023-06-10T21:03: 2023-06-10T21:03:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: 15.636392+0000 mgr.x (mgr.24857) 52 2023-06-10T21:03:16.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:16 smithi064 conmon[97224]: : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi152 2023-06-10T21:03:16.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.575Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-06-10T21:03:16.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.576Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.576Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=44.547µs wal_replay_duration=1.047126862s total_replay_duration=1.047191011s 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.582Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.582Z caller=main.go:947 level=info msg="TSDB started" 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.582Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.605Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.567542ms db_storage=1.166µs remote_storage=1.8µs web_handler=454ns query_engine=1.288µs scrape=622.494µs scrape_sd=117.366µs notify=18.189µs notify_sd=13.399µs rules=22.336275ms 2023-06-10T21:03:16.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[140617]: ts=2023-06-10T21:03:16.606Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-10T21:03:16.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[97365]: audit 2023-06-10T21:03:15.579060+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[97365]: audit 2023-06-10T21:03:15.589992+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:16.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[97365]: cephadm 2023-06-10T21:03:15.635753+0000 mgr.x (mgr.24857) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-10T21:03:16.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:16 smithi152 conmon[97365]: cephadm 2023-06-10T21:03:15.636392+0000 mgr.x (mgr.24857) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi152 2023-06-10T21:03:17.446 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:17 smithi152 bash[140869]: afdc33042e265e15f64b27d5525685d6d170c3ca5de4a07fc8bf795d067b11bf 2023-06-10T21:03:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:17 smithi064 conmon[97224]: cluster 2023-06-10T21:03:16.443425+0000 mgr.x (mgr.24857) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:17 smithi064 conmon[97224]: audit 2023-06-10T21:03:16.960508+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.64:0/2426510333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:17.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:17 smithi064 conmon[107122]: cluster 2023-06-10T21:03:16.443425+0000 mgr.x (mgr.24857 2023-06-10T21:03:17.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:17 smithi064 conmon[107122]: ) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:17.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:17 smithi064 conmon[107122]: audit 2023-06-10T21:03:16.960508+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.64:0/2426510333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:17.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:17 smithi152 conmon[97365]: cluster 2023-06-10T21:03:16.443425+0000 mgr.x (mgr.24857) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-10T21:03:17.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:17 smithi152 conmon[97365]: audit 2023-06-10T21:03:16.960508+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.64:0/2426510333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:18.444 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:17 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:03:18.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:17 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.b.service: Failed with result 'exit-code'. 2023-06-10T21:03:18.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:17 smithi152 systemd[1]: Stopped Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:03:18.445 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:17 smithi152 systemd[1]: Starting Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:03:19.197 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.931Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.931Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.931Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.931Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=arp 2023-06-10T21:03:19.198 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-10T21:03:19.199 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=edac 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-10T21:03:19.200 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-10T21:03:19.201 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=os 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.932Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-10T21:03:19.202 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=stat 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=time 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=uname 2023-06-10T21:03:19.203 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 conmon[141219]: ts=2023-06-10T21:03:18.933Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 bash[141182]: 91ff512a05b1ce3d4218a5678d8c3eff0b41087521eb9896fbf2000792223583 2023-06-10T21:03:19.204 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:03:18 smithi152 systemd[1]: Started Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:03:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[97224]: cluster 2023-06-10T21:03:18.443927+0000 mgr.x (mgr. 2023-06-10T21:03:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[97224]: 24857) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-06-10T21:03:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[97224]: audit 2023-06-10T21:03:19.025012+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[97224]: audit 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[97224]: 2023-06-10T21:03:19.049283+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: cluster 2023-06-10T21:03: 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: 18.443927+0000 mgr.x (mgr.24857) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: audit 2023-06-10T21:03:19.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: 2023-06-10T21:03:19.025012+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:19.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: 2023-06-10T21:03:19.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: audit 2023-06-10T21:03:19.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: 2023-06-10T21:03:19.049283 2023-06-10T21:03:19.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: +0000 mon.a (mon.0) 845 : audit 2023-06-10T21:03:19.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:19 smithi064 conmon[107122]: [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:19 smithi152 conmon[97365]: cluster 2023-06-10T21:03:18.443927+0000 mgr.x 2023-06-10T21:03:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:19 smithi152 conmon[97365]: (mgr.24857) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s 2023-06-10T21:03:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:19 smithi152 conmon[97365]: audit 2023-06-10T21:03:19.025012+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:19 smithi152 conmon[97365]: audit 2023-06-10T21:03:19.049283+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:03:20.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[97224]: cephadm 2023-06-10T21:03:19.102942+0000 mgr.x (mgr.24857) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-10T21:03:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[97224]: cephadm 2023-06-10T21:03:19.121147+0000 mgr.x (mgr.24857) 56 : cephadm [INF] Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:03:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[97224]: audit 2023-06-10T21:03:19.610862+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.64:0/2203419332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: cephadm 2023-06-10T21:03: 2023-06-10T21:03:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: 19.102942+0000 mgr.x (mgr.24857) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: cephadm 2023-06 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: -10T21:03:19.121147+0000 mgr.x 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: (mgr.24857) 56 : cephadm [INF] 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: T21:03:19.610862+0000 mon.a ( 2023-06-10T21:03:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: mon.0) 846 : audit [DBG] from='client.? 172.21.15.64:0/2203419332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:20 smithi064 conmon[107122]: 2023-06-10T21:03:20.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:20 smithi152 conmon[97365]: cephadm 2023-06-10T21:03:19.102942+0000 mgr.x (mgr.24857) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-10T21:03:20.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:20 smithi152 conmon[97365]: cephadm 2023-06-10T21:03:19.121147+0000 mgr.x (mgr.24857) 56 : cephadm [INF] Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:03:20.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:20 smithi152 conmon[97365]: audit 2023-06-10T21:03:19.610862+0000 mon.a (mon.0) 846 : audit [DBG] from='client.? 172.21.15.64:0/2203419332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:21.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[97224]: cluster 2023-06-10T21:03:20.446265+0000 2023-06-10T21:03:21.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[97224]: mgr.x (mgr.24857) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-10T21:03:21.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[97224]: audit 2023-06-10T21:03:20.601222+0000 mon.b (mon.1 2023-06-10T21:03:21.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[97224]: ) 81 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:21.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[107122]: cluster 2023-06-10T21:03:20.446265+0000 mgr.x (mgr.24857) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-10T21:03:21.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:21 smithi064 conmon[107122]: audit 2023-06-10T21:03:20.601222+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:21.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:21 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:21] "GET /metrics HTTP/1.1" 200 190031 "" "Prometheus/2.33.4" 2023-06-10T21:03:21.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:21 smithi152 conmon[97365]: cluster 2023-06-10T21:03:20.446265+0000 mgr.x ( 2023-06-10T21:03:21.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:21 smithi152 conmon[97365]: mgr.24857) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-10T21:03:21.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:21 smithi152 conmon[97365]: audit 2023-06-10T21:03:20.601222+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:22.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:22 smithi064 conmon[97224]: audit 2023-06-10T21:03:22.248433+0000 mon.a (mon.0 2023-06-10T21:03:22.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:22 smithi064 conmon[97224]: ) 847 : audit [DBG] from='client.? 172.21.15.64:0/165071028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:22.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:22 smithi064 conmon[107122]: audit 2023-06-10T21:03:22.248433+0000 2023-06-10T21:03:22.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:22 smithi064 conmon[107122]: mon.a (mon.0) 847 : audit [DBG] from='client.? 172.21.15.64:0/165071028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:22 smithi152 conmon[97365]: audit 2023-06-10T21:03:22.248433+0000 mon.a (mon.0) 847 : audit 2023-06-10T21:03:22.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:22 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/165071028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:23 smithi064 conmon[107122]: cluster 2023-06-10T21:03:22.448409+0000 mgr.x (mgr.24857) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-06-10T21:03:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:23 smithi064 conmon[97224]: cluster 2023-06-10T21:03:22. 2023-06-10T21:03:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:23 smithi064 conmon[97224]: 448409+0000 mgr.x (mgr.24857) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-06-10T21:03:23.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:23 smithi152 conmon[97365]: cluster 2023-06-10T21:03:22.448409+0000 mgr.x (mgr.24857) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2023-06-10T21:03:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:25 smithi064 conmon[97224]: cluster 2023-06-10T21:03:24.448783+0000 mgr.x (mgr.24857) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:25 smithi064 conmon[97224]: audit 2023-06-10T21:03:24.918255+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.64:0/524939370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:25 smithi064 conmon[107122]: cluster 2023-06-10T21:03:24.448783+0000 mgr.x (mgr.24857) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:25 smithi064 conmon[107122]: audit 2023-06-10T21:03:24.918255+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.64:0/524939370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:25 smithi152 conmon[97365]: cluster 2023-06-10T21:03:24.448783+0000 mgr.x (mgr. 2023-06-10T21:03:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:25 smithi152 conmon[97365]: 24857) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:25 smithi152 conmon[97365]: audit 2023-06-10T21:03:24.918255+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.64:0/524939370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:27.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:27 smithi064 conmon[97224]: cluster 2023-06-10T21:03:26. 2023-06-10T21:03:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:27 smithi064 conmon[97224]: 449178+0000 mgr.x (mgr.24857) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:27 smithi064 conmon[107122]: cluster 2023-06-10T21:03:26 2023-06-10T21:03:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:27 smithi064 conmon[107122]: .449178+0000 mgr.x (mgr.24857) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:27 smithi152 conmon[97365]: cluster 2023-06-10T21:03:26.449178+0000 mgr.x 2023-06-10T21:03:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:27 smithi152 conmon[97365]: (mgr.24857) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:03:28.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:28 smithi064 conmon[97224]: audit 2023-06-10T21:03:27.550875+0000 2023-06-10T21:03:28.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:28 smithi064 conmon[97224]: mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.64:0/4158829626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:28 smithi064 conmon[107122]: audit 2023-06-10T21:03:27.550875+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.64:0/4158829626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:28 smithi152 conmon[97365]: audit 2023-06-10T21:03:27.550875+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.64:0/4158829626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:28.952 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:28 smithi152 conmon[140617]: ts=2023-06-10T21:03:28.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:28.953 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:28 smithi152 conmon[140617]: ts=2023-06-10T21:03:28.665Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:29 smithi064 conmon[97224]: cluster 2023-06-10T21:03:28.449816+0000 mgr.x (mgr.24857) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-10T21:03:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:29 smithi064 conmon[107122]: cluster 2023-06-10T21:03:28. 2023-06-10T21:03:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:29 smithi064 conmon[107122]: 449816+0000 mgr.x (mgr.24857) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-10T21:03:29.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:29 smithi152 conmon[97365]: cluster 2023-06-10T21:03:28.449816+0000 mgr.x ( 2023-06-10T21:03:29.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:29 smithi152 conmon[97365]: mgr.24857) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-10T21:03:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:30 smithi064 conmon[107122]: audit 2023-06-10T21:03:30.200392+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.64:0/3221009290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:30 smithi064 conmon[97224]: audit 2023-06-10T21:03:30.200392 2023-06-10T21:03:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:30 smithi064 conmon[97224]: +0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.64:0/3221009290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:30.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:30 smithi152 conmon[97365]: audit 2023-06-10T21:03:30.200392+0000 mon.a (mon.0 2023-06-10T21:03:30.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:30 smithi152 conmon[97365]: ) 849 : audit [DBG] from='client.? 172.21.15.64:0/3221009290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:31 smithi064 conmon[97224]: cluster 2023-06-10T21:03:30.455352+0000 2023-06-10T21:03:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:31 smithi064 conmon[97224]: mgr.x (mgr.24857) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-10T21:03:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:31 smithi064 conmon[107122]: cluster 2023-06-10T21:03:30.455352 2023-06-10T21:03:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:31 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-10T21:03:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:31 smithi152 conmon[97365]: cluster 2023-06-10T21:03:30.455352+0000 mgr.x (mgr.24857) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-10T21:03:32.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:31 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:31] "GET /metrics HTTP/1.1" 200 190031 "" "Prometheus/2.33.4" 2023-06-10T21:03:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[97224]: cluster 2023-06-10T21:03:32.455993+0000 mgr.x ( 2023-06-10T21:03:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[97224]: mgr.24857) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-06-10T21:03:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[97224]: audit 2023-06-10T21:03:32.822543+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.64:0/3548473616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:33.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[107122]: cluster 2023-06-10T21:03: 2023-06-10T21:03:33.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[107122]: 32.455993+0000 mgr.x (mgr.24857) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-06-10T21:03:33.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[107122]: audit 2023-06-10T21:03:32. 2023-06-10T21:03:33.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[107122]: 822543+0000 mon.c (mon.2) 63 : audit 2023-06-10T21:03:33.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:33 smithi064 conmon[107122]: [DBG] from='client.? 172.21.15.64:0/3548473616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:33.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:33 smithi152 conmon[97365]: cluster 2023-06-10T21:03:32.455993+0000 mgr.x 2023-06-10T21:03:33.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:33 smithi152 conmon[97365]: (mgr.24857) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2023-06-10T21:03:33.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:33 smithi152 conmon[97365]: audit 2023-06-10T21:03:32.822543+0000 mon.c (mon.2) 63 : audit [DBG] from='client.? 172.21.15.64:0/3548473616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[107122]: cluster 2023-06-10T21:03:34 2023-06-10T21:03:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[107122]: .456332+0000 mgr.x (mgr.24857) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-06-10T21:03:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[107122]: audit 2023-06-10T21:03:35.462536+0000 2023-06-10T21:03:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[107122]: mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.64:0/357434531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[97224]: cluster 2023-06-10T21:03:34.456332+0000 mgr.x (mgr.24857) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-06-10T21:03:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[97224]: audit 2023-06-10T21:03:35.462536+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.64:0/357434531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:35 smithi064 conmon[97224]: 2023-06-10T21:03:35.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:35 smithi152 conmon[97365]: cluster 2023-06-10T21:03:34.456332+0000 mgr.x (mgr.24857) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-06-10T21:03:35.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:35 smithi152 conmon[97365]: audit 2023-06-10T21:03:35.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:35 smithi152 conmon[97365]: 2023-06-10T21:03:35.462536 2023-06-10T21:03:35.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:35 smithi152 conmon[97365]: +0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.64:0/357434531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:36 smithi064 conmon[107122]: audit 2023-06-10T21:03:35.601986+0000 mon.b (mon.1 2023-06-10T21:03:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:36 smithi064 conmon[107122]: ) 82 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:36 smithi064 conmon[97224]: audit 2023-06-10T21:03:35.601986 2023-06-10T21:03:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:36 smithi064 conmon[97224]: +0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:36 smithi152 conmon[97365]: audit 2023-06-10T21:03:35.601986+0000 mon.b (mon 2023-06-10T21:03:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:36 smithi152 conmon[97365]: .1) 82 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:37 smithi064 conmon[107122]: cluster 2023-06-10T21:03:36.460303+0000 mgr.x (mgr.24857) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-06-10T21:03:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:37 smithi064 conmon[97224]: cluster 2023-06-10T21:03:36.460303+0000 2023-06-10T21:03:37.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:37 smithi064 conmon[97224]: mgr.x (mgr.24857) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-06-10T21:03:37.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:37 smithi152 conmon[97365]: cluster 2023-06-10T21:03:36.460303+0000 mgr.x (mgr.24857) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-06-10T21:03:38.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:38 smithi064 conmon[97224]: audit 2023-06-10T21:03:38.089942+0000 2023-06-10T21:03:38.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:38 smithi064 conmon[97224]: mon.a (mon.0) 851 : audit [DBG] from='client.? 172.21.15.64:0/1074865334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:38.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:38 smithi064 conmon[107122]: audit 2023-06-10T21:03:38.089942+0000 mon.a (mon.0) 851 : audit [DBG] from='client.? 172.21.15.64:0/1074865334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:38.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:38 smithi152 conmon[140617]: ts=2023-06-10T21:03:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:38.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:38 smithi152 conmon[140617]: ts=2023-06-10T21:03:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:38 smithi152 conmon[97365]: audit 2023-06-10T21:03:38.089942+0000 mon.a (mon 2023-06-10T21:03:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:38 smithi152 conmon[97365]: .0) 851 : audit [DBG] from='client.? 172.21.15.64:0/1074865334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:39 smithi064 conmon[97224]: cluster 2023-06-10T21:03:38.460934+0000 mgr.x (mgr.24857) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2023-06-10T21:03:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:39 smithi064 conmon[107122]: cluster 2023-06-10T21:03:38.460934+0000 mgr.x (mgr.24857) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2023-06-10T21:03:39.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:39 smithi152 conmon[97365]: cluster 2023-06-10T21:03:38.460934+0000 mgr.x ( 2023-06-10T21:03:39.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:39 smithi152 conmon[97365]: mgr.24857) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2023-06-10T21:03:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[97224]: cluster 2023-06-10T21:03:40.462349+0000 mgr.x (mgr.24857) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-06-10T21:03:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[97224]: audit 2023-06-10T21:03:40.747620+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.64:0/100844874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[107122]: cluster 2023-06-10T21:03:40.462349+0000 2023-06-10T21:03:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[107122]: mgr.x (mgr.24857) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-06-10T21:03:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:03:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[107122]: 06-10T21:03:40.747620+0000 mon.c (mon.2) 64 : audit [DBG] 2023-06-10T21:03:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:41 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/100844874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:41.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:41 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:41] "GET /metrics HTTP/1.1" 200 190024 "" "Prometheus/2.33.4" 2023-06-10T21:03:41.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:41 smithi152 conmon[97365]: cluster 2023-06-10T21:03:40.462349+0000 mgr.x ( 2023-06-10T21:03:41.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:41 smithi152 conmon[97365]: mgr.24857) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-06-10T21:03:41.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:41 smithi152 conmon[97365]: audit 2023-06-10T21:03:40.747620+0000 mon.c (mon.2) 64 : audit [DBG] from='client.? 172.21.15.64:0/100844874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[97224]: cluster 2023-06-10T21:03:42.463022+0000 mgr.x (mgr.24857) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-06-10T21:03:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[97224]: audit 2023-06-10T21:03:43.372439+0000 mon.b (mon.1 2023-06-10T21:03:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[97224]: ) 83 : audit [DBG] from='client.? 172.21.15.64:0/3192347316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[107122]: cluster 2023-06-10T21:03:42.463022 2023-06-10T21:03:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-06-10T21:03:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[107122]: audit 2023-06-10T21:03:43.372439+0000 2023-06-10T21:03:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:43 smithi064 conmon[107122]: mon.b (mon.1) 83 : audit [DBG] from='client.? 172.21.15.64:0/3192347316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:43.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:43 smithi152 conmon[97365]: cluster 2023-06-10T21:03:42.463022+0000 mgr.x (mgr.24857 2023-06-10T21:03:43.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:43 smithi152 conmon[97365]: ) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2023-06-10T21:03:43.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:43 smithi152 conmon[97365]: audit 2023-06-10T21:03:43.372439+0000 mon.b (mon.1) 83 : audit [DBG] from='client.? 172.21.15.64:0/3192347316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:45.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:45 smithi064 conmon[97224]: cluster 2023-06-10T21:03:44.463444+0000 mgr.x (mgr 2023-06-10T21:03:45.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:45 smithi064 conmon[97224]: .24857) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:45 smithi064 conmon[107122]: cluster 2023-06-10T21:03:44.463444+0000 mgr.x (mgr.24857) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:45 smithi152 conmon[97365]: cluster 2023-06-10T21:03:44.463444+0000 mgr.x (mgr.24857) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:46.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:46 smithi064 conmon[107122]: audit 2023-06-10T21:03:46.001010+0000 2023-06-10T21:03:46.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:46 smithi064 conmon[107122]: mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.64:0/1615737798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:46 smithi064 conmon[97224]: audit 2023-06-10T21:03:46.001010 2023-06-10T21:03:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:46 smithi064 conmon[97224]: +0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.64:0/1615737798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:46 smithi152 conmon[97365]: audit 2023-06-10T21:03:46.001010+0000 mon.a (mon.0) 852 : audit [DBG] from='client.? 172.21.15.64:0/1615737798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:47.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:47 smithi064 conmon[107122]: cluster 2023-06-10T21:03:46.463813 2023-06-10T21:03:47.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:47 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:47 smithi064 conmon[97224]: cluster 2023-06-10T21:03:46.463813+0000 mgr.x ( 2023-06-10T21:03:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:47 smithi064 conmon[97224]: mgr.24857) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:47 smithi152 conmon[97365]: cluster 2023-06-10T21:03:46.463813+0000 mgr.x (mgr.24857) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-10T21:03:48.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:48 smithi152 conmon[140617]: ts=2023-06-10T21:03:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:48.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:48 smithi152 conmon[140617]: ts=2023-06-10T21:03:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:49.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[107122]: cluster 2023-06-10T21:03:48 2023-06-10T21:03:49.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[107122]: .464452+0000 mgr.x (mgr.24857) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-06-10T21:03:49.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:03:49.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[107122]: 21:03:48.627928+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.64:0/395285827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:49.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[97224]: cluster 2023-06-10T21:03:48.464452+0000 mgr.x 2023-06-10T21:03:49.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[97224]: (mgr.24857) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-06-10T21:03:49.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:49 smithi064 conmon[97224]: audit 2023-06-10T21:03:48.627928+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.64:0/395285827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:49.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:49 smithi152 conmon[97365]: cluster 2023-06-10T21:03:48.464452+0000 mgr.x 2023-06-10T21:03:49.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:49 smithi152 conmon[97365]: (mgr.24857) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 939 op/s 2023-06-10T21:03:49.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:49 smithi152 conmon[97365]: audit 2023-06-10T21:03:48.627928+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.64:0/395285827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:50.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:50 smithi064 conmon[97224]: audit 2023-06-10T21:03:50.611835+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:50.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:50 smithi064 conmon[107122]: audit 2023-06-10T21:03:50 2023-06-10T21:03:50.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:50 smithi064 conmon[107122]: .611835+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:50.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:50 smithi152 conmon[97365]: audit 2023-06-10T21:03:50.611835+0000 mon.b 2023-06-10T21:03:50.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:50 smithi152 conmon[97365]: (mon.1) 84 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:03:51.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:03:51 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:03:51] "GET /metrics HTTP/1.1" 200 190052 "" "Prometheus/2.33.4" 2023-06-10T21:03:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:51 smithi152 conmon[97365]: cluster 2023-06-10T21:03:50.464871+0000 mgr.x ( 2023-06-10T21:03:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:51 smithi152 conmon[97365]: mgr.24857) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-10T21:03:51.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:51 smithi152 conmon[97365]: audit 2023-06-10T21:03:51.233660+0000 mon.a (mon.0) 854 : audit [DBG] from='client.? 172.21.15.64:0/687346986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:52.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[97224]: cluster 2023-06-10T21:03:50.464871+0000 mgr.x 2023-06-10T21:03:52.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[97224]: (mgr.24857) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-10T21:03:52.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[97224]: audit 2023-06-10T21:03:51.233660+0000 mon.a (mon.0) 2023-06-10T21:03:52.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[97224]: 854 : audit [DBG] from='client.? 172.21.15.64:0/687346986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:52.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[107122]: cluster 2023-06-10T21:03:50 2023-06-10T21:03:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[107122]: .464871+0000 mgr.x (mgr.24857) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-10T21:03:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:03:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[107122]: -10T21:03:51.233660+0000 mon.a (mon.0) 854 : audit [DBG] 2023-06-10T21:03:52.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:51 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/687346986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:53.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:52 smithi064 conmon[107122]: cluster 2023-06-10T21:03:51.656740+0000 mon.a ( 2023-06-10T21:03:53.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:52 smithi064 conmon[107122]: mon.0) 855 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-06-10T21:03:53.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:52 smithi064 conmon[97224]: cluster 2023-06-10T21:03:51.656740+0000 2023-06-10T21:03:53.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:52 smithi064 conmon[97224]: mon.a (mon.0) 855 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-06-10T21:03:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:52 smithi152 conmon[97365]: cluster 2023-06-10T21:03:51.656740+0000 mon.a (mon.0) 855 : cluster [DBG] mgrmap e26: x(active, since 91s), standbys: y 2023-06-10T21:03:54.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:53 smithi064 conmon[107122]: cluster 2023-06-10T21:03:52.465533+0000 mgr.x ( 2023-06-10T21:03:54.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:53 smithi064 conmon[107122]: mgr.24857) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 913 op/s 2023-06-10T21:03:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:53 smithi064 conmon[97224]: cluster 2023-06-10T21:03:52.465533+0000 2023-06-10T21:03:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:53 smithi064 conmon[97224]: mgr.x (mgr.24857) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 913 op/s 2023-06-10T21:03:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:53 smithi152 conmon[97365]: cluster 2023-06-10T21:03:52.465533+0000 mgr.x ( 2023-06-10T21:03:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:53 smithi152 conmon[97365]: mgr.24857) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 913 op/s 2023-06-10T21:03:55.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:54 smithi064 conmon[107122]: audit 2023-06-10T21:03:53.864817 2023-06-10T21:03:55.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:54 smithi064 conmon[107122]: +0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.64:0/3174367502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:55.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:54 smithi064 conmon[97224]: audit 2023-06-10T21:03:53.864817+0000 2023-06-10T21:03:55.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:54 smithi064 conmon[97224]: mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.64:0/3174367502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:55.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:54 smithi152 conmon[97365]: audit 2023-06-10T21:03:53.864817+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.64:0/3174367502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:56.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:55 smithi064 conmon[107122]: cluster 2023-06-10T21:03:54 2023-06-10T21:03:56.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:55 smithi064 conmon[107122]: .468626+0000 mgr.x (mgr.24857) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:56.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:55 smithi064 conmon[97224]: cluster 2023-06-10T21:03:54.468626+0000 mgr.x ( 2023-06-10T21:03:56.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:55 smithi064 conmon[97224]: mgr.24857) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:56.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:55 smithi152 conmon[97365]: cluster 2023-06-10T21:03:54.468626+0000 2023-06-10T21:03:56.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:55 smithi152 conmon[97365]: mgr.x (mgr.24857) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:57 smithi064 conmon[97224]: audit 2023-06-10T21:03:56.447083+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.64:0/3807902215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:57 smithi064 conmon[97224]: cluster 2023-06-10T21:03:56.469044+0000 mgr.x (mgr.24857) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:57 smithi064 conmon[107122]: audit 2023-06-10T21:03:56.447083+0000 mon.c ( 2023-06-10T21:03:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:57 smithi064 conmon[107122]: mon.2) 66 : audit [DBG] from='client.? 172.21.15.64:0/3807902215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:57 smithi064 conmon[107122]: cluster 2023-06-10T21:03:56.469044+0000 mgr.x (mgr.24857) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:57.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:57 smithi152 conmon[97365]: audit 2023-06-10T21:03:56.447083+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.64:0/3807902215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:57.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:57 smithi152 conmon[97365]: cluster 2023-06-10T21:03:56.469044+0000 mgr.x (mgr.24857) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 597 op/s 2023-06-10T21:03:58.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:58 smithi152 conmon[140617]: ts=2023-06-10T21:03:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:58.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:03:58 smithi152 conmon[140617]: ts=2023-06-10T21:03:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:03:59.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[107122]: cluster 2023-06-10T21:03:58.473173+0000 mgr.x (mgr.24857) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 873 op/s 2023-06-10T21:03:59.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[107122]: audit 2023-06-10T21:03:59.075262+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.64:0/487167427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:59.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[97224]: cluster 2023-06-10T21: 2023-06-10T21:03:59.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[97224]: 03:58.473173+0000 mgr.x (mgr.24857) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 873 op/s 2023-06-10T21:03:59.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[97224]: audit 2023-06-10T21:03:59. 2023-06-10T21:03:59.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:03:59 smithi064 conmon[97224]: 075262+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.64:0/487167427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:03:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:59 smithi152 conmon[97365]: cluster 2023-06-10T21:03:58.473173+0000 mgr.x (mgr 2023-06-10T21:03:59.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:59 smithi152 conmon[97365]: .24857) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 873 op/s 2023-06-10T21:03:59.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:03:59 smithi152 conmon[97365]: audit 2023-06-10T21:03:59.075262+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.64:0/487167427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:01 smithi064 conmon[107122]: cluster 2023-06-10T21:04:00.473520 2023-06-10T21:04:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:01 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-10T21:04:01.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:01 smithi064 conmon[97224]: cluster 2023-06-10T21:04:00.473520+0000 2023-06-10T21:04:01.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:01 smithi064 conmon[97224]: mgr.x (mgr.24857) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-10T21:04:01.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:01 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:04:01] "GET /metrics HTTP/1.1" 200 190052 "" "Prometheus/2.33.4" 2023-06-10T21:04:01.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:01 smithi152 conmon[97365]: cluster 2023-06-10T21:04:00.473520+0000 mgr.x ( 2023-06-10T21:04:01.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:01 smithi152 conmon[97365]: mgr.24857) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-10T21:04:02.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:02 smithi064 conmon[107122]: audit 2023-06-10T21:04:01.693903+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.64:0/1958696744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:02.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:02 smithi064 conmon[97224]: audit 2023-06-10T21:04:01.693903+0000 2023-06-10T21:04:02.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:02 smithi064 conmon[97224]: mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.64:0/1958696744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:02 smithi152 conmon[97365]: audit 2023-06-10T21:04:01.693903+0000 mon.c (mon.2) 68 2023-06-10T21:04:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:02 smithi152 conmon[97365]: : audit [DBG] from='client.? 172.21.15.64:0/1958696744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:03.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:03 smithi064 conmon[97224]: cluster 2023-06-10T21:04:02.474191+0000 mgr.x (mgr.24857) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 255 B/s wr, 883 op/s 2023-06-10T21:04:03.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:03 smithi064 conmon[107122]: cluster 2023-06-10T21:04:02.474191+0000 mgr.x (mgr.24857 2023-06-10T21:04:03.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:03 smithi064 conmon[107122]: ) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 255 B/s wr, 883 op/s 2023-06-10T21:04:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:03 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T21:04:03.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:03 smithi152 conmon[97365]: 21:04:02.474191+0000 mgr.x (mgr.24857) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 255 B/s wr, 883 op/s 2023-06-10T21:04:04.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:04 smithi064 conmon[107122]: audit 2023-06-10T21:04:04.320838 2023-06-10T21:04:04.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:04 smithi064 conmon[107122]: +0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.64:0/4187002775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:04.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:04 smithi064 conmon[97224]: audit 2023-06-10T21:04:04.320838+0000 mon.a 2023-06-10T21:04:04.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:04 smithi064 conmon[97224]: (mon.0) 856 : audit [DBG] from='client.? 172.21.15.64:0/4187002775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:04 smithi152 conmon[97365]: audit 2023-06-10T21:04:04.320838+0000 mon.a (mon. 2023-06-10T21:04:04.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:04 smithi152 conmon[97365]: 0) 856 : audit [DBG] from='client.? 172.21.15.64:0/4187002775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:05 smithi064 conmon[107122]: cluster 2023-06-10T21:04:04.474575 2023-06-10T21:04:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:05 smithi064 conmon[107122]: +0000 mgr.x (mgr.24857) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:05.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:05 smithi064 conmon[97224]: cluster 2023-06-10T21:04:04.474575+0000 2023-06-10T21:04:05.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:05 smithi064 conmon[97224]: mgr.x (mgr.24857) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:05.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:05 smithi152 conmon[97365]: cluster 2023-06-10T21:04:04.474575+0000 mgr.x (mgr.24857) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:06.592 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:06 smithi152 conmon[97365]: audit 2023-06-10T21:04:05.612244+0000 2023-06-10T21:04:06.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:06 smithi152 conmon[97365]: mon.b (mon.1) 85 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:06.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:06 smithi064 conmon[107122]: audit 2023-06-10T21:04:05.612244+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:06.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:06 smithi064 conmon[97224]: audit 2023-06-10T21:04:05.612244+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:07.695 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:07 smithi152 systemd[1]: Stopping Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:07.696 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:07 smithi152 bash[141617]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-grafana.a found: no such container 2023-06-10T21:04:07.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:07 smithi152 conmon[97365]: cluster 2023-06-10T21:04:06.475176+0000 mgr.x (mgr.24857) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:07.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:07 smithi152 conmon[97365]: audit 2023-06-10T21:04:06.925209+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.64:0/986454872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:07.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[107122]: cluster 2023-06-10T21:04:06.475176+0000 2023-06-10T21:04:07.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[107122]: mgr.x (mgr.24857) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:07.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[107122]: audit 2023-06-10T21:04:06.925209+0000 mon.a ( 2023-06-10T21:04:07.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[107122]: mon.0) 857 : audit [DBG] from='client.? 172.21.15.64:0/986454872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:07.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[97224]: cluster 2023-06-10T21:04:06.475176+0000 mgr.x (mgr.24857) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-10T21:04:07.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[97224]: audit 2023-06-10T21:04:07.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:07 smithi064 conmon[97224]: 2023-06-10T21:04:06.925209+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.64:0/986454872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:08.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:08 smithi152 conmon[128666]: t=2023-06-10T21:04:08+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-06-10T21:04:08.422 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:08 smithi152 bash[141722]: 9249018f5990b07027ecd762167d7b33417c77c9c80f57fa8ead13094e79c461 2023-06-10T21:04:08.670 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:08 smithi152 conmon[140617]: ts=2023-06-10T21:04:08.665Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:09.598 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:08 smithi152 conmon[140617]: ts=2023-06-10T21:04:08.666Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:09.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:09 smithi152 conmon[97365]: cluster 2023-06-10T21:04:08.481518+0000 mgr.x 2023-06-10T21:04:09.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:09 smithi152 conmon[97365]: (mgr.24857) 81 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-06-10T21:04:09.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:09 smithi152 conmon[97365]: audit 2023-06-10T21:04:09.539006+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.64:0/1331667886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:09.599 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:09 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@grafana.a.service: Succeeded. 2023-06-10T21:04:09.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:09 smithi152 systemd[1]: Stopped Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:09.600 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:09 smithi152 systemd[1]: Starting Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[107122]: cluster 2023-06-10T21: 2023-06-10T21:04:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[107122]: 04:08.481518+0000 mgr.x (mgr.24857) 81 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-06-10T21:04:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[107122]: audit 2023-06-10T21:04:09.539006+0000 mon.c (mon. 2023-06-10T21:04:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[107122]: 2) 69 : audit [DBG] from='client.? 172.21.15.64:0/1331667886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:09.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[97224]: cluster 2023-06-10T21:04:08.481518 2023-06-10T21:04:09.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 81 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2023-06-10T21:04:09.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[97224]: audit 2023-06-10T21:04:09.539006+0000 mon.c ( 2023-06-10T21:04:09.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:09 smithi064 conmon[97224]: mon.2) 69 : audit [DBG] from='client.? 172.21.15.64:0/1331667886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:10.562 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STOPPING 2023-06-10T21:04:10.563 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:04:10.563 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STOPPED 2023-06-10T21:04:10.563 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STARTING 2023-06-10T21:04:10.563 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Serving on http://:::9283 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 bash[142035]: 95a95736b9e95c3172311e8e28d208b26344dff7fc50ed69d3b805c48804e074 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 systemd[1]: Started Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-06-10T21:04:10.564 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-06-10T21:04:10.565 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="App mode production" logger=settings 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=555.515µs 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-06-10T21:04:10.566 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-06-10T21:04:10.567 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-06-10T21:04:10.567 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-06-10T21:04:10.567 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-06-10T21:04:10.567 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-06-10T21:04:10.944 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-06-10T21:04:10.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-06-10T21:04:10.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-06-10T21:04:10.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-06-10T21:04:10.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-06-10T21:04:10.945 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[142072]: t=2023-06-10T21:04:10+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STARTED 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STOPPING 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STOPPED 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STARTING 2023-06-10T21:04:10.946 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Serving on http://:::9283 2023-06-10T21:04:10.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:10 smithi152 conmon[135060]: [10/Jun/2023:21:04:10] ENGINE Bus STARTED 2023-06-10T21:04:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.340110+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.355580+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.361648+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.362074+0000 mgr.x (mgr.24857) 82 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.365399+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.365776+0000 mgr.x (mgr.24857) 83 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.366605+0000 mon.b (mon.1) 88 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.371151+0000 mgr.x (mgr.24857) 84 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.385188+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.401177+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.401690+0000 mgr.x (mgr.24857) 85 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.455576+0000 mon.b (mon.1) 90 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.456238+0000 mgr.x (mgr.24857) 86 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.465231+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: cluster 2023-06-10T21:04:10.481892+0000 mgr.x (mgr.24857) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-10T21:04:11.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: audit 2023-06-10T21:04:10.502420+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:11.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:10.526361+0000 mgr.x (mgr.24857) 88 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:11.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:10.526914+0000 mgr.x (mgr.24857) 89 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:11.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.340110+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.355580+0000 mon.a ( 2023-06-10T21:04:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mon.0) 859 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.361648+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.362074+0000 mgr.x (mgr.24857) 82 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.365399+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.365776+0000 mgr.x (mgr.24857) 83 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.366605+0000 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mon.b (mon.1) 88 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04: 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 10.371151+0000 mgr.x (mgr.24857) 84 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T21:04:11.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 21:04:10.385188+0000 mon.a (mon 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: .0) 860 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: -06-10T21:04:10.401177 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: +0000 mon.b (mon.1) 89 : audit 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:11.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 2023-06-10T21:04:10.401690+0000 mgr.x 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: (mgr.24857) 85 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 06-10T21:04:10.455576+0000 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mon.b (mon.1) 90 : audit [INF] 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: -06-10T21:04:10.456238+0000 mgr.x ( 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mgr.24857) 86 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06-10T21:04:10.465231+0000 mon.a ( 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mon.0) 861 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: T21:04:10.481892+0000 mgr.x (mgr 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: .24857) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-10T21:04:11.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: -10T21:04:10.502420+0000 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: mon.b (mon.1) 91 : audit [DBG] 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: cephadm 2023 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: -06-10T21:04:10.526361+0000 mgr.x 2023-06-10T21:04:11.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: (mgr.24857) 88 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 2023-06-10T21:04:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T21:04:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: 06-10T21:04:10.526914+0000 mgr.x (mgr.24857) 89 2023-06-10T21:04:11.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:11 smithi064 conmon[97224]: : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:11.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21:04:10.340110+0000 mon.a (mon 2023-06-10T21:04:11.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: .0) 858 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21:04:10.355580+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: T21:04:10.361648+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21:04 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: :10.362074+0000 mgr.x (mgr.24857) 82 : 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: T21:04:10.365399+0000 mon.b (mon. 2023-06-10T21:04:11.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 1) 87 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 06-10T21:04:10.365776+0000 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mgr.x (mgr.24857) 83 : audit [DBG] 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:04:11.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 04:10.366605+0000 mon.b (mon.1) 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 88 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 10T21:04:10.371151+0000 mgr.x ( 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mgr.24857) 84 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152:9095"}]: dispatch 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 06-10T21:04:10.385188+0000 2023-06-10T21:04:11.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mon.a (mon.0) 860 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 21:04:10.401177+0000 mon.b (mon.1) 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 89 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: -10T21:04:10.401690+0000 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mgr.x (mgr.24857) 85 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-10T21:04:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 2023-06-10T21:04:10.455576 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: +0000 mon.b (mon.1) 90 : audit [INF] 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023-06-10T21:04: 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 10.456238+0000 mgr.x (mgr.24857) 86 : audit 2023-06-10T21:04:11.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152:3000"}]: dispatch 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: -06-10T21:04:10.465231+0000 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mon.a (mon.0) 861 : audit [INF] 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: cluster 2023-06- 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 10T21:04:10.481892+0000 mgr.x ( 2023-06-10T21:04:11.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mgr.24857) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-10T21:04:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 06-10T21:04:10.502420+0000 2023-06-10T21:04:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mon.b (mon.1) 91 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:11.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: 2023-06-10T21:04:11.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:10.526361+0000 mgr.x 2023-06-10T21:04:11.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: (mgr.24857) 88 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:11.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: cephadm 2023-06 2023-06-10T21:04:11.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: -10T21:04:10.526914+0000 mgr.x ( 2023-06-10T21:04:11.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: mgr.24857) 89 : cephadm [INF] 2023-06-10T21:04:11.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[97365]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:12.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:11 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:04:11] "GET /metrics HTTP/1.1" 200 190047 "" "Prometheus/2.33.4" 2023-06-10T21:04:12.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:12 smithi064 conmon[107122]: audit 2023-06-10T21:04:12. 2023-06-10T21:04:12.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:12 smithi064 conmon[107122]: 234171+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.64:0/1331303330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:12.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:12 smithi064 conmon[97224]: audit 2023-06-10T21:04:12.234171+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.64:0/1331303330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:12.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:12 smithi152 conmon[97365]: audit 2023-06-10T21:04:12.234171+0000 mon.b (mon. 2023-06-10T21:04:12.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:12 smithi152 conmon[97365]: 1) 92 : audit [DBG] from='client.? 172.21.15.64:0/1331303330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:13 smithi064 conmon[97224]: cluster 2023-06-10T21:04:12.483505 2023-06-10T21:04:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:13 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-06-10T21:04:13.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:13 smithi064 conmon[107122]: cluster 2023-06-10T21:04:12.483505+0000 mgr.x (mgr. 2023-06-10T21:04:13.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:13 smithi064 conmon[107122]: 24857) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-06-10T21:04:13.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:13 smithi152 conmon[97365]: cluster 2023-06-10T21:04:12.483505+0000 mgr.x (mgr.24857) 90 : cluster 2023-06-10T21:04:13.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:13 smithi152 conmon[97365]: [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2023-06-10T21:04:15.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:14.136952+0000 2023-06-10T21:04:15.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: mgr.x (mgr.24857) 91 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-10T21:04:15.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023-06-10T21:04:14.145676+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023-06-10T21:04:14.155590+0000 mon.b (mon.1) 93 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023-06-10T21:04:14.157170+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023-06-10T21:04:14.163009+0000 mon.b (mon.1) 94 : audit 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023-06-10T21:04:14.164162+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: cephadm 2023-06 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: -10T21:04:14.165286+0000 mgr.x (mgr 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: .24857) 92 : cephadm [INF] Deploying daemon mgr.y on smithi064 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: cluster 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: 2023-06-10T21 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: :04:14.483909+0000 mgr.x (mgr.24857) 2023-06-10T21:04:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: 93 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: 06-10T21:04:14.915820+0000 mon.a 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[97224]: (mon.0) 864 : audit [DBG] from='client.? 172.21.15.64:0/4123759981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:14 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: .136952+0000 mgr.x (mgr.24857) 91 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-10T21:04:15.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 06-10T21:04:14.145676+0000 mon.a (mon.0) 862 : audit 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 10T21:04:14.155590 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: +0000 mon.b (mon.1) 93 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: -10T21:04:14. 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 157170+0000 mon.a (mon.0) 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 863 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023-06-10T21:04:14. 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 163009+0000 mon.b (mon.1) 94 : audit [DBG] 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:15.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: -10T21:04:14.164162+0000 mon.b (mon.1) 95 : 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: cephadm 2023 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: -06-10T21:04:14.165286+0000 mgr.x (mgr 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: .24857) 92 : cephadm [INF] Deploying daemon mgr.y on smithi064 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: cluster 2023-06-10T21 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: :04:14.483909+0000 mgr.x 2023-06-10T21:04:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: (mgr.24857) 93 : cluster [DBG] 2023-06-10T21:04:15.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:15.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:15.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: -06-10T21:04:14.915820+0000 mon.a (mon. 2023-06-10T21:04:15.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:15 smithi064 conmon[107122]: 0) 864 : audit [DBG] from='client.? 172.21.15.64:0/4123759981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:14.136952+0000 mgr.x (mgr.24857) 91 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-10T21:04:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:14.145676+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:15.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: 2023-06-10T21:04:14.155590+0000 mon.b (mon.1) 93 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:14.157170+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:14.163009+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:14.164162+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:14.165286+0000 mgr.x (mgr.24857) 92 : cephadm [INF] Deploying daemon mgr.y on smithi064 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: cluster 2023-06-10T21:04:14.483909+0000 mgr.x (mgr.24857) 93 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:15.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:15 smithi152 conmon[97365]: audit 2023-06-10T21:04:14.915820+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.64:0/4123759981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:16.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:16 smithi064 systemd[1]: Stopping Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:17.148 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:16 smithi064 bash[171455]: 081c9920c7713501fd0d72bddad5889e532d8b49cfaf0cfdb39f6a521da36d9f 2023-06-10T21:04:17.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:17 smithi064 conmon[107122]: cluster 2023-06-10T21 2023-06-10T21:04:17.149 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:17 smithi064 conmon[107122]: :04:16.484326+0000 mgr.x (mgr.24857) 94 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:17.150 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:17 smithi064 conmon[97224]: cluster 2023-06-10T21:04:16.484326+0000 mgr.x (mgr.24857) 94 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:17 smithi152 conmon[97365]: cluster 2023-06-10T21:04:16.484326+0000 mgr.x (mgr.24857) 94 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-10T21:04:17.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:17 smithi152 conmon[97365]: 2023-06-10T21:04:17.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:17 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:04:17.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:17 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service: Failed with result 'exit-code'. 2023-06-10T21:04:17.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:17 smithi064 systemd[1]: Stopped Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:17.910 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:17 smithi064 systemd[1]: Starting Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:18 smithi064 conmon[97224]: audit 2023-06-10T21:04:17.571170 2023-06-10T21:04:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:18 smithi064 conmon[97224]: +0000 mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.64:0/3178710473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:18.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:18 smithi064 conmon[107122]: audit 2023-06-10T21:04:17.571170+0000 2023-06-10T21:04:18.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:18 smithi064 conmon[107122]: mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.64:0/3178710473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:18 smithi152 conmon[97365]: audit 2023-06-10T21:04:17.571170+0000 mon.b (mon.1) 96 : audit [DBG] from='client.? 172.21.15.64:0/3178710473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:18.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:18 smithi152 conmon[140617]: ts=2023-06-10T21:04:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:18.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:18 smithi152 conmon[140617]: ts=2023-06-10T21:04:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:19.142 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:18 smithi064 ceph-mgr[171892]: -- 172.21.15.64:0/250302490 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5630cb94f1e0 con 0x5630cb926800 2023-06-10T21:04:19.142 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:18 smithi064 bash[171829]: 2bda388cec1de0737fc16b6e6df8e21be326d4fe8918bfeca94862c45972f86a 2023-06-10T21:04:19.143 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:18 smithi064 systemd[1]: Started Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:19.143 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:18 smithi064 conmon[171866]: 2023-06-10T21:04:18.986+0000 7f1233d8e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:04:19.143 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[171866]: 2023-06-10T21:04:19.134+0000 7f1233d8e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:04:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[107122]: cluster 2023-06-10T21:04:18.484954+0000 mgr.x (mgr.24857) 95 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-06-10T21:04:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[107122]: audit 2023-06-10T21:04:18.904512+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[107122]: audit 2023-06-10T21:04:18.912837+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[107122]: audit 2023-06-10T21:04:18.945151+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:19.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[171866]: 2023-06-10T21:04:19.297+0000 7f1233d8e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:04:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: cluster 2023-06-10T21:04:18.484954 2023-06-10T21:04:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 95 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: audit 2023-06-10T21:04:18.904512+0000 mon.a 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: (mon.0) 865 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: 21:04:18.912837+0000 mon.a (mon.0 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: ) 866 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:04:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: -10T21:04:18.945151+0000 mon.b 2023-06-10T21:04:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[97224]: (mon.1) 97 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:19 smithi152 conmon[97365]: cluster 2023-06-10T21:04:18.484954+0000 mgr.x (mgr 2023-06-10T21:04:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:19 smithi152 conmon[97365]: .24857) 95 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-06-10T21:04:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:19 smithi152 conmon[97365]: audit 2023-06-10T21:04:18.904512+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:19 smithi152 conmon[97365]: audit 2023-06-10T21:04:18.912837+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:19 smithi152 conmon[97365]: audit 2023-06-10T21:04:18.945151+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:19.788 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[171866]: 2023-06-10T21:04:19.468+0000 7f1233d8e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:04:19.789 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:19 smithi064 conmon[171866]: 2023-06-10T21:04:19.784+0000 7f1233d8e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:04:20.811 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[171866]: 2023-06-10T21:04:20.407+0000 7f1233d8e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:04:20.811 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[171866]: 2023-06-10T21:04:20.578+0000 7f1233d8e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:04:20.812 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[171866]: 2023-06-10T21:04:20.806+0000 7f1233d8e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:04:20.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: audit 2023-06-10T21:04:19. 2023-06-10T21:04:20.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: 582705+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: audit 2023-06-10T21:04:19.609400+0000 mon.a ( 2023-06-10T21:04:20.812 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: mon.0) 868 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: 04:20.322758+0000 mon.b (mon.1) 98 : audit 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[107122]: [DBG] from='client.? 172.21.15.64:0/2993226162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[97224]: audit 2023-06-10T21:04:19.582705+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[97224]: audit 2023-06-10T21:04:19.609400+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.813 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[97224]: audit 2023-06-10T21:04:20.322758+0000 mon.b (mon.1) 98 : audit [DBG] from='client.? 172.21.15.64:0/2993226162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:20 smithi152 conmon[97365]: audit 2023-06-10T21:04:19.582705+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:20 smithi152 conmon[97365]: audit 2023-06-10T21:04:19.609400+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:20.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:20 smithi152 conmon[97365]: audit 2023-06-10T21:04:20.322758+0000 mon.b (mon.1) 98 : audit [DBG] from='client.? 172.21.15.64:0/2993226162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:21.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:20 smithi064 conmon[171866]: 2023-06-10T21:04:20.972+0000 7f1233d8e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:04:21.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[97224]: cluster 2023-06-10T21:04:20.486448 2023-06-10T21:04:21.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 96 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-06-10T21:04:21.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[97224]: audit 2023-06-10T21:04:20.612291+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:21.910 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[171866]: 2023-06-10T21:04:21.751+0000 7f1233d8e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:04:21.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[107122]: cluster 2023-06-10T21:04:20. 2023-06-10T21:04:21.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[107122]: 486448+0000 mgr.x (mgr.24857) 96 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-06-10T21:04:21.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[107122]: audit 2023-06-10T21:04:20.612291+0000 2023-06-10T21:04:21.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:21 smithi064 conmon[107122]: mon.b (mon.1) 99 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:21.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:21 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:04:21] "GET /metrics HTTP/1.1" 200 190048 "" "Prometheus/2.33.4" 2023-06-10T21:04:21.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:21 smithi152 conmon[97365]: cluster 2023-06-10T21:04:20.486448+0000 2023-06-10T21:04:21.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:21 smithi152 conmon[97365]: mgr.x (mgr.24857) 96 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-06-10T21:04:21.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:21 smithi152 conmon[97365]: audit 2023-06-10T21:04:20.612291+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:23.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:22 smithi064 conmon[171866]: 2023-06-10T21:04:22.815+0000 7f1233d8e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:04:23.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[171866]: 2023-06-10T21:04:23.137+0000 7f1233d8e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:04:23.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[107122]: cluster 2023-06-10T21:04:22 2023-06-10T21:04:23.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[107122]: .486992+0000 mgr.x (mgr.24857) 97 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-10T21:04:23.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[107122]: audit 2023-06-10T21:04:23.079680+0000 mon.a ( 2023-06-10T21:04:23.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[107122]: mon.0) 869 : audit [DBG] from='client.? 172.21.15.64:0/3268073031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:23.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[97224]: cluster 2023-06-10T21:04:22.486992+0000 mgr.x (mgr 2023-06-10T21:04:23.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[97224]: .24857) 97 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-10T21:04:23.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:23 smithi064 conmon[97224]: audit 2023-06-10T21:04:23.079680+0000 mon.a (mon.0) 869 : audit [DBG] from='client.? 172.21.15.64:0/3268073031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:23.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:23 smithi152 conmon[97365]: cluster 2023-06-10T21:04:22.486992+0000 mgr.x (mgr.24857) 97 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-10T21:04:23.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:23 smithi152 conmon[97365]: audit 2023-06-10T21:04:23.079680+0000 mon.a (mon.0) 869 : audit [DBG] from='client.? 172.21.15.64:0/3268073031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:24.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[171866]: 2023-06-10T21:04:24.564+0000 7f1233d8e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:04:24.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[171866]: 2023-06-10T21:04:24.733+0000 7f1233d8e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:04:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[97224]: audit 2023-06-10T21:04:23.762027+0000 mon.a (mon.0 2023-06-10T21:04:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[97224]: ) 870 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[97224]: audit 2023-06-10T21:04:23.789483+0000 mon.a (mon.0) 871 : audit 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[97224]: [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[107122]: audit 2023-06-10T21:04 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[107122]: :23.762027+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:04:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[107122]: 04:23.789483+0000 mon.a (mon.0 2023-06-10T21:04:24.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:24 smithi064 conmon[107122]: ) 871 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:24 smithi152 conmon[97365]: audit 2023-06-10T21:04:23.762027+0000 mon.a (mon 2023-06-10T21:04:25.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:24 smithi152 conmon[97365]: .0) 870 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:24 smithi152 conmon[97365]: audit 2023-06-10T21:04:23.789483+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.410 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[171866]: 2023-06-10T21:04:25.062+0000 7f1233d8e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:04:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[107122]: cluster 2023-06-10T21:04:24.487383+0000 mgr.x (mgr.24857) 98 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[107122]: audit 2023-06-10T21:04:24.809852+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[107122]: audit 2023-06-10T21:04:24.843248+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: cluster 2023-06-10T21:04:24 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: .487383+0000 mgr.x (mgr.24857) 98 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: 06-10T21:04:24.809852+0000 mon.a (mon.0) 872 : audit [INF] 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: audit 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: 2023-06-10T21:04:24.843248+0000 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: mon.a (mon.0) 873 : audit [INF] 2023-06-10T21:04:25.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:25.919 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[171866]: 2023-06-10T21:04:25.633+0000 7f1233d8e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:04:26.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:25 smithi152 conmon[97365]: cluster 2023-06-10T21:04:24.487383+0000 mgr.x (mgr.24857) 98 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:26.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:25 smithi152 conmon[97365]: audit 2023-06-10T21:04:24.809852+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:26.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:25 smithi152 conmon[97365]: audit 2023-06-10T21:04:24.843248+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:26.280 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:25 smithi064 conmon[171866]: 2023-06-10T21:04:25.998+0000 7f1233d8e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:04:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: audit 2023-06-10T21:04:25.839161+0000 mon.a (mon. 2023-06-10T21:04:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: 0) 874 : audit [DBG] from='client.? 172.21.15.64:0/2959029173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: audit 2023-06-10T21:04:26.238834+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: -06-10T21:04:26.257435+0000 2023-06-10T21:04:27.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: mon.a (mon.0) 876 : audit [INF] 2023-06-10T21:04:27.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:27.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[171866]: 2023-06-10T21:04:26.274+0000 7f1233d8e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:04:27.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[171866]: 2023-06-10T21:04:26.799+0000 7f1233d8e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: audit 2023-06-10T21:04:25.839161 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: +0000 mon.a (mon.0) 874 : audit [DBG] from='client.? 172.21.15.64:0/2959029173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: audit 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: 2023-06-10T21: 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: 04:26.238834+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: 06-10T21:04:26.257435+0000 2023-06-10T21:04:27.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: mon.a (mon.0) 876 : audit [INF] 2023-06-10T21:04:27.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:26 smithi064 conmon[107122]: from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:26 smithi152 conmon[97365]: audit 2023-06-10T21:04:25.839161+0000 mon.a (mon.0) 874 : audit 2023-06-10T21:04:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:26 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/2959029173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:26 smithi152 conmon[97365]: audit 2023-06-10T21:04:26.238834+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:26 smithi152 conmon[97365]: audit 2023-06-10T21:04:26.257435+0000 mon.a (mon.0 2023-06-10T21:04:27.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:26 smithi152 conmon[97365]: ) 876 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:28.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:27 smithi064 conmon[97224]: cluster 2023-06-10T21:04:26.487764 2023-06-10T21:04:28.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:27 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 99 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:28.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[171866]: 2023-06-10T21:04:28.046+0000 7f1233d8e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:04:28.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:27 smithi064 conmon[107122]: cluster 2023-06-10T21:04:26.487764+0000 mgr.x (mgr.24857) 99 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:27 smithi152 conmon[97365]: cluster 2023-06-10T21:04:26.487764+0000 mgr.x (mgr 2023-06-10T21:04:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:27 smithi152 conmon[97365]: .24857) 99 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 691 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 570 op/s 2023-06-10T21:04:28.662 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[171866]: 2023-06-10T21:04:28.287+0000 7f1233d8e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:04:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:28 smithi152 conmon[97365]: audit 2023-06-10T21:04:28.553746+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.64:0/2755705566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:28.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:28 smithi152 conmon[140617]: ts=2023-06-10T21:04:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:28.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:28 smithi152 conmon[140617]: ts=2023-06-10T21:04:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:29.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[107122]: audit 2023-06-10T21:04:28.553746+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.64:0/2755705566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:29.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[97224]: audit 2023-06-10T21:04:28.553746+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.64:0/2755705566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:29.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[171866]: 2023-06-10T21:04:28.677+0000 7f1233d8e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:04:29.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:28 smithi064 conmon[171866]: 2023-06-10T21:04:28.880+0000 7f1233d8e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:04:29.611 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:29 smithi064 conmon[171866]: 2023-06-10T21:04:29.183+0000 7f1233d8e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:04:29.862 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:29 smithi064 conmon[97224]: cluster 2023-06-10T21:04:28.488397+0000 mgr.x (mgr.24857) 100 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2023-06-10T21:04:30.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:29 smithi064 conmon[107122]: cluster 2023-06-10T21:04:28.488397+0000 mgr.x (mgr.24857) 100 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2023-06-10T21:04:30.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:29 smithi152 conmon[97365]: cluster 2023-06-10T21:04:28.488397+0000 mgr.x ( 2023-06-10T21:04:30.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:29 smithi152 conmon[97365]: mgr.24857) 100 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2023-06-10T21:04:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: cluster 2023-06-10T21:04:30.488773 2023-06-10T21:04:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 101 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: cluster 2023-06-10T21:04:30.749663+0000 mon.a ( 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: mon.0) 877 : cluster [DBG] Standby manager daemon y restarted 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: cluster 2023-06-10T21:04 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: :30.749793+0000 mon.a (mon.0 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: ) 878 : cluster [DBG] Standby manager daemon y started 2023-06-10T21:04:31.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: -10T21:04:30.750328+0000 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: mon.b (mon.1) 100 : audit 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: 04:30.755624+0000 mon.b (mon. 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: 1) 101 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:31.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: -06-10T21:04:30. 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: 760023+0000 mon.b (mon.1) 102 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: T21:04:30.761063+0000 mon.b 2023-06-10T21:04:31.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: (mon.1) 103 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:31.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[97224]: 2023-06-10T21:04:31.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: 2023-06-10T21:04:30.739+0000 7f1233d8e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:04:31.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: [10/Jun/2023:21:04:30] ENGINE Bus STARTING 2023-06-10T21:04:31.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: CherryPy Checker: 2023-06-10T21:04:31.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: The Application mounted at '' has an empty config. 2023-06-10T21:04:31.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: 2023-06-10T21:04:31.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: [10/Jun/2023:21:04:30] ENGINE Serving on http://:::9283 2023-06-10T21:04:31.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[171866]: [10/Jun/2023:21:04:30] ENGINE Bus STARTED 2023-06-10T21:04:31.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: cluster 2023-06-10T21:04:30.488773+0000 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: mgr.x (mgr.24857) 101 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: cluster 2023-06-10T21:04:30.749663+0000 mon.a (mon.0) 877 : cluster [DBG] Standby manager daemon y restarted 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: cluster 2023-06-10T21:04:30. 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 749793+0000 mon.a (mon.0) 878 : cluster 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: [DBG] Standby manager daemon y started 2023-06-10T21:04:31.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: audit 2023-06-10T21:04:31.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 2023-06-10T21:04:30.750328+0000 mon.b 2023-06-10T21:04:31.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: (mon.1) 100 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:04:31.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:31.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 06-10T21:04:30.755624+0000 2023-06-10T21:04:31.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: mon.b (mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:31.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 2023-06-10T21:04:31.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:31.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: -10T21:04:30.760023+0000 mon.b (mon.1) 2023-06-10T21:04:31.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 102 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:04:31.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:04:31.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: 04:30.761063+0000 mon.b (mon.1 2023-06-10T21:04:31.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:30 smithi064 conmon[107122]: ) 103 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: cluster 2023-06-10T21:04:30.488773+0000 mgr.x (mgr.24857) 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 101 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 493 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: cluster 2023-06-10T21:04:30.749663+0000 mon.a (mon.0) 877 : cluster [DBG] Standby manager daemon y restarted 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: cluster 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 2023-06-10T21:04:30.749793+0000 mon.a (mon.0) 878 : cluster [DBG] Standby manager daemon y started 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:31.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: -10T21:04:30.750328+0000 mon.b (mon. 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 1) 100 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: audit 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 2023-06-10T 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 21:04:30.755624+0000 mon.b ( 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: mon.1) 101 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: audit 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 2023-06-10T21:04: 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 30.760023+0000 mon.b (mon.1) 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: 102 : audit [DBG] from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: -10T21:04:30.761063+0000 mon.b ( 2023-06-10T21:04:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: mon.1) 103 : audit [DBG] 2023-06-10T21:04:31.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:30 smithi152 conmon[97365]: from='mgr.? 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:32.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[107122]: cluster 2023-06-10T21:04 2023-06-10T21:04:32.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[107122]: :30.869334+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e27: x(active, since 2m), standbys: y 2023-06-10T21:04:32.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[107122]: audit 2023-06-10T21:04:31.252957+0000 mon.b (mon.1) 104 : audit [DBG] from='client.? 172.21.15.64:0/618477130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:32.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[97224]: cluster 2023-06-10T21:04:30.869334+0000 mon.a ( 2023-06-10T21:04:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[97224]: mon.0) 879 : cluster [DBG] mgrmap e27: x(active, since 2m), standbys: y 2023-06-10T21:04:32.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:31 smithi064 conmon[97224]: audit 2023-06-10T21:04:31.252957+0000 mon.b (mon.1) 104 : audit [DBG] from='client.? 172.21.15.64:0/618477130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:32.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:31 smithi152 conmon[135060]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:04:31] "GET /metrics HTTP/1.1" 200 190048 "" "Prometheus/2.33.4" 2023-06-10T21:04:32.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:31 smithi152 conmon[97365]: cluster 2023-06-10T21:04:30.869334+0000 mon.a (mon.0) 879 : cluster [DBG] mgrmap e27: x(active, since 2m), standbys: y 2023-06-10T21:04:32.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:31 smithi152 conmon[97365]: audit 2023-06-10T21:04:31.252957+0000 mon.b (mon.1) 104 : audit [DBG] from='client.? 172.21.15.64:0/618477130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:33.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06-10T21:04:32.242812 2023-06-10T21:04:33.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: +0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06-10T21:04:32.265281+0000 mon.a (mon.0) 881 : 2023-06-10T21:04:33.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: -10T21:04:32.270965+0000 mon.b (mon.1) 105 : 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06-10T21:04:32.279797+0000 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: mon.b (mon.1) 106 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 2023-06-10T21:04:33.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: -10T21:04:32.298685+0000 mon.a (mon.0) 882 : 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cluster 2023-06-10T21:04:32. 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 489384+0000 mgr.x (mgr.24857) 102 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:33.579 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: -10T21:04:32.570118+0000 mon.b (mon.1) 107 : 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 2023-06-10T21:04:32.573836+0000 mgr.x (mgr. 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 24857) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cephadm 2023-06-10T21: 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 04:32.574317+0000 mgr.x (mgr.24857) 104 2023-06-10T21:04:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:32.577056+0000 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: mgr.x (mgr.24857) 105 : cephadm [INF] Failing over to other MGR 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: T21:04:32.577581+0000 mon.b (mon. 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 1) 108 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: T21:04:32.578210+0000 mon.a (mon. 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 0) 883 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cluster 2023-06-10T21 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: :04:32.583101+0000 mon.a (mon.0) 884 : 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: cluster 2023-06-10T21:04:32. 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: 988404+0000 mon.a (mon.0) 885 : cluster [DBG] 2023-06-10T21:04:33.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[97365]: Standby manager daemon x started 2023-06-10T21:04:33.583 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.327+0000 7f35c538e700 -1 mgr handle_mgr_map I was active but no longer am 2023-06-10T21:04:33.583 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: ignoring --setuser ceph since I am not root 2023-06-10T21:04:33.583 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: ignoring --setgroup ceph since I am not root 2023-06-10T21:04:33.583 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.400+0000 7f17485cb700 1 -- 172.21.15.152:0/2730201647 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f2172ed1e0 con 0x55f2172c5000 2023-06-10T21:04:33.583 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.572+0000 7f17522f9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:04:33.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023-06-10T21:04:32.242812+0000 mon.a (mon 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: .0) 880 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023-06-10T21:04:32.265281+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023-06-10T21:04:32. 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 270965+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: -06-10T21:04:32.279797+0000 2023-06-10T21:04:33.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: mon.b (mon.1) 106 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 06-10T21:04:32.298685+0000 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: mon.a (mon.0) 882 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cluster 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 2023-06-10T21:04:32.489384 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: +0000 mgr.x (mgr.24857) 102 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-06-10T21:04:33.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 2023-06-10T21:04:32 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: .570118+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:32.573836+0000 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: mgr.x (mgr.24857) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 06-10T21:04:32.574317+0000 mgr.x 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: (mgr.24857) 104 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cephadm 2023- 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 06-10T21:04:32.577056+0000 mgr.x 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: (mgr.24857) 105 : cephadm [INF] Failing over to other MGR 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 06-10T21:04:32.577581+0000 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: mon.b (mon.1) 108 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:04:33.633 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: -10T21:04:32.578210+0000 mon.a (mon 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: .0) 883 : audit [INF] 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cluster 2023-06-10T21:04:32 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: .583101+0000 mon.a (mon.0) 884 : cluster 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: cluster 2023-06-10T21:04: 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: 32.988404+0000 mon.a (mon.0) 885 : cluster 2023-06-10T21:04:33.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[97224]: [DBG] Standby manager daemon x started 2023-06-10T21:04:33.635 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE Bus STOPPING 2023-06-10T21:04:33.635 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:04:33.635 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE Bus STOPPED 2023-06-10T21:04:33.635 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE Bus STARTING 2023-06-10T21:04:33.636 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE Serving on http://:::9283 2023-06-10T21:04:33.636 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: [10/Jun/2023:21:04:33] ENGINE Bus STARTED 2023-06-10T21:04:33.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023-06-10T21:04:32.242812 2023-06-10T21:04:33.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: +0000 mon.a (mon.0) 880 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.636 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: :04:32.265281+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: -06-10T21:04:32.270965+0000 mon.b 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: (mon.1) 105 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 06-10T21:04:32.279797+0000 mon.b ( 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: mon.1) 106 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 06-10T21:04:32.298685+0000 mon.a 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: (mon.0) 882 : audit [INF] from='mgr.24857 ' entity='mgr.x' 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cluster 2023 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: -06-10T21:04:32.489384+0000 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: mgr.x (mgr.24857) 102 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 294 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 06-10T21:04:32.570118+0000 mon.b 2023-06-10T21:04:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: (mon.1) 107 : audit [DBG] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 06-10T21:04:32.573836+0000 mgr.x 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: (mgr.24857) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cephadm 2023 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: -06-10T21:04:32.574317+0000 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: mgr.x (mgr.24857) 104 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 2023-06-10T21:04:33.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 2023-06-10T21:04:32.577056+0000 mgr.x (mgr. 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 24857) 105 : cephadm [INF] Failing over to other MGR 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: T21:04:32.577581+0000 mon.b (mon. 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 1) 108 : audit [INF] from='mgr.24857 172.21.15.152:0/3726797934' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:04:33.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 10T21:04:32.578210+0000 mon.a (mon. 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 0) 883 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cluster 2023-06-10 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: T21:04:32.583101+0000 mon.a (mon. 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 0) 884 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: cluster 2023-06- 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: 10T21:04:32.988404+0000 mon.a (mon 2023-06-10T21:04:33.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[107122]: .0) 885 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:04:33.851 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.680+0000 7f17522f9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:04:33.851 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.815+0000 7f17522f9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:04:33.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.792+0000 7f11914aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.792+0000 7f11914aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.792+0000 7f11914aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.792+0000 7f11914aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.792+0000 7f11914aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.813+0000 7f11944b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.813+0000 7f11944b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.813+0000 7f11944b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.813+0000 7f11944b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:33.910 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:33 smithi064 conmon[171866]: 2023-06-10T21:04:33.813+0000 7f11944b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:04:34.199 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:33 smithi152 conmon[135060]: debug 2023-06-10T21:04:33.956+0000 7f17522f9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:04:34.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.323642+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-10T21:04:34.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: cluster 2023-06-10T21:04:33.323735+0000 mon.a (mon.0) 887 : cluster [DBG] mgrmap e28: y(active, starting, since 0.74464s), standbys: x 2023-06-10T21:04:34.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.336173+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.336345+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.336502+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.336886+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33. 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 337097+0000 mon.c (mon.2) 75 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.338636+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.339242+0000 mon.c (mon.2) 77 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:04:34.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.339447+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.339794+0000 mon.c (mon.2) 79 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.340087+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.340320+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.340521+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.340720+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.341051+0000 mon.c (mon.2) 84 : audit 2023-06-10T21:04:34.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:04:34.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.341262+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:04:34.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T21:04:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 21:04:33.350670+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:04:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04: 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 33.351071+0000 mon.c (mon.2) 87 : audit 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: -06-10T21:04:33.351564+0000 mon.c (mon.2) 88 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: cluster 2023-06-10T21: 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 04:33.387894+0000 mon.a (mon.0) 888 : 2023-06-10T21:04:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: cluster [INF] Manager daemon y is now available 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: .439935+0000 mon.c (mon.2) 89 : audit 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 2023-06-10T21:04:33.451700+0000 mon.c (mon 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: .2) 90 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 04:33.456539+0000 mon.c (mon.2) 91 : 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.487015 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: +0000 mon.c (mon.2) 92 : audit [INF] 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 06-10T21:04:33.487670+0000 mon.a (mon.0) 889 : 2023-06-10T21:04:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21:04:33.633754 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: +0000 mon.c (mon.2) 93 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: -10T21:04:33.634244+0000 mon.a (mon.0) 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 890 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: audit 2023-06-10T21 2023-06-10T21:04:34.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: :04:34.009596+0000 mon.b (mon.1) 2023-06-10T21:04:34.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[97224]: 109 : audit [DBG] from='client.? 172.21.15.64:0/2160698495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:34.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:33.323642+0000 mon.a ( 2023-06-10T21:04:34.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.0) 886 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: cluster 2023-06-10T21:04: 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 33.323735+0000 mon.a (mon.0) 887 : cluster [DBG] mgrmap e28: y(active, starting, since 0.74464s), standbys: x 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 10T21:04:33.336173+0000 mon.c (mon. 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 2) 71 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: T21:04:33.336345+0000 mon.c (mon.2) 72 2023-06-10T21:04:34.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: :04:33.336502+0000 mon.c (mon.2) 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 73 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 21:04:33.336886+0000 mon.c (mon. 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 2) 74 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:04:34.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 21:04:33.337097+0000 mon.c (mon.2 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: ) 75 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: :04:33.338636+0000 mon.c (mon.2) 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 76 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 04:33.339242+0000 mon.c ( 2023-06-10T21:04:34.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 77 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.339447+0000 mon.c ( 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 78 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.339794+0000 mon.c 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: (mon.2) 79 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:34.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: -06-10T21:04:33.340087+0000 mon.c 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: (mon.2) 80 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: -06-10T21:04:33.340320+0000 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.c (mon.2) 81 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.340521+0000 mon.c ( 2023-06-10T21:04:34.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 82 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 10T21:04:33.340720+0000 mon.c ( 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 83 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: -10T21:04:33.341051+0000 mon.c ( 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 84 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:34.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: -10T21:04:33.341262+0000 mon.c ( 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 85 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.350670+0000 mon.c 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: (mon.2) 86 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.351071+0000 mon.c 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: (mon.2) 87 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:04:34.670 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 06-10T21:04:33.351564+0000 mon.c ( 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: mon.2) 88 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: cluster 2023-06 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: -10T21:04: 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 33.387894+0000 mon.a (mon.0) 888 : cluster 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: [INF] Manager daemon y is now available 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04: 2023-06-10T21:04:34.671 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 33.439935+0000 mon.c (mon.2) 89 : audit 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04: 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 33.451700+0000 mon.c (mon.2) 90 : audit 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:33 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: .456539+0000 mon.c (mon.2) 91 : audit [DBG] 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.672 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:33.487015 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: +0000 mon.c (mon.2) 92 : audit [INF] 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:33.487670 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: +0000 mon.a (mon.0) 889 : audit [INF] 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:33 2023-06-10T21:04:34.673 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: .633754+0000 mon.c (mon.2) 93 : audit 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: 21:04:33.634244+0000 mon.a (mon.0 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: ) 890 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: audit 2023-06-10T21:04:34.009596+0000 mon.b (mon.1 2023-06-10T21:04:34.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:34 smithi064 conmon[107122]: ) 109 : audit [DBG] from='client.? 172.21.15.64:0/2160698495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:34.699 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[135060]: debug 2023-06-10T21:04:34.211+0000 7f17522f9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:04:34.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:33.323642+0000 mon.a ( 2023-06-10T21:04:34.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.0) 886 : audit [INF] from='mgr.24857 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-10T21:04:34.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: cluster 2023-06-10T21:04:33.323735+0000 mon.a (mon.0) 887 : 2023-06-10T21:04:34.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: cluster [DBG] mgrmap e28: y(active, starting, since 0.74464s), standbys: x 2023-06-10T21:04:34.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04:33.336173+0000 mon.c ( 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.2) 71 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04: 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 33.336345+0000 mon.c (mon.2) 72 : audit 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:04:34.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 21:04:33.336502+0000 mon.c (mon.2 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: ) 73 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.336886+0000 mon.c 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.2) 74 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.337097+0000 2023-06-10T21:04:34.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.c (mon.2) 75 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: :33.338636+0000 mon.c (mon.2) 76 : 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.339242+0000 2023-06-10T21:04:34.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.c (mon.2) 77 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04:33 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .339447+0000 mon.c (mon.2) 78 : audit [DBG] 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04: 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 33.339794+0000 mon.c (mon.2) 79 : 2023-06-10T21:04:34.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -10T21:04:33.340087+0000 mon.c (mon 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .2) 80 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.340320+0000 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.c (mon.2) 81 : 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:04:34.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 2023-06-10T21:04:33.340521+0000 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.c (mon.2) 82 : audit [DBG] 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21:04:33 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .340720+0000 mon.c (mon.2) 83 : audit 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10T21 2023-06-10T21:04:34.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: :04:33.341051+0000 mon.c (mon.2) 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 84 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: T21:04:33.341262+0000 mon.c (mon 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .2) 85 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.350670+0000 mon.c ( 2023-06-10T21:04:34.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.2) 86 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.351071+0000 mon.c ( 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.2) 87 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.351564+0000 mon.c 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.2) 88 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: cluster 2023 2023-06-10T21:04:34.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21: 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 04:33.387894+0000 mon.a (mon.0) 888 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: : cluster [INF] Manager daemon y is now available 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 10T21:04:33.439935+0000 mon.c (mon 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .2) 89 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.451700+0000 mon.c 2023-06-10T21:04:34.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.2) 90 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.456539+0000 mon.c (mon 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: .2) 91 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.487015+0000 mon.c 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.2) 92 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.710 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.487670+0000 mon.a ( 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: mon.0) 889 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: 06-10T21:04:33.633754+0000 mon.c 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.2) 93 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:33.634244+0000 mon.a 2023-06-10T21:04:34.711 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.0) 890 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:04:34.712 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:34.712 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: -06-10T21:04:34.009596+0000 mon.b 2023-06-10T21:04:34.712 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[97365]: (mon.1) 109 : audit [DBG] from='client.? 172.21.15.64:0/2160698495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:35.056 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[135060]: debug 2023-06-10T21:04:34.711+0000 7f17522f9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:04:35.057 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:34 smithi152 conmon[135060]: debug 2023-06-10T21:04:34.849+0000 7f17522f9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:04:35.057 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[135060]: debug 2023-06-10T21:04:35.048+0000 7f17522f9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:04:35.377 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[135060]: debug 2023-06-10T21:04:35.208+0000 7f17522f9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:04:35.378 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: cluster 2023-06-10T21:04:34.369808+0000 mon.a 2023-06-10T21:04:35.378 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: (mon.0) 891 : cluster [DBG] mgrmap e29: y(active, since 1.79071s), standbys: x 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: cluster 2023-06-10T21:04:34.432840+0000 mgr.y (mgr.24961 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: ) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: audit 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: 2023-06-10T21:04:34.797290+0000 mon.a (mon. 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: 0) 892 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:35.379 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: 06-10T21:04:34.822349+0000 2023-06-10T21:04:35.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: mon.a (mon.0) 893 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:35.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[97365]: 2023-06-10T21:04:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[97224]: cluster 2023-06-10T21:04:34.369808+0000 mon.a (mon.0) 891 : cluster [DBG] mgrmap e29: y(active, since 1.79071s), standbys: x 2023-06-10T21:04:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[97224]: cluster 2023-06-10T21:04:34.432840+0000 mgr.y (mgr.24961) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[97224]: audit 2023-06-10T21:04:34.797290+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[97224]: audit 2023-06-10T21:04:34.822349+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: cluster 2023-06-10T21:04:34 2023-06-10T21:04:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: .369808+0000 mon.a (mon.0) 891 : cluster [DBG] mgrmap e29: y(active, since 1.79071s), standbys: x 2023-06-10T21:04:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: cluster 2023-06-10T21:04:34. 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: 432840+0000 mgr.y (mgr.24961) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: 06-10T21:04:34.797290+0000 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: mon.a (mon.0) 892 : audit [INF] 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: audit 2023-06-10T21:04:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: 2023-06-10T21:04:34.822349+0000 2023-06-10T21:04:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: mon.a (mon.0) 893 : audit [INF] 2023-06-10T21:04:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:35 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:36.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:35 smithi152 conmon[135060]: debug 2023-06-10T21:04:35.908+0000 7f17522f9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:04:36.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:36 smithi152 conmon[97365]: cluster 2023-06-10T21:04:35.345064+0000 mgr.y (mgr 2023-06-10T21:04:36.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:36 smithi152 conmon[97365]: .24961) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:36.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:36 smithi152 conmon[97365]: cluster 2023-06-10T21:04:35.394235+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 2s), standbys: x 2023-06-10T21:04:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:36 smithi064 conmon[107122]: cluster 2023-06-10T21:04:35.345064+0000 2023-06-10T21:04:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:36 smithi064 conmon[107122]: mgr.y (mgr.24961) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:36 smithi064 conmon[107122]: cluster 2023-06-10T21:04:35.394235+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 2s), standbys: x 2023-06-10T21:04:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:36 smithi064 conmon[97224]: cluster 2023-06-10T21:04:35.345064+0000 mgr.y (mgr.24961) 2 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:36 smithi064 conmon[97224]: cluster 2023-06-10T21:04:35.394235+0000 mon.a (mon.0) 894 : cluster [DBG] mgrmap e30: y(active, since 2s), standbys: x 2023-06-10T21:04:37.055 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:36 smithi152 conmon[135060]: debug 2023-06-10T21:04:36.759+0000 7f17522f9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:04:37.056 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[135060]: debug 2023-06-10T21:04:37.050+0000 7f17522f9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:04:37.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:36.517723 2023-06-10T21:04:37.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: +0000 mgr.y (mgr.24961) 3 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:36.628566+0000 mgr.y (mgr.24961) 4 : 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: cephadm 2023-06-10T21 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: :04:36.684068+0000 mgr.y (mgr.24961) 5 : cephadm [INF] 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: cephadm 2023 2023-06-10T21:04:37.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: -06-10T21:04:36.684213+0000 mgr.y (mgr. 2023-06-10T21:04:37.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: 24961) 6 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:04:37.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:37.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: T21:04:36.696076+0000 mon.a (mon 2023-06-10T21:04:37.452 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:37 smithi152 conmon[97365]: .0) 895 : audit [DBG] from='client.? 172.21.15.64:0/3496493183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:36 2023-06-10T21:04:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: .517723+0000 mgr.y (mgr.24961) 3 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:04:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:36.628566+0000 mgr.y (mgr.24961 2023-06-10T21:04:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: ) 4 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:04:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: cephadm 2023 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: -06-10T21:04:36.684068 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: +0000 mgr.y (mgr.24961) 5 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: 06-10T21:04:36 2023-06-10T21:04:37.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: .684213+0000 mgr.y (mgr. 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: 24961) 6 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: audit 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[107122]: 2023-06-10T21:04:36.696076+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.64:0/3496493183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:36.517723+0000 mgr.y (mgr 2023-06-10T21:04:37.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: .24961) 3 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:36.628566+0000 mgr.y (mgr.24961) 4 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:36.684068+0000 mgr.y (mgr 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: .24961) 5 : cephadm [INF] [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: cephadm 2023-06-10T21 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: :04:36.684213+0000 mgr.y (mgr.24961) 6 : cephadm 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: [INF] [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: audit 2023-06-10T21:04:37.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: 2023-06-10T21:04:36.696076+0000 mon.a (mon 2023-06-10T21:04:37.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:37 smithi064 conmon[97224]: .0) 895 : audit [DBG] from='client.? 172.21.15.64:0/3496493183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:38.669 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[135060]: debug 2023-06-10T21:04:38.331+0000 7f17522f9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:04:38.669 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[135060]: debug 2023-06-10T21:04:38.483+0000 7f17522f9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:04:38.670 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[97365]: cluster 2023-06-10T21:04:37.345508+0000 mgr.y ( 2023-06-10T21:04:38.670 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[97365]: mgr.24961) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:38.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:38 smithi064 conmon[107122]: cluster 2023-06-10T21:04: 2023-06-10T21:04:38.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:38 smithi064 conmon[107122]: 37.345508+0000 mgr.y (mgr.24961) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:38.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:38 smithi064 conmon[97224]: cluster 2023-06-10T21:04:37.345508+0000 mgr.y ( 2023-06-10T21:04:38.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:38 smithi064 conmon[97224]: mgr.24961) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:38.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[135060]: debug 2023-06-10T21:04:38.796+0000 7f17522f9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:04:38.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[140617]: ts=2023-06-10T21:04:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:38.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:38 smithi152 conmon[140617]: ts=2023-06-10T21:04:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:04:38.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 119324 objects 2023-06-10T21:04:38.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :163.893 2023-06-10T21:04:39.411 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[135060]: debug 2023-06-10T21:04:39.156+0000 7f17522f9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:04:39.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[135060]: debug 2023-06-10T21:04:39.407+0000 7f17522f9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:04:39.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[135060]: debug 2023-06-10T21:04:39.569+0000 7f17522f9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: audit 2023-06-10T21:04:38.578858+0000 mon.a (mon 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: .0) 896 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: audit 2023-06-10T21:04:38.601141+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: audit 2023-06-10T21:04:39.206479+0000 mon.a ( 2023-06-10T21:04:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: mon.0) 898 : audit [DBG] from='client.? 172.21.15.64:0/1812540234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: audit 2023-06-10T21:04 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: :39.446740+0000 mon.a (mon.0) 899 : audit 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: 04:39.456893+0000 mon.a (mon.0) 900 2023-06-10T21:04:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: audit 2023-06-10T21:04:38.578858+0000 mon.a (mon.0) 896 : audit 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: audit 2023-06-10T21:04:38.601141+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: audit 2023-06-10T21:04:39.206479+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.64:0/1812540234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: audit 2023-06-10T21:04:39.446740+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: audit 2023-06-10T21:04:39.456893+0000 mon.a (mon.0) 900 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[97224]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: audit 2023-06-10T21:04:38.578858+0000 mon.a ( 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: mon.0) 896 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: audit 2023-06-10T21:04:38.601141+0000 mon.a (mon.0) 897 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: audit 2023-06-10T21:04 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: :39.206479+0000 mon.a (mon.0) 898 : audit [DBG] from='client.? 172.21.15.64:0/1812540234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: -06-10T21:04: 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: 39.446740+0000 mon.a (mon.0 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: ) 899 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: audit 2023 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: -06-10T21:04: 2023-06-10T21:04:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: 39.456893+0000 mon.a (mon.0) 2023-06-10T21:04:39.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:39 smithi064 conmon[107122]: 900 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:39 smithi152 conmon[135060]: debug 2023-06-10T21:04:39.866+0000 7f17522f9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:04:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[107122]: cluster 2023-06-10T21:04:39.345837+0000 mgr.y (mgr.24961) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[107122]: audit 2023-06-10T21:04:40.160465+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[107122]: audit 2023-06-10T21:04:40.166429+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[97224]: cluster 2023-06-10T21:04:39.345837+0000 mgr.y (mgr.24961) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[97224]: audit 2023-06-10T21:04:40.160465+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:40 smithi064 conmon[97224]: audit 2023-06-10T21:04:40.166429+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[135060]: debug 2023-06-10T21:04:40.632+0000 7f17522f9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:04:40.945 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[135060]: debug 2023-06-10T21:04:40.781+0000 7f17522f9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:04:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[97365]: cluster 2023-06-10T21:04:39.345837+0000 mgr.y (mgr 2023-06-10T21:04:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[97365]: .24961) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 174 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[97365]: audit 2023-06-10T21:04:40.160465+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[97365]: audit 2023-06-10T21:04:40.166429+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:41.217 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-06-10T21:04:41.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-06-10T21:04:41.336 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:40 smithi152 conmon[135060]: debug 2023-06-10T21:04:40.989+0000 7f17522f9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:04:41.336 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[135060]: debug 2023-06-10T21:04:41.106+0000 7f17522f9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:04:41.673 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:04:41.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[135060]: debug 2023-06-10T21:04:41.331+0000 7f17522f9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:04:42.124 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[107122]: audit 2023-06-10T21:04 2023-06-10T21:04:42.125 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[107122]: :40.859664+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.125 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[107122]: audit 2023-06-10T21:04:40.865508+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.125 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[107122]: cluster 2023-06-10T21:04:41.346455+0000 mgr.y (mgr.24961) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-06-10T21:04:42.125 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[107122]: audit 2023-06-10T21:04:41.667484+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.64:0/3979573520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:42.125 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[97224]: audit 2023-06-10T21:04:40.859664+0000 mon.a (mon.0 2023-06-10T21:04:42.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[97224]: ) 903 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[97224]: audit 2023-06-10T21:04:40.865508+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[97224]: cluster 2023-06-10T21:04:41.346455+0000 mgr.y (mgr.24961) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-06-10T21:04:42.126 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:41 smithi064 conmon[97224]: audit 2023-06-10T21:04:41.667484+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.64:0/3979573520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: audit 2023-06-10T21:04:40.859664+0000 mon.a (mon.0) 903 2023-06-10T21:04:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: audit 2023-06-10T21:04:40.865508+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: cluster 2023-06-10T21:04:41.346455+0000 mgr.y (mgr.24961) 9 2023-06-10T21:04:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 402 op/s 2023-06-10T21:04:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: audit 2023-06-10T21:04:41.667484+0000 mon.a (mon.0) 2023-06-10T21:04:42.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:41 smithi152 conmon[97365]: 905 : audit [DBG] from='client.? 172.21.15.64:0/3979573520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:42.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: debug 2023-06-10T21:04:42.219+0000 7f17522f9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:04:42.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: [10/Jun/2023:21:04:42] ENGINE Bus STARTING 2023-06-10T21:04:42.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: CherryPy Checker: 2023-06-10T21:04:42.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: The Application mounted at '' has an empty config. 2023-06-10T21:04:42.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: 2023-06-10T21:04:42.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: [10/Jun/2023:21:04:42] ENGINE Serving on http://:::9283 2023-06-10T21:04:42.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:42 smithi152 conmon[135060]: [10/Jun/2023:21:04:42] ENGINE Bus STARTED 2023-06-10T21:04:43.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: cluster 2023-06-10T21:04:42.233630+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:04:43.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: cluster 2023-06-10T21:04:42.233738+0000 mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:04:43.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: audit 2023-06-10T21:04:42.235386+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:04:43.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: audit 2023-06-10T21:04:42.236267+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: audit 2023-06-10T21:04:42.238693+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[107122]: audit 2023-06-10T21:04:42.239029+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: cluster 2023-06-10T21:04:42.233630+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: cluster 2023-06-10T21:04:42.233738+0000 mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: audit 2023-06-10T21:04:42.235386+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:04:43.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: audit 2023-06-10T21:04:42.236267+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:43.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: audit 2023-06-10T21:04:42.238693+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:04:43.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:43 smithi064 conmon[97224]: audit 2023-06-10T21:04:42.239029+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:43.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: cluster 2023-06-10T21:04:42.233630+0000 mon.a (mon.0) 906 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: cluster 2023-06-10T21:04:42.233738 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: +0000 mon.a (mon.0) 907 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: audit 2023-06-10T21:04:42.235386+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: audit 2023-06-10T21:04:42.236267+0000 mon.c (mon.2 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: ) 95 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: audit 2023-06-10T21:04:42.238693+0000 mon.c (mon.2) 96 : audit 2023-06-10T21:04:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:04:43.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: audit 2023-06-10T21:04:42.239029+0000 mon.c (mon. 2023-06-10T21:04:43.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:43 smithi152 conmon[97365]: 2) 97 : audit [DBG] from='mgr.? 172.21.15.152:0/3840144284' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:04:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: cluster 2023-06-10T21:04:43.131697+0000 mon.a (mon.0) 908 : cluster [DBG] mgrmap e31: y(active, since 10s), standbys: x 2023-06-10T21:04:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: cluster 2023-06-10T21:04:43.346927+0000 mgr.y (mgr.24961) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 311 op/s 2023-06-10T21:04:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:43.438983+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:43.445967+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:43.447732+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:43.448500+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:43.449507+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:43.449659+0000 mgr.y (mgr.24961) 12 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:44.020273+0000 mon.b (mon.1) 110 : audit [INF] from='client.? 172.21.15.64:0/2361726615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[107122]: audit 2023-06-10T21:04:44.020804+0000 mon.a (mon.0) 911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: cluster 2023-06-10T21:04:43.131697+0000 mon.a (mon.0) 908 : cluster [DBG] mgrmap e31: y(active, since 10s), standbys: x 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: cluster 2023-06-10T21:04:43.346927+0000 mgr.y (mgr.24961) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 311 op/s 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06-10T21:04:43.438983+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: 10T21:04:43.445967+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06-10T21:04:43.447732+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06-10T21:04:43.448500+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:44.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:43.449507+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:04:44.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:43.449659+0000 mgr.y (mgr.24961) 12 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:04:44.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06-10T21:04:44.020273+0000 mon.b (mon.1) 110 : audit [INF] from='client.? 172.21.15.64:0/2361726615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:44 smithi064 conmon[97224]: audit 2023-06-10T21:04:44.020804+0000 mon.a (mon.0) 911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: cluster 2023-06-10T21:04:43.131697+0000 mon.a (mon.0 2023-06-10T21:04:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: ) 908 : cluster [DBG] mgrmap e31: y(active, since 10s), standbys: x 2023-06-10T21:04:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: cluster 2023-06-10T21:04:43.346927+0000 mgr.y (mgr.24961) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 581 KiB data, 741 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 311 op/s 2023-06-10T21:04:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06-10T21:04:43.438983+0000 mon.a (mon.0) 909 : audit 2023-06-10T21:04:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: -10T21:04:43.445967+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: T21:04:43.447732+0000 mon.c (mon.2) 98 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: 2023-06-10T21:04:43.448500+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:04:44.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: cephadm 2023-06-10T21:04: 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: 43.449507+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:43.449659+0000 mgr.y 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: (mgr.24961) 12 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06-10T21:04:44.020273+0000 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: mon.b (mon.1) 110 : audit [INF] from='client.? 172.21.15.64:0/2361726615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: audit 2023-06-10T21:04:44.020804 2023-06-10T21:04:44.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:44 smithi152 conmon[97365]: +0000 mon.a (mon.0) 911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:04:44.496 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_0' removed 2023-06-10T21:04:45.301 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:04:45.302 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:04:45.303 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-06-10T21:04:45.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_1 16 2023-06-10T21:04:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:43.876130+0000 mgr.y (mgr 2023-06-10T21:04:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: .24961) 13 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:43.876305+0000 mgr.y (mgr.24961) 14 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023-06-10T21:04:44.184798+0000 mon.a (mon.0) 912 : audit [DBG] from='client.? 172.21.15.64:0/2951814282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:44.191550+0000 mgr.y (mgr.24961) 15 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023-06-10T21: 2023-06-10T21:04:45.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 04:44.191714+0000 mgr.y (mgr.24961) 16 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:45.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: -06-10T21:04:44.447726+0000 mon.a (mon.0) 913 2023-06-10T21:04:45.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:04:45.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cluster 2023- 2023-06-10T21:04:45.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 06-10T21:04:44.447797+0000 mon.a 2023-06-10T21:04:45.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: (mon.0) 914 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-06-10T21:04:45.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:45.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 2023-06-10T21:04:44.458678+0000 mgr.y 2023-06-10T21:04:45.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: (mgr.24961) 17 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: cephadm 2023 2023-06-10T21:04:45.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: -06-10T21:04:44.458883+0000 mgr.y ( 2023-06-10T21:04:45.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: mgr.24961) 18 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023 2023-06-10T21:04:45.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: -06-10T21:04:44.746369+0000 2023-06-10T21:04:45.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: mon.a (mon.0) 915 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023-06-10T21:04:45.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 2023-06-10T21:04:44.763445+0000 2023-06-10T21:04:45.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: mon.a (mon.0) 916 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023-06-10T21:04:45.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 2023-06-10T21: 2023-06-10T21:04:45.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 04:44.792482+0000 mon.a (mon.0) 917 2023-06-10T21:04:45.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:45.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 06-10T21:04:44.811237+0000 mon.a 2023-06-10T21:04:45.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: (mon.0) 918 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: audit 2023-06- 2023-06-10T21:04:45.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 10T21:04:44.817310+0000 mon.a (mon.0) 2023-06-10T21:04:45.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[97224]: 919 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.418 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:43. 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 876130+0000 mgr.y (mgr.24961) 13 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:43.876305+0000 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: mgr.y (mgr.24961) 14 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 21:04:44.184798+0000 mon.a (mon.0) 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 912 : audit [DBG] from='client.? 172.21.15.64:0/2951814282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06-10 2023-06-10T21:04:45.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: T21:04:44.191550+0000 mgr.y (mgr.24961) 15 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06-10T21:04 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: :44.191714+0000 mgr.y (mgr.24961) 16 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 2023-06-10T21:04:44.447726+0000 mon.a (mon.0) 913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:04:45.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cluster 2023-06-10T 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 21:04:44.447797+0000 mon.a (mon. 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 0) 914 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 10T21:04:44.458678+0000 mgr.y (mgr.24961) 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 17 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: cephadm 2023-06-10T21:04 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: :44.458883+0000 mgr.y (mgr.24961) 18 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10T21:04:44 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: .746369+0000 mon.a (mon.0) 915 : audit [INF] 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: T21:04:44.763445+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: T21:04:44.792482+0000 mon.a (mon.0) 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 917 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.422 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:04:45.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: T21:04:44.811237+0000 mon.a (mon.0) 918 : audit 2023-06-10T21:04:45.423 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: audit 2023- 2023-06-10T21:04:45.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: 06-10T21:04:44.817310+0000 mon.a (mon.0) 919 : audit [INF] 2023-06-10T21:04:45.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:45 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:43.876130+0000 mgr.y (mgr. 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 24961) 13 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:43.876305+0000 mgr.y (mgr.24961) 14 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023-06-10T21:04:44.184798+0000 mon.a (mon.0) 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 912 : audit [DBG] from='client.? 172.21.15.64:0/2951814282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023 2023-06-10T21:04:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44. 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 191550+0000 mgr.y (mgr.24961) 15 : 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 2023-06-10T21:04: 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 44.191714+0000 mgr.y (mgr.24961 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: ) 16 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 06-10T21:04:44. 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 447726+0000 mon.a (mon.0) 913 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cluster 2023- 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 06-10T21:04:44. 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 447797+0000 mon.a (mon.0) 914 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023 2023-06-10T21:04:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44. 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 458678+0000 mgr.y (mgr.24961) 17 : 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 2023-06-10T21:04: 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 44.458883+0000 mgr.y (mgr. 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 24961) 18 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:04:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44.746369 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: +0000 mon.a (mon.0) 915 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44.763445 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: +0000 mon.a (mon.0) 916 : 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44. 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 792482+0000 mon.a (mon.0) 917 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44. 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: 811237+0000 mon.a (mon.0) 918 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit 2023 2023-06-10T21:04:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: -06-10T21:04:44.817310 2023-06-10T21:04:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: +0000 mon.a (mon.0) 919 : 2023-06-10T21:04:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:45 smithi152 conmon[97365]: audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:45.660 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:04:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:44.918668+0000 mgr.y (mgr 2023-06-10T21:04:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[97224]: .24961) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:04:46.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:45.172976+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:04:46.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[97224]: cluster 2023-06-10T21:04:45.347279+0000 mgr.y (mgr.24961) 21 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:44.918668+0000 2023-06-10T21:04:46.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[107122]: mgr.y (mgr.24961) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:04:46.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:45.172976+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:04:46.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:46 smithi064 conmon[107122]: cluster 2023-06-10T21:04:45.347279+0000 mgr.y (mgr.24961) 21 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:46 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:44.918668+0000 mgr.y (mgr.24961) 19 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:04:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:46 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:45.172976+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:04:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:46 smithi152 conmon[97365]: cluster 2023-06-10T21:04:45.347279+0000 mgr.y (mgr.24961) 21 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:47.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:47 smithi152 conmon[97365]: audit 2023-06-10T21:04:46.584301+0000 mon.a (mon.0) 2023-06-10T21:04:47.391 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:47 smithi152 conmon[97365]: 920 : audit [DBG] from='client.? 172.21.15.64:0/1598478066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:47.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:47 smithi064 conmon[97224]: audit 2023-06-10T21:04:46.584301+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.64:0/1598478066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:47 smithi064 conmon[107122]: audit 2023-06-10T21:04:46.584301+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.64:0/1598478066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:48.153 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:47 smithi152 systemd[1]: Stopping Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:48.154 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:47 smithi152 bash[146067]: Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-prometheus.a found: no such container 2023-06-10T21:04:48.163 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' created 2023-06-10T21:04:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:48 smithi064 conmon[107122]: audit 2023-06-10T21:04:47.285857+0000 mon.a (mon.0) 921 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-06-10T21:04:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:48 smithi064 conmon[107122]: cluster 2023-06-10T21:04:47.347804+0000 2023-06-10T21:04:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:48 smithi064 conmon[107122]: mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:48.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:48 smithi064 conmon[97224]: audit 2023-06-10T21:04:47.285857+0000 mon.a (mon.0) 921 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-06-10T21:04:48.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:48 smithi064 conmon[97224]: cluster 2023-06-10T21:04:47.347804+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:04:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[97365]: 21:04:47.285857+0000 mon.a (mon.0) 921 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-06-10T21:04:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[97365]: cluster 2023-06-10T21:04:47.347804+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:48.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:04:48.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-10T21:04:48.445 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-10T21:04:48.446 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-10T21:04:48.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-10T21:04:48.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-10T21:04:48.447 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 conmon[140617]: ts=2023-06-10T21:04:48.173Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-10T21:04:48.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-06-10T21:04:49.158 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: audit 2023-06-10T21:04:48.154170+0000 mon.a (mon 2023-06-10T21:04:49.159 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: .0) 922 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-06-10T21:04:49.159 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: cluster 2023-06-10T21:04:48.154252+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-06-10T21:04:49.159 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 bash[146183]: ae230a4afbd61866b53a0bebde758aa04e8130051a2dc0281dfecb6de30eb699 2023-06-10T21:04:49.160 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-10T21:04:49.160 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 systemd[1]: Stopped Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:49.160 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:48 smithi152 systemd[1]: Starting Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:49.184 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:04:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: audit 2023-06-10T21:04:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: 2023-06-10T21:04:48.452085+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: audit 2023-06-10T21:04:48. 2023-06-10T21:04:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: 987880+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.64:0/2158136273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[97365]: 2023-06-10T21:04:49.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[97224]: audit 2023-06-10T21:04:48.154170+0000 mon.a (mon.0) 922 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-06-10T21:04:49.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[97224]: cluster 2023-06-10T21:04:48.154252+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-06-10T21:04:49.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[97224]: audit 2023-06-10T21:04:48.452085+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:49.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[97224]: audit 2023-06-10T21:04:48.987880+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.64:0/2158136273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: audit 2023-06-10T21:04:48.154170 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: +0000 mon.a (mon.0) 922 : audit [INF] from='client.? 172.21.15.64:0/2205801578' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: cluster 2023-06-10T21:04:48 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: .154252+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:04:49.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: 04:48.452085+0000 mon.c (mon.2) 100 2023-06-10T21:04:49.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:04:49.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: audit 2023-06-10T21:04 2023-06-10T21:04:49.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: :48.987880+0000 mon.c (mon.2) 101 : audit [DBG] 2023-06-10T21:04:49.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[107122]: from='client.? 172.21.15.64:0/2158136273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:49.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE Bus STOPPING 2023-06-10T21:04:49.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:04:49.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE Bus STOPPED 2023-06-10T21:04:49.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE Bus STARTING 2023-06-10T21:04:49.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE Serving on http://:::9283 2023-06-10T21:04:49.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:04:49 smithi064 conmon[171866]: [10/Jun/2023:21:04:49] ENGINE Bus STARTED 2023-06-10T21:04:49.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 bash[146517]: 2013ef6ec5fb6e41f76a46df16388bf8078907cd30dff62cb48b38ebf9ea89e1 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.537Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.537Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.537Z caller=main.go:518 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi152 (none))" 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.537Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.537Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.539Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-10T21:04:49.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.539Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.540Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 systemd[1]: Started Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.542Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.542Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.909µs 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.542Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-10T21:04:49.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:49 smithi152 conmon[146558]: ts=2023-06-10T21:04:49.752Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-06-10T21:04:50.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: cluster 2023-06-10T21:04:49.151821+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-06-10T21:04:50.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: cluster 2023-06-10T21:04:49.348228+0000 mgr.y (mgr.24961) 23 : cluster [DBG] pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:50.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:49.568898+0000 mon.a ( 2023-06-10T21:04:50.424 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: mon.0) 925 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: 2023-06-10T21:04:49.575980+0000 mon.a 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: (mon.0) 926 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:49.603912+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:49.604465+0000 mgr.y (mgr.24961) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.425 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: :49.605340+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: 2023-06-10T21:04:49.605702+0000 mgr.y (mgr.24961) 25 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:49.610099+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[107122]: audit 2023-06-10T21:04:49.625257+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: cluster 2023-06-10T21:04:49.151821+0000 mon.a (mon 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: .0) 924 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: cluster 2023-06-10T21:04:49.348228+0000 mgr.y ( 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: mgr.24961) 23 : cluster [DBG] pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04:50.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 2023-06-10T21:04:49.568898+0000 mon.a (mon.0) 925 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04:49.575980+0000 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: mon.a (mon.0) 926 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04:49.603912+0000 mon.c 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: (mon.2) 102 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04:50.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 2023-06-10T21:04:49.604465+0000 mgr.y (mgr.24961) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04: 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 49.605340+0000 mon.c (mon.2) 103 : audit 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21:04:49.605702 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: +0000 mgr.y (mgr.24961) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.429 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T 2023-06-10T21:04:50.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 21:04:49.610099+0000 mon.a (mon. 2023-06-10T21:04:50.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 0) 927 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:04:50.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: 04:49.625257+0000 mon.c (mon.2) 104 2023-06-10T21:04:50.430 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:50 smithi064 conmon[97224]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:50.432 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.123Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-06-10T21:04:50.433 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.330Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-06-10T21:04:50.433 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.330Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-06-10T21:04:50.433 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.330Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=52.004µs wal_replay_duration=788.529103ms total_replay_duration=788.596841ms 2023-06-10T21:04:50.433 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.335Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-10T21:04:50.433 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.335Z caller=main.go:947 level=info msg="TSDB started" 2023-06-10T21:04:50.434 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.335Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-10T21:04:50.434 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.356Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.569818ms db_storage=760ns remote_storage=1.616µs web_handler=385ns query_engine=604ns scrape=767.09µs scrape_sd=81.661µs notify=15.529µs notify_sd=9.567µs rules=20.339599ms 2023-06-10T21:04:50.434 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[146558]: ts=2023-06-10T21:04:50.356Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-10T21:04:50.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: cluster 2023-06-10T21:04:49.151821+0000 mon.a (mon. 2023-06-10T21:04:50.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 0) 924 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-06-10T21:04:50.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: cluster 2023-06-10T21:04:49.348228+0000 mgr.y (mgr.24961) 23 : cluster [DBG] pgmap v14: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 730 MiB used, 715 GiB / 715 GiB avail 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 2023-06-10T21:04:49.568898+0000 mon.a (mon.0) 925 : audit [INF] 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 2023-06-10T21: 2023-06-10T21:04:50.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 04:49.575980+0000 mon.a (mon. 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 0) 926 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 2023-06-10T21:04:49.603912 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: +0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 21:04:49.604465+0000 mgr.y (mgr.24961) 24 : 2023-06-10T21:04:50.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T21:04:49.605340+0000 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: mon.c (mon.2) 103 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 10T21:04:49.605702+0000 mgr.y ( 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: mgr.24961) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 2023-06-10T21:04:50.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023-06-10T21:04:49 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: .610099+0000 mon.a (mon.0) 927 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: 06-10T21:04:49.625257 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: +0000 mon.c (mon.2) 104 : audit 2023-06-10T21:04:50.438 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:50 smithi152 conmon[97365]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:51.176 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-06-10T21:04:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: cluster 2023-06-10T21:04:50.166650+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:50.797571+0000 mon.c (mon.2) 105 : audit [INF] from='client.? 172.21.15.64:0/1863061151' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:50.797933+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.144913+0000 mon.c (mon.2) 106 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.145231+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.146196+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:51.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:51 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.147008+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: cluster 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: 2023-06-10T21:04:50.166650+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023-06-10T21:04:50.797571+0000 mon.c (mon.2) 105 : audit [INF] from='client.? 172.21.15.64:0/1863061151' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023-06-10T21:04:50.797933+0000 mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023-06-10T21:04:51. 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: 144913+0000 mon.c (mon.2) 106 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023- 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: 06-10T21:04:51.145231+0000 mon.a (mon.0 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: ) 930 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: 2023-06-10T21:04: 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: 51.146196+0000 mon.c (mon.2) 107 2023-06-10T21:04:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: audit 2023-06-10 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: T21:04:51.147008+0000 mon.c ( 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[97224]: mon.2) 108 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: cluster 2023-06-10T21:04:50 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: .166650+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-06-10T21:04:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06-10T21:04:50.797571+0000 mon.c (mon.2) 105 : audit [INF] from='client.? 172.21.15.64:0/1863061151' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06-10T21:04:50.797933+0000 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: mon.a (mon.0) 929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: 2023-06-10T21:04:51 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: .144913+0000 mon.c (mon.2) 106 : audit 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06-10T 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: 21:04:51.145231+0000 mon.a (mon.0 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: ) 930 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: 10T21:04:51.146196+0000 mon.c ( 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: mon.2) 107 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:04:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: -10T21:04:51.147008+0000 mon.c 2023-06-10T21:04:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: (mon.2) 108 : audit 2023-06-10T21:04:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:51 smithi064 conmon[107122]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:04:52.415 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:51.144468+0000 mgr.y (mgr.24961) 26 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-10T21:04:52.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: cephadm 2023-06-10T21:04:51.147806+0000 mgr.y (mgr.24961) 27 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:04:52.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.169339+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:04:52.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: cluster 2023-06-10T21:04:51.169445+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-10T21:04:52.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: cluster 2023-06-10T21:04:51.348608+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v17: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:52.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:52 smithi152 conmon[97365]: audit 2023-06-10T21:04:51.397968+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.64:0/4216904647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:51.144468+0000 mgr.y (mgr.24961) 26 : cephadm 2023-06-10T21:04:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: [INF] Upgrade: Updating mgr.x 2023-06-10T21:04:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: cephadm 2023-06-10T21:04:51.147806+0000 mgr.y (mgr.24961) 27 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:04:52.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: audit 2023-06-10T21:04:51.169339+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: cluster 2023-06-10T21:04:51.169445+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: cluster 2023-06-10T21:04:51.348608+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v17: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[97224]: audit 2023-06-10T21:04:51.397968+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.64:0/4216904647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:51.144468+0000 mgr.y (mgr. 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: 24961) 26 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-10T21:04:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: cephadm 2023-06-10T21:04:51.147806+0000 mgr.y (mgr.24961) 27 : cephadm [INF] Deploying daemon mgr.x on smithi152 2023-06-10T21:04:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: audit 2023-06-10T21:04:51.169339+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:04:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: cluster 2023-06-10T21:04:51.169445+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-10T21:04:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: cluster 2023-06-10T21:04:51.348608+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v17: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:52.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:52 smithi064 conmon[107122]: audit 2023-06-10T21:04:51.397968+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.64:0/4216904647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:52.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:52 smithi152 systemd[1]: Stopping Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:52.869 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-06-10T21:04:52.872 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:04:52.872 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:04:52.927 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:04:52.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:04:52.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_179001 2023-06-10T21:04:52.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:04:52.929 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:04:53.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:53 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:04:53.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:53 smithi152 bash[147196]: b20b3f89b2be3e06d3d44bb6701a50abab2868e77b4e449c37fe1d8535f68f39 2023-06-10T21:04:53.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3175 3159 197.412 197.438 0.00298149 0.00504338 2023-06-10T21:04:54.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:53 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2023-06-10T21:04:54.196 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:53 smithi152 systemd[1]: Stopped Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:54.196 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:53 smithi152 systemd[1]: Starting Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:04:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:54 smithi152 conmon[97365]: cluster 2023-06-10T21:04:53.349040+0000 2023-06-10T21:04:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:54 smithi152 conmon[97365]: mgr.y (mgr.24961) 29 : cluster [DBG] pgmap v18: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:54 smithi152 conmon[97365]: audit 2023-06-10T21:04:54.196371+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.64:0/2664299715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:54.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[97224]: cluster 2023-06-10T21:04:53.349040+0000 mgr.y (mgr.24961) 29 : cluster 2023-06-10T21:04:54.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[97224]: [DBG] pgmap v18: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:54.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[97224]: audit 2023-06-10T21:04:54.196371+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.64:0/2664299715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:54.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[107122]: cluster 2023-06-10T21:04:53.349040+0000 2023-06-10T21:04:54.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[107122]: mgr.y (mgr.24961) 29 : cluster [DBG] pgmap v18: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2023-06-10T21:04:54.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:54 smithi064 conmon[107122]: audit 2023-06-10T21:04:54.196371+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.64:0/2664299715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:54.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 15 6571 6556 204.846 212.312 0.00362913 0.0048718 2023-06-10T21:04:55.151 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:54 smithi152 bash[147536]: 0e93fb4f51a4c3d1986bde05b103250156734856f8455663aad32c980d50127f 2023-06-10T21:04:55.152 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:54 smithi152 systemd[1]: Started Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:04:55.152 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:54 smithi152 conmon[147575]: 2023-06-10T21:04:54.988+0000 7f6119cd6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:04:55.152 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[147575]: 2023-06-10T21:04:55.147+0000 7f6119cd6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:04:55.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[147575]: 2023-06-10T21:04:55.318+0000 7f6119cd6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:04:55.830 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[147575]: 2023-06-10T21:04:55.496+0000 7f6119cd6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:04:55.830 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[147575]: 2023-06-10T21:04:55.814+0000 7f6119cd6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:04:55.929 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9583 9567 199.288 188.188 0.00568952 0.00500902 2023-06-10T21:04:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: audit 2023-06-10T21:04:54.852783+0000 2023-06-10T21:04:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: mon.a (mon.0) 935 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: audit 2023-06-10T21:04:54.863458+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: audit 2023-06-10T21: 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: 04 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: :54.871954+0000 mon.c (mon.2) 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: 109 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[97224]: cluster 2023-06-10T21:04:55.349809+0000 mgr.y (mgr.24961) 30 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: audit 2023-06-10T21:04:54 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: .852783+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: audit 2023-06-10T21:04:54.863458+0000 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: mon.a (mon.0) 936 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: audit 2023-06-10T21:04:54 2023-06-10T21:04:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: .871954+0000 mon.c (mon.2) 109 : audit [DBG] 2023-06-10T21:04:56.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:56.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: cluster 2023-06-10T21:04:56.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: 2023-06-10T21:04:55.349809+0000 mgr.y (mgr 2023-06-10T21:04:56.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:55 smithi064 conmon[107122]: .24961) 30 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-06-10T21:04:56.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: audit 2023-06-10T21:04:54.852783+0000 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: mon.a (mon.0) 935 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: audit 2023-06-10T21:04:54.863458+0000 mon.a (mon.0 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: ) 936 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: audit 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: 2023-06-10T21:04:54.871954+0000 mon.c (mon.2) 109 2023-06-10T21:04:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:04:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: cluster 2023 2023-06-10T21:04:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: -06-10T21:04:55.349809+0000 mgr.y ( 2023-06-10T21:04:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:55 smithi152 conmon[97365]: mgr.24961) 30 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-06-10T21:04:56.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:56 smithi152 conmon[147575]: 2023-06-10T21:04:56.373+0000 7f6119cd6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:04:56.696 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:56 smithi152 conmon[147575]: 2023-06-10T21:04:56.532+0000 7f6119cd6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:04:56.929 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 12322 12306 192.211 171.188 0.00602696 0.00519237 2023-06-10T21:04:57.133 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:56 smithi152 conmon[147575]: 2023-06-10T21:04:56.763+0000 7f6119cd6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:04:57.133 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:56 smithi152 conmon[147575]: 2023-06-10T21:04:56.929+0000 7f6119cd6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:04:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:57 smithi064 conmon[107122]: audit 2023-06-10T21:04:56.973125 2023-06-10T21:04:57.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:57 smithi064 conmon[107122]: +0000 mon.a (mon.0) 937 : audit [DBG] from='client.? 172.21.15.64:0/550281129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:57.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:57 smithi064 conmon[97224]: audit 2023-06-10T21:04:56.973125+0000 mon.a (mon. 2023-06-10T21:04:57.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:57 smithi064 conmon[97224]: 0) 937 : audit [DBG] from='client.? 172.21.15.64:0/550281129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:57 smithi152 conmon[97365]: audit 2023-06-10T21:04:56.973125+0000 mon.a (mon.0) 937 : audit 2023-06-10T21:04:57.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:57 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/550281129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:04:57.930 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15019 15003 187.47 168.562 0.00957142 0.0053268 2023-06-10T21:04:58.147 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:57 smithi152 conmon[147575]: 2023-06-10T21:04:57.694+0000 7f6119cd6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:04:58.148 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:04:58 smithi152 conmon[97365]: cluster 2023-06-10T21:04:57.350332+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s 2023-06-10T21:04:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:04:58 smithi064 conmon[107122]: cluster 2023-06-10T21:04:57.350332+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s 2023-06-10T21:04:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:58 smithi064 conmon[97224]: cluster 2023-06-10 2023-06-10T21:04:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:04:58 smithi064 conmon[97224]: T21:04:57.350332+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 MiB/s wr, 594 op/s 2023-06-10T21:04:58.930 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17509 17493 182.159 155.625 0.00567131 0.00548367 2023-06-10T21:04:59.012 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:58 smithi152 conmon[147575]: 2023-06-10T21:04:58.713+0000 7f6119cd6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:04:59.013 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:04:59 smithi152 conmon[147575]: 2023-06-10T21:04:59.007+0000 7f6119cd6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:04:59.931 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 19830 19814 176.847 145.062 0.0102374 0.00564699 2023-06-10T21:05:00.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[147575]: 2023-06-10T21:05:00.326+0000 7f6119cd6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:00.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[147575]: 2023-06-10T21:05:00.486+0000 7f6119cd6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:05:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[97365]: cluster 2023-06-10T21:04:59.350919+0000 mgr.y (mgr.24961) 32 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s 2023-06-10T21:05:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[97365]: audit 2023-06-10T21:04:59.739575+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.64:0/730393999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[97365]: audit 2023-06-10T21:05:00.087323+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[97365]: audit 2023-06-10T21:05:00.099924+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[97224]: cluster 2023-06-10T21:04:59.350919+0000 mgr.y (mgr.24961) 32 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s 2023-06-10T21:05:00.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[97224]: audit 2023-06-10T21:04:59.739575+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.64:0/730393999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[97224]: audit 2023-06-10T21:05:00.087323+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[97224]: audit 2023-06-10T21:05:00.099924+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: cluster 2023-06-10T21:04:59.350919+0000 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: mgr.y (mgr.24961) 32 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 298 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: audit 2023-06-10T21:05:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: 2023-06-10T21:04:59.739575+0000 mon.b (mon.1) 111 : audit [DBG] from='client.? 172.21.15.64:0/730393999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:00.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: audit 2023-06-10T21:05:00.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: 2023-06-10T21:05:00.087323+0000 mon.a 2023-06-10T21:05:00.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: (mon.0) 938 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: audit 2023-06-10T21:05:00.099924+0000 mon.a (mon.0) 939 : audit [INF] 2023-06-10T21:05:00.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:00 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:00.931 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 22117 22101 172.607 142.938 0.00342615 0.00578576 2023-06-10T21:05:01.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:00 smithi152 conmon[147575]: 2023-06-10T21:05:00.803+0000 7f6119cd6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:01.676 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:01 smithi152 conmon[147575]: 2023-06-10T21:05:01.304+0000 7f6119cd6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:05:01.931 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 24306 24290 168.629 136.812 0.00616769 0.00592351 2023-06-10T21:05:01.932 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:01 smithi152 conmon[147575]: 2023-06-10T21:05:01.672+0000 7f6119cd6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:05:01.932 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:01 smithi152 conmon[147575]: 2023-06-10T21:05:01.928+0000 7f6119cd6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:05:02.690 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[147575]: 2023-06-10T21:05:02.393+0000 7f6119cd6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:05:02.690 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[97365]: cluster 2023-06-10T21:05:01.351778+0000 mgr.y (mgr.24961) 33 2023-06-10T21:05:02.690 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[97365]: : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-06-10T21:05:02.691 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[97365]: audit 2023-06-10T21:05:01.413956+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.691 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[97365]: audit 2023-06-10T21:05:01.425337+0000 mon.a (mon.0) 2023-06-10T21:05:02.691 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:02 smithi152 conmon[97365]: 941 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[107122]: cluster 2023-06-10T21:05:01.351778+0000 mgr.y (mgr.24961) 33 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-06-10T21:05:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[107122]: 2023-06-10T21:05:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[107122]: audit 2023-06-10T21:05:01.413956+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[107122]: audit 2023-06-10T21:05:01.425337+0000 mon.a 2023-06-10T21:05:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[107122]: (mon.0) 941 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.887 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[97224]: cluster 2023-06-10T21:05:01.351778+0000 mgr.y (mgr.24961) 33 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-06-10T21:05:02.887 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[97224]: audit 2023-06-10T21:05:01.413956+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[97224]: audit 2023-06-10T21:05:01.425337+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:02.933 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 26392 26376 164.765 130.375 0.00383495 0.00606417 2023-06-10T21:05:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:05:02] "GET /metrics HTTP/1.1" 200 190003 "" "Prometheus/2.33.4" 2023-06-10T21:05:03.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:03 smithi152 conmon[147575]: 2023-06-10T21:05:03.541+0000 7f6119cd6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:05:03.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:03 smithi152 conmon[97365]: audit 2023-06-10T21:05:02.493656+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.64:0/2240293543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:03 smithi064 conmon[97224]: audit 2023-06-10T21:05:02.493656+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.64:0/2240293543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:03 smithi064 conmon[107122]: audit 2023-06-10T21:05:02.493656+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.64:0/2240293543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:03.933 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 28568 28552 162.15 136 0.00827452 0.00616137 2023-06-10T21:05:04.101 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:03 smithi152 conmon[147575]: 2023-06-10T21:05:03.771+0000 7f6119cd6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:05:04.102 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[147575]: 2023-06-10T21:05:04.097+0000 7f6119cd6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:05:04.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[147575]: 2023-06-10T21:05:04.297+0000 7f6119cd6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:05:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[97224]: cluster 2023-06-10T21:05:03.352932+0000 mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:05:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[97224]: audit 2023-06-10T21:05:03.443247+0000 mon.a (mon.0) 942 2023-06-10T21:05:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[97224]: : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[97224]: audit 2023-06-10T21:05:03.453975+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[107122]: cluster 2023-06-10T21:05:03.352932+0000 mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:05:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[107122]: audit 2023-06-10T21:05:03.443247+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:04 smithi064 conmon[107122]: audit 2023-06-10T21:05:03.453975+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:04.933 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 30581 30565 159.122 125.812 0.0085625 0.00627824 2023-06-10T21:05:04.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[147575]: 2023-06-10T21:05:04.647+0000 7f6119cd6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:05:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: cluster 2023-06-10T21:05:03.352932+0000 2023-06-10T21:05:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:05:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: audit 2023-06-10T21:05:03.443247+0000 mon.a (mon.0) 942 : 2023-06-10T21:05:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:05:04.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: 21:05:03.453975+0000 mon.c (mon.2) 111 : audit [DBG] 2023-06-10T21:05:04.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:04 smithi152 conmon[97365]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:05.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:05 smithi064 conmon[97224]: audit 2023-06-10T21:05:05.224169 2023-06-10T21:05:05.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:05 smithi064 conmon[97224]: +0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.64:0/2068784337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:05.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:05 smithi064 conmon[107122]: audit 2023-06-10T21:05:05.224169+0000 2023-06-10T21:05:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:05 smithi064 conmon[107122]: mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.64:0/2068784337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:05.933 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 32437 32421 155.805 116 0.00799742 0.0064127 2023-06-10T21:05:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:05 smithi152 conmon[97365]: audit 2023-06-10T21:05:05.224169+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.64:0/2068784337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:06.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:05 smithi152 conmon[147575]: 2023-06-10T21:05:05.970+0000 7f6119cd6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:05:06.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:05 smithi152 conmon[147575]: [10/Jun/2023:21:05:05] ENGINE Bus STARTING 2023-06-10T21:05:06.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[147575]: CherryPy Checker: 2023-06-10T21:05:06.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[147575]: The Application mounted at '' has an empty config. 2023-06-10T21:05:06.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[147575]: 2023-06-10T21:05:06.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[147575]: [10/Jun/2023:21:05:06] ENGINE Serving on http://:::9283 2023-06-10T21:05:06.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[147575]: [10/Jun/2023:21:05:06] ENGINE Bus STARTED 2023-06-10T21:05:06.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: cluster 2023-06-10T21:05:05.353752+0000 mgr.y (mgr.24961) 35 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-10T21:05:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: cluster 2023-06-10T21:05:05.986198+0000 mon.a (mon.0) 943 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:05:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: cluster 2023-06-10T21:05:05.986391+0000 mon.a (mon.0) 944 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:05:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: audit 2023-06-10T21:05:05.988538+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:05:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: audit 2023-06-10T21:05:05.989016+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:05:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: audit 2023-06-10T21:05:05.993081+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: audit 2023-06 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[97224]: -10T21:05:05.993951+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: cluster 2023-06-10T21:05:05.353752+0000 mgr.y 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: (mgr.24961) 35 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: cluster 2023-06-10T21:05:05.986198+0000 mon.a (mon.0) 2023-06-10T21:05:06.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 943 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: cluster 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 2023-06-10T21:05:05.986391+0000 mon.a (mon.0 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: ) 944 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: audit 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 2023-06-10T21:05:05.988538+0000 mon.b 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: (mon.1) 112 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:05:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: audit 2023-06 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: -10T21:05:05.989016+0000 mon.b (mon 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: .1) 113 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: audit 2023-06- 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 10T21:05:05.993081+0000 mon.b (mon. 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 1) 114 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:05:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: T21:05:05.993951+0000 mon.b (mon. 2023-06-10T21:05:06.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:06 smithi064 conmon[107122]: 1) 115 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:05:06.933 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 34381 34365 153.355 121.5 0.00589558 0.0065155 2023-06-10T21:05:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: cluster 2023-06-10T21:05:05.353752+0000 mgr.y (mgr.24961) 35 : cluster 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: [DBG] pgmap v24: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: cluster 2023-06-10T21:05:05.986198+0000 mon.a (mon.0) 943 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: cluster 2023-06-10T21:05:05.986391+0000 mon.a (mon.0 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: ) 944 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: audit 2023-06-10T21:05:05.988538+0000 2023-06-10T21:05:06.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: mon.b (mon.1) 112 : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: 05:05.989016+0000 mon.b (mon.1) 113 : audit 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: audit 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: 2023-06-10T21:05:05.993081+0000 mon.b (mon.1) 114 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: : audit [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: audit 2023-06-10T21:05: 2023-06-10T21:05:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: 05.993951+0000 mon.b (mon.1) 115 : audit 2023-06-10T21:05:06.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:06 smithi152 conmon[97365]: [DBG] from='mgr.? 172.21.15.152:0/4134105234' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:05:07.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:07 smithi152 conmon[97365]: cluster 2023-06-10T21:05:06.483930+0000 mon.a (mon.0) 945 : cluster [DBG] mgrmap e32: y(active, since 33s), standbys: x 2023-06-10T21:05:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:07 smithi064 conmon[107122]: cluster 2023-06-10T21:05:06.483930+0000 2023-06-10T21:05:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:07 smithi064 conmon[107122]: mon.a (mon.0) 945 : cluster [DBG] mgrmap e32: y(active, since 33s), standbys: x 2023-06-10T21:05:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:07 smithi064 conmon[97224]: cluster 2023-06-10T21:05:06.483930+0000 mon.a ( 2023-06-10T21:05:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:07 smithi064 conmon[97224]: mon.0) 945 : cluster [DBG] mgrmap e32: y(active, since 33s), standbys: x 2023-06-10T21:05:07.936 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 36194 36178 150.68 113.312 0.0072046 0.00663108 2023-06-10T21:05:08.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[97224]: cluster 2023-06-10T21:05:07.354695+0000 mgr.y (mgr.24961) 36 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[97224]: audit 2023-06-10T21:05:07.950166+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.64:0/3906122290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[107122]: cluster 2023-06-10T21:05:07 2023-06-10T21:05:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[107122]: .354695+0000 mgr.y (mgr.24961) 36 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:08.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[107122]: audit 2023-06-10T21:05:08.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:08 smithi064 conmon[107122]: 2023-06-10T21:05:07.950166+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.64:0/3906122290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:08.934 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 38085 38069 148.649 118.188 0.00421794 0.00672191 2023-06-10T21:05:08.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:08 smithi152 conmon[146558]: ts=2023-06-10T21:05:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:08.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:08 smithi152 conmon[146558]: ts=2023-06-10T21:05:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:08 smithi152 conmon[97365]: cluster 2023-06-10T21:05:07.354695+0000 mgr.y (mgr.24961) 36 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:08 smithi152 conmon[97365]: audit 2023-06-10T21:05:07.950166+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.64:0/3906122290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:09.937 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 39836 39820 146.32 109.438 0.00736298 0.00682835 2023-06-10T21:05:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: cluster 2023-06-10T21:05:09.355139+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.073643+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.088438+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.097606+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.108670+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: T21:05:10.118449+0000 mon.a (mon. 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 0) 949 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.327476+0000 mon.c (mon.2) 115 : audit 2023-06-10T21:05:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05: 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 10.337054+0000 mon.c (mon.2) 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 116 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: -06-10T21:05:10. 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 348848+0000 mon.a (mon.0) 950 : audit 2023-06-10T21:05:10.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: T21:05:10.356874+0000 mon.c 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: (mon.2) 117 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: :10.357388+0000 mon.a (mon.0 2023-06-10T21:05:10.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: ) 951 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: -06-10T21:05:10.364598 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: +0000 mon.a (mon.0) 952 : audit 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 05:10.373056+0000 mon.c (mon.2 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: ) 118 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: -06-10T21:05:10.373467+0000 mon.a 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: (mon.0) 953 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 2023-06-10T21:05:10.381573+0000 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: mon.a (mon.0) 954 : audit [INF] 2023-06-10T21:05:10.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: T21:05:10.393099+0000 mon.c 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: (mon.2) 119 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: audit 2023-06-10T21 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: :05:10.394346+0000 mon.c (mon 2023-06-10T21:05:10.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[107122]: .2) 120 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-10T21:05:10.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: cluster 2023-06-10T21:05:09.355139+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:10.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.073643+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.088438+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.097606+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:10.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.108670+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.118449+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.327476+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.337054+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.348848+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.356874+0000 mon.c (mon.2) 117 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.357388+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.364598+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-10T21:05:10.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.373056+0000 mon.c (mon.2) 118 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.373467+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.381573+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-10T21:05:10.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.393099+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:05:10.917 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:10 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.394346+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-10T21:05:10.939 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 41316 41300 143.313 92.5 0.0132793 0.00697255 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: cluster 2023-06-10T21:05:09.355139+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 1.8 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.073643+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.088438+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.097606+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.108670+0000 mon.c (mon.2) 114 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.118449+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.327476+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.337054+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.348848+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: -06-10T21:05:10.356874+0000 mon.c (mon.2) 117 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.357388+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.364598+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-10T21:05:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.373056+0000 mon.c (mon.2) 118 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: .373467+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10. 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: 381573+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: 2023-06-10 2023-06-10T21:05:10.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: T21:05:10.393099+0000 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: mon.c (mon.2) 119 : 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: audit 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: 2023-06-10T 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: 21:05:10.394346+0000 mon.c 2023-06-10T21:05:10.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: (mon.2) 120 : audit [DBG] 2023-06-10T21:05:10.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:10 smithi152 conmon[97365]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-10T21:05:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:10.338530+0000 mgr.y (mgr.24961) 38 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-10T21:05:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:10.395077+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-10T21:05:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[107122]: audit 2023-06-10T21:05:10.720267+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.64:0/1919845246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[97224]: cephadm 2023-06-10T21:05:10.338530+0000 mgr.y (mgr.24961) 38 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-10T21:05:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[97224]: cephadm 2023-06-10T21:05:10.395077+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-10T21:05:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:11 smithi064 conmon[97224]: audit 2023-06-10T21:05:10.720267+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.64:0/1919845246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:11.939 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 42934 42918 141.093 101.125 0.00852611 0.00708159 2023-06-10T21:05:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:11 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:10.338530+0000 mgr.y (mgr.24961) 38 2023-06-10T21:05:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:11 smithi152 conmon[97365]: : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-10T21:05:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:11 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:10.395077+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-10T21:05:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:11 smithi152 conmon[97365]: audit 2023-06-10T21:05: 2023-06-10T21:05:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:11 smithi152 conmon[97365]: 10.720267+0000 mon.c (mon.2) 121 : audit [DBG] from='client.? 172.21.15.64:0/1919845246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:12 smithi064 conmon[97224]: cluster 2023-06-10T21:05:11.356584+0000 mgr.y (mgr.24961) 40 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.85k op/s 2023-06-10T21:05:12.883 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:05:12] "GET /metrics HTTP/1.1" 200 190094 "" "Prometheus/2.33.4" 2023-06-10T21:05:12.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:12 smithi064 conmon[107122]: cluster 2023-06-10T21:05:11.356584+0000 mgr.y (mgr.24961) 40 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.85k op/s 2023-06-10T21:05:12.939 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:05:12.937279+0000 min lat: 0.00128113 max lat: 0.108764 avg lat: 0.0071706 2023-06-10T21:05:12.940 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:05:12.940 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 44639 44623 139.367 106.562 0.0114251 0.0071706 2023-06-10T21:05:12.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:12 smithi152 conmon[97365]: cluster 2023-06-10T21:05:11.356584+0000 mgr.y (mgr 2023-06-10T21:05:12.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:12 smithi152 conmon[97365]: .24961) 40 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 178 MiB/s wr, 2.85k op/s 2023-06-10T21:05:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:13 smithi064 conmon[97224]: audit 2023-06-10T21:05:13.463834+0000 mon.c ( 2023-06-10T21:05:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:13 smithi064 conmon[97224]: mon.2) 122 : audit [DBG] from='client.? 172.21.15.64:0/1333235783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:13.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:13 smithi064 conmon[107122]: audit 2023-06-10T21:05:13.463834+0000 2023-06-10T21:05:13.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:13 smithi064 conmon[107122]: mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.64:0/1333235783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:13.940 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 45748 45732 136.032 69.3125 0.00707917 0.00734624 2023-06-10T21:05:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:13 smithi152 conmon[97365]: audit 2023-06-10T21:05:13.463834+0000 mon.c (mon.2) 122 : audit [DBG] from='client.? 172.21.15.64:0/1333235783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:14.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: cluster 2023-06-10T21:05:13.357040+0000 mgr.y ( 2023-06-10T21:05:14.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: mgr.24961) 41 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-06-10T21:05:14.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: audit 2023-06-10T21:05:14.299802+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:14.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: audit 2023-06-10T21:05:14.309354+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:05:14.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: audit 2023-06-10T21:05:14.311031+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:05:14.545 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[107122]: audit 2023-06-10T21:05:14.314200+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:14.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[97224]: cluster 2023-06-10T21:05:13.357040+0000 mgr.y (mgr.24961) 41 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-06-10T21:05:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[97224]: audit 2023-06-10T21:05:14.299802+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[97224]: audit 2023-06-10T21:05:14.309354+0000 mon.c (mon.2) 123 : audit [INF] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:05:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[97224]: audit 2023-06-10T21:05:14.311031+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:05:14.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:14 smithi064 conmon[97224]: audit 2023-06-10T21:05:14.314200+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:14.942 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 47301 47285 134.244 97.0625 0.0125128 0.00744317 2023-06-10T21:05:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: cluster 2023-06-10T21:05: 2023-06-10T21:05:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: 13.357040+0000 mgr.y (mgr.24961) 41 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: audit 2023-06-10T21:05: 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: 14.299802+0000 mon.a ( 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: mon.0) 955 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: audit 2023-06-10T21:05 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: :14.309354+0000 mon.c (mon.2) 123 : audit [INF] 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:05:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: -10T21:05:14.311031+0000 mon.c (mon.2) 124 : audit 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: audit 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: 2023-06-10T21: 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: 05:14.314200+0000 mon.c (mon.2) 2023-06-10T21:05:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:14 smithi152 conmon[97365]: 125 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[97224]: cephadm 2023-06-10T21: 2023-06-10T21:05:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[97224]: 05:14.291805+0000 mgr.y (mgr.24961) 42 : cephadm [INF] Upgrade: Updating mon.a 2023-06-10T21:05:15.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[97224]: cephadm 2023-06-10T21:05:14.315915+0000 mgr.y (mgr.24961) 43 : cephadm [INF] Deploying daemon mon.a on smithi064 2023-06-10T21:05:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:14.291805+0000 2023-06-10T21:05:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[107122]: mgr.y (mgr.24961) 42 : cephadm [INF] Upgrade: Updating mon.a 2023-06-10T21:05:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:14.315915+0000 mgr.y (mgr.24961) 2023-06-10T21:05:15.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:15 smithi064 conmon[107122]: 43 : cephadm [INF] Deploying daemon mon.a on smithi064 2023-06-10T21:05:15.943 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 48750 48734 132.345 90.5625 0.00177122 0.00753794 2023-06-10T21:05:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:15 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:14.291805+0000 mgr.y 2023-06-10T21:05:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:15 smithi152 conmon[97365]: (mgr.24961) 42 : cephadm [INF] Upgrade: Updating mon.a 2023-06-10T21:05:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:15 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:14.315915+0000 mgr.y (mgr.24961) 43 : cephadm [INF] Deploying daemon mon.a on smithi064 2023-06-10T21:05:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[107122]: cluster 2023-06-10T21:05:15.360121 2023-06-10T21:05:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[107122]: +0000 mgr.y (mgr.24961) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-06-10T21:05:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[107122]: audit 2023-06-10T21:05:16.048241+0000 2023-06-10T21:05:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[107122]: mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.64:0/331079789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:16.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[97224]: cluster 2023-06-10T21:05:15.360121+0000 mgr.y (mgr.24961) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-06-10T21:05:16.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:16 smithi064 conmon[97224]: audit 2023-06-10T21:05:16.048241+0000 mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.64:0/331079789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:16.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:16 smithi064 systemd[1]: Stopping Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:05:16.943 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 49972 49956 130.014 76.375 0.0170835 0.00768588 2023-06-10T21:05:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:16 smithi152 conmon[97365]: cluster 2023-06-10T21:05:15.360121+0000 mgr.y (mgr.24961 2023-06-10T21:05:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:16 smithi152 conmon[97365]: ) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-06-10T21:05:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:16 smithi152 conmon[97365]: audit 2023-06-10T21:05:16.048241+0000 mon.b (mon.1) 116 : audit [DBG] from='client.? 172.21.15.64:0/331079789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:17.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:17 smithi064 conmon[97224]: debug 2023-06-10T21:05:17.228+0000 7f7c9eeee700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-10T21:05:17.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:17 smithi064 conmon[97224]: debug 2023-06-10T21:05:17.228+0000 7f7c9eeee700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-06-10T21:05:17.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:17 smithi064 bash[179868]: 27c52af4dbabdc27e40383bf71b2aee38f38993973229eb6a4ebc8d295de4232 2023-06-10T21:05:17.943 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 51436 51420 128.474 91.5 0.012466 0.00777804 2023-06-10T21:05:18.943 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 52620 52604 126.379 74 0.00645351 0.00790788 2023-06-10T21:05:18.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:18 smithi152 conmon[146558]: ts=2023-06-10T21:05:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:18.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:18 smithi152 conmon[146558]: ts=2023-06-10T21:05:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:18 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service: Succeeded. 2023-06-10T21:05:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:18 smithi064 systemd[1]: Stopped Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:05:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:18 smithi064 systemd[1]: Starting Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:05:19.943 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 53996 53980 124.884 86 0.01201 0.00800005 2023-06-10T21:05:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 bash[180233]: 6f8d9991dea01bfb33585ce34ac946675958e6d11fc02020bacb43172552ba19 2023-06-10T21:05:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 systemd[1]: Started Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:05:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: set uid:gid to 167:167 (ceph:ceph) 2023-06-10T21:05:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc), process ceph-mon, pid 7 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: pidfile_write: ignore empty --pid-file 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: load: jerasure load: lrc 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: RocksDB version: 7.9.2 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Git sha 0 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Compile date 2023-06-09 19:19:26 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: DB SUMMARY 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: DB Session ID: I5TT0KK9QDFCBTOFJ54L 2023-06-10T21:05:20.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: CURRENT file: CURRENT 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: IDENTITY file: IDENTITY 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: MANIFEST file: MANIFEST-000008 size: 981 Bytes 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 3133934 ; 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.error_if_exists: 0 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.create_if_missing: 0 2023-06-10T21:05:20.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.paranoid_checks: 1 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.flush_verify_memtable_count: 1 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.env: 0x5561c3db6720 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.fs: PosixFileSystem 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.info_log: 0x5561c6e0b3e0 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_file_opening_threads: 16 2023-06-10T21:05:20.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.statistics: (nil) 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.use_fsync: 0 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_log_file_size: 0 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.keep_log_file_num: 1000 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.recycle_log_file_num: 0 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_fallocate: 1 2023-06-10T21:05:20.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_mmap_reads: 0 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_mmap_writes: 0 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.use_direct_reads: 0 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.create_missing_column_families: 0 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.db_log_dir: 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.wal_dir: 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T21:05:20.167 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.advise_random_on_open: 1 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.db_write_buffer_size: 0 2023-06-10T21:05:20.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.write_buffer_manager: 0x5561c609ac80 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.rate_limiter: (nil) 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.wal_recovery_mode: 2 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enable_thread_tracking: 0 2023-06-10T21:05:20.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enable_pipelined_write: 0 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.unordered_write: 0 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.row_cache: None 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.wal_filter: None 2023-06-10T21:05:20.170 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_ingest_behind: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.two_write_queues: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.manual_wal_flush: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.wal_compression: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.atomic_flush: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T21:05:20.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.log_readahead_size: 0 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.best_efforts_recovery: 0 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.allow_data_in_errors: 0 2023-06-10T21:05:20.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.db_host_id: __hostname__ 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enforce_single_del_contracts: true 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_background_jobs: 2 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_background_compactions: -1 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_subcompactions: 1 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T21:05:20.173 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_total_wal_size: 0 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_open_files: -1 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bytes_per_sync: 0 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T21:05:20.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_readahead_size: 0 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_background_flushes: -1 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Compression algorithms supported: 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kZSTD supported: 0 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kXpressCompression supported: 0 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kLZ4HCCompression supported: 1 2023-06-10T21:05:20.175 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kZlibCompression supported: 1 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kSnappyCompression supported: 1 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kLZ4Compression supported: 1 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: kBZip2Compression supported: 0 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: DMutex implementation: pthread_mutex_t 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-10T21:05:20.176 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.merge_operator: 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_filter: None 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_filter_factory: None 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.sst_partitioner_factory: None 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5561c6e0b500) 2023-06-10T21:05:20.177 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-06-10T21:05:20.178 INFO:journalctl@ceph.mon.a.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T21:05:20.178 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T21:05:20.178 INFO:journalctl@ceph.mon.a.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_type: 0 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_index_type: 0 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_shortening: 1 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: checksum: 4 2023-06-10T21:05:20.179 INFO:journalctl@ceph.mon.a.smithi064.stdout: no_block_cache: 0 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache: 0x5561c611d350 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_options: 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: capacity : 536870912 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_shard_bits : 4 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: strict_capacity_limit : 0 2023-06-10T21:05:20.180 INFO:journalctl@ceph.mon.a.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_cache_compressed: (nil) 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: persistent_cache: (nil) 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size: 4096 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_size_deviation: 10 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_restart_interval: 16 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: index_block_restart_interval: 1 2023-06-10T21:05:20.181 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_block_size: 4096 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: partition_filters: 0 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: use_delta_encoding: 1 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: filter_policy: bloomfilter 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: whole_key_filtering: 1 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: verify_compression: 0 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: format_version: 5 2023-06-10T21:05:20.182 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_index_compression: 1 2023-06-10T21:05:20.183 INFO:journalctl@ceph.mon.a.smithi064.stdout: block_align: 0 2023-06-10T21:05:20.183 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_auto_readahead_size: 262144 2023-06-10T21:05:20.183 INFO:journalctl@ceph.mon.a.smithi064.stdout: prepopulate_block_cache: 0 2023-06-10T21:05:20.183 INFO:journalctl@ceph.mon.a.smithi064.stdout: initial_auto_readahead_size: 8192 2023-06-10T21:05:20.183 INFO:journalctl@ceph.mon.a.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2023-06-10T21:05:20.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.write_buffer_size: 33554432 2023-06-10T21:05:20.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_write_buffer_number: 2 2023-06-10T21:05:20.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression: NoCompression 2023-06-10T21:05:20.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression: Disabled 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.prefix_extractor: nullptr 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.num_levels: 7 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T21:05:20.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T21:05:20.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T21:05:20.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T21:05:20.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T21:05:20.186 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T21:05:20.187 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.level: 32767 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.strategy: 0 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.enabled: false 2023-06-10T21:05:20.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.target_file_size_base: 67108864 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T21:05:20.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T21:05:20.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T21:05:20.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T21:05:20.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-10T21:05:20.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.arena_block_size: 1048576 2023-06-10T21:05:20.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T21:05:20.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T21:05:20.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.disable_auto_compactions: 0 2023-06-10T21:05:20.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T21:05:20.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T21:05:20.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T21:05:20.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T21:05:20.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.table_properties_collectors: 2023-06-10T21:05:20.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.inplace_update_support: 0 2023-06-10T21:05:20.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T21:05:20.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.bloom_locality: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.max_successive_merges: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.paranoid_file_checks: 0 2023-06-10T21:05:20.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.force_consistency_checks: 1 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.report_bg_io_stats: 0 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.ttl: 2592000 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enable_blob_files: false 2023-06-10T21:05:20.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.min_blob_size: 0 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_file_size: 268435456 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_compression_type: NoCompression 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.enable_blob_garbage_collection: false 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-10T21:05:20.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.blob_file_starting_level: 0 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11331, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 29736972-00a5-43bf-90ff-5a4de7242d2f 2023-06-10T21:05:20.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431119911216, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431119923116, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 3019749, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11332, "largest_seqno": 12064, "table_properties": {"data_size": 3014609, "index_size": 3417, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 5608, "raw_average_key_size": 21, "raw_value_size": 3007018, "raw_average_value_size": 11565, "num_data_blocks": 152, "num_entries": 260, "num_filter_entries": 260, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1686431119, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "29736972-00a5-43bf-90ff-5a4de7242d2f", "db_session_id": "I5TT0KK9QDFCBTOFJ54L", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431119923260, "job": 1, "event": "recovery_finished"} 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5561c61b6000 2023-06-10T21:05:20.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: DB pointer 0x5561c61a4000 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** DB Stats ** 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:05:20.200 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:05:20.201 INFO:journalctl@ceph.mon.a.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-10T21:05:20.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: L0 1/0 2.88 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.8 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-06-10T21:05:20.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: L6 1/0 24.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-10T21:05:20.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: Sum 2/0 27.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.8 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-06-10T21:05:20.202 INFO:journalctl@ceph.mon.a.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 492.8 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-06-10T21:05:20.203 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.203 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** Compaction Stats [default] ** 2023-06-10T21:05:20.203 INFO:journalctl@ceph.mon.a.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:05:20.203 INFO:journalctl@ceph.mon.a.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T21:05:20.203 INFO:journalctl@ceph.mon.a.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 492.8 0.01 0.00 1 0.006 0 0 0.0 0.0 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T21:05:20.204 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: Cumulative compaction: 0.00 GB write, 100.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: Interval compaction: 0.00 GB write, 100.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache BinnedLRUCache@0x5561c611d350#7 capacity: 512.00 MB usage: 78.28 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2e-05 secs_since: 0 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: Block cache entry stats(count,size,portion): DataBlock(4,27.31 KB,0.00520945%) FilterBlock(2,6.81 KB,0.00129938%) IndexBlock(2,44.16 KB,0.00842214%) Misc(1,0.00 KB,0%) 2023-06-10T21:05:20.205 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: starting mon.a rank 0 at public addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] at bind addrs [v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???) e3 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).mds e7 new map 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).mds e7 print_map 2023-06-10T21:05:20.206 INFO:journalctl@ceph.mon.a.smithi064.stdout: e7 2023-06-10T21:05:20.207 INFO:journalctl@ceph.mon.a.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T21:05:20.207 INFO:journalctl@ceph.mon.a.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:05:20.207 INFO:journalctl@ceph.mon.a.smithi064.stdout: legacy client fscid: 1 2023-06-10T21:05:20.207 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.208 INFO:journalctl@ceph.mon.a.smithi064.stdout: Filesystem 'foo' (1) 2023-06-10T21:05:20.208 INFO:journalctl@ceph.mon.a.smithi064.stdout: fs_name foo 2023-06-10T21:05:20.208 INFO:journalctl@ceph.mon.a.smithi064.stdout: epoch 5 2023-06-10T21:05:20.208 INFO:journalctl@ceph.mon.a.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: created 2023-06-10T20:59:25.413290+0000 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: modified 2023-06-10T20:59:39.018592+0000 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: tableserver 0 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: root 0 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_timeout 60 2023-06-10T21:05:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout: session_autoclose 300 2023-06-10T21:05:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_file_size 1099511627776 2023-06-10T21:05:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout: required_client_features {} 2023-06-10T21:05:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure 0 2023-06-10T21:05:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout: last_failure_osd_epoch 0 2023-06-10T21:05:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:05:20.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: max_mds 1 2023-06-10T21:05:20.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: in 0 2023-06-10T21:05:20.211 INFO:journalctl@ceph.mon.a.smithi064.stdout: up {0=24548} 2023-06-10T21:05:20.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: failed 2023-06-10T21:05:20.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: damaged 2023-06-10T21:05:20.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: stopped 2023-06-10T21:05:20.212 INFO:journalctl@ceph.mon.a.smithi064.stdout: data_pools [4] 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: metadata_pool 3 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: inline_data disabled 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: balancer 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: bal_rank_mask -1 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: standby_count_wanted 1 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi064.wuuunt{0:24548} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:05:20.213 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: Standby daemons: 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout: [mds.foo.smithi152.ciyxdy{-1:24553} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:05:20.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:05:20.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-06-10T21:05:20.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).mgr e0 loading version 32 2023-06-10T21:05:20.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).mgr e32 active server: [v2:172.21.15.64:6800/1079194554,v1:172.21.15.64:6801/1079194554](24961) 2023-06-10T21:05:20.215 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:05:19 smithi064 ceph-mon[180297]: mon.a@-1(???).mgr e32 mkfs or daemon transitioned to available, loading commands 2023-06-10T21:05:20.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[171866]: ignoring --setuser ceph since I am not root 2023-06-10T21:05:20.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[171866]: ignoring --setgroup ceph since I am not root 2023-06-10T21:05:20.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:20 smithi064 ceph-mgr[171892]: -- 172.21.15.64:0/2025442071 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562351b151e0 con 0x562351aec800 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:17.360583+0000 mgr.y ( 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: mgr.24961) 45 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: audit 2023-06-10T21:05:18.454894+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: audit 2023-06-10T21:05:18.661135+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.64:0/816058925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:19.361026+0000 mgr.y (mgr.24961) 46 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:05:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.555565+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.570711+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.583606+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.584403+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.584441+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.586034+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 48s), standbys: x 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.620792+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e33: y(active, since 48s), standbys: x 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: cluster 2023-06-10T21:05:20.620836+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-06-10T21:05:20.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[107122]: audit 2023-06-10T21:05:20.620932+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.24961 ' entity='' 2023-06-10T21:05:20.947 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[147575]: ignoring --setuser ceph since I am not root 2023-06-10T21:05:20.948 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[147575]: ignoring --setgroup ceph since I am not root 2023-06-10T21:05:20.948 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[147575]: 2023-06-10T21:05:20.839+0000 7f8f8e4d7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:05:20.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21:05:17.360583 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: +0000 mgr.y (mgr.24961) 45 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: audit 2023-06-10T21:05:18.454894+0000 mon.c (mon.2) 126 : audit [DBG] from='mgr.24961 172.21.15.64:0/1819055587' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: audit 2023-06-10T21:05:18.661135+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.64:0/816058925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21:05:19.361026+0000 mgr.y (mgr.24961) 46 : cluster [DBG] 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: pgmap v31: 89 pgs: 89 active+clean; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:05:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21:05:20.555565+0000 mon.a (mon.0) 1 : cluster [INF] 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: mon.a calling monitor election 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: -10T21:05:20.570711+0000 mon.a (mon.0) 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21:05:20.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 2023-06-10T21:05:20.583606+0000 mon.a (mon.0) 3 : cluster 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21:05: 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 20.584403+0000 mon.a (mon.0) 4 : cluster 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: :05:20.584441+0000 mon.a (mon.0) 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 5 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-10T21:05:20.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: :05:20.586034+0000 mon.a (mon.0) 6 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: : cluster [DBG] mgrmap e32: y(active, since 48s), standbys: x 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10T21: 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 05:20.620792+0000 mon.a (mon.0) 7 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: : cluster [DBG] mgrmap e33: y(active, since 48s), standbys: x 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: cluster 2023-06-10 2023-06-10T21:05:20.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: T21:05:20.620836+0000 mon.a (mon. 2023-06-10T21:05:20.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 0) 8 : cluster [INF] overall HEALTH_OK 2023-06-10T21:05:20.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:05:20.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 10T21:05:20.620932+0000 mon.a (mon. 2023-06-10T21:05:20.953 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[97365]: 0) 9 : audit [INF] from='mgr.24961 ' entity='' 2023-06-10T21:05:20.953 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 55372 55356 123.463 86 0.0162041 0.00809385 2023-06-10T21:05:21.224 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:20 smithi152 conmon[147575]: 2023-06-10T21:05:20.960+0000 7f8f8e4d7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:05:21.224 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[147575]: 2023-06-10T21:05:21.086+0000 7f8f8e4d7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:05:21.224 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[147575]: 2023-06-10T21:05:21.219+0000 7f8f8e4d7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:05:21.279 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:20 smithi064 conmon[171866]: 2023-06-10T21:05:20.933+0000 7f6d426ce200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-10T21:05:21.279 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[171866]: 2023-06-10T21:05:21.094+0000 7f6d426ce200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-10T21:05:21.279 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[171866]: 2023-06-10T21:05:21.271+0000 7f6d426ce200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-10T21:05:21.504 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 1 2023-06-10T21:05:21.664 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[171866]: 2023-06-10T21:05:21.462+0000 7f6d426ce200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-10T21:05:21.678 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[147575]: 2023-06-10T21:05:21.465+0000 7f8f8e4d7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:05:21.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[97365]: audit 2023-06-10T21:05:21.465944+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.64:0/1775509512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:21.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[97365]: cluster 2023-06-10T21:05:21.623579+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e34: y(active, since 49s), standbys: x 2023-06-10T21:05:21.944 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:21 smithi152 conmon[147575]: 2023-06-10T21:05:21.905+0000 7f8f8e4d7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:05:21.950 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 56506 56490 121.651 70.875 0.00743805 0.00821475 2023-06-10T21:05:22.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[171866]: 2023-06-10T21:05:21.802+0000 7f6d426ce200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-10T21:05:22.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[107122]: audit 2023-06-10T21:05:21 2023-06-10T21:05:22.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[107122]: .465944+0000 mon.c (mon.2) 128 : audit [DBG] from='client.? 172.21.15.64:0/1775509512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:05:22.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:21 smithi064 conmon[107122]: cluster 2023-06-10T21:05:21.623579+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e34: y(active, since 49s), standbys: x 2023-06-10T21:05:22.375 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:22 smithi152 conmon[147575]: 2023-06-10T21:05:22.041+0000 7f8f8e4d7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:05:22.375 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:22 smithi152 conmon[147575]: 2023-06-10T21:05:22.230+0000 7f8f8e4d7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:05:22.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[171866]: 2023-06-10T21:05:22.408+0000 7f6d426ce200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-10T21:05:22.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[171866]: 2023-06-10T21:05:22.566+0000 7f6d426ce200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-10T21:05:22.661 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:22.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-06-10T21:05:22.661 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:22.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2023-06-10T21:05:22.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:22 smithi152 conmon[147575]: 2023-06-10T21:05:22.371+0000 7f8f8e4d7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:05:22.951 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 57718 57702 120.12 75.75 0.0104676 0.00831965 2023-06-10T21:05:23.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:05:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[171866]: 2023-06-10T21:05:22.795+0000 7f6d426ce200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-10T21:05:23.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:22 smithi064 conmon[171866]: 2023-06-10T21:05:22.988+0000 7f6d426ce200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-10T21:05:23.446 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:22 smithi152 conmon[147575]: 2023-06-10T21:05:22.991+0000 7f8f8e4d7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:05:23.817 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:23 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-06-10T21:05:23.817 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:23 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi152:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2023-06-10T21:05:23.817 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:23 smithi064 conmon[171866]: 2023-06-10T21:05:23.809+0000 7f6d426ce200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-10T21:05:23.954 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 58828 58812 118.475 69.375 0.0143451 0.00843477 2023-06-10T21:05:24.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:23 smithi152 conmon[147575]: 2023-06-10T21:05:23.781+0000 7f8f8e4d7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:05:24.195 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:24 smithi152 conmon[147575]: 2023-06-10T21:05:24.016+0000 7f8f8e4d7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:05:24.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 59621 59605 116.32 49.5625 0.00573728 0.00859001 2023-06-10T21:05:25.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:24 smithi064 conmon[171866]: 2023-06-10T21:05:24.777+0000 7f6d426ce200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-10T21:05:25.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:25 smithi064 conmon[171866]: 2023-06-10T21:05:25.006+0000 7f6d426ce200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-10T21:05:25.392 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:24 smithi152 conmon[147575]: 2023-06-10T21:05:24.997+0000 7f8f8e4d7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:25.392 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:25 smithi152 conmon[147575]: 2023-06-10T21:05:25.137+0000 7f8f8e4d7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:05:25.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:25 smithi152 conmon[147575]: 2023-06-10T21:05:25.388+0000 7f8f8e4d7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:25.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 60738 60722 114.911 69.8125 0.0144552 0.00869586 2023-06-10T21:05:26.108 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:25 smithi152 conmon[147575]: 2023-06-10T21:05:25.806+0000 7f8f8e4d7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:05:26.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:26 smithi152 conmon[147575]: 2023-06-10T21:05:26.104+0000 7f8f8e4d7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:05:26.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:26 smithi152 conmon[147575]: 2023-06-10T21:05:26.312+0000 7f8f8e4d7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:05:26.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:26 smithi064 conmon[171866]: 2023-06-10T21:05:26.485+0000 7f6d426ce200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:26.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:26 smithi064 conmon[171866]: 2023-06-10T21:05:26.662+0000 7f6d426ce200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-10T21:05:26.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 61875 61859 113.622 71.0625 0.0101815 0.00879646 2023-06-10T21:05:27.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:26 smithi152 conmon[147575]: 2023-06-10T21:05:26.713+0000 7f8f8e4d7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:05:27.409 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:27 smithi064 conmon[171866]: 2023-06-10T21:05:27.003+0000 7f6d426ce200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-10T21:05:27.912 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:27 smithi064 conmon[171866]: 2023-06-10T21:05:27.548+0000 7f6d426ce200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-10T21:05:27.955 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 63062 63046 112.494 74.1875 0.00627064 0.00888338 2023-06-10T21:05:28.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:27 smithi152 conmon[147575]: 2023-06-10T21:05:27.742+0000 7f8f8e4d7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:05:28.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:27 smithi152 conmon[147575]: 2023-06-10T21:05:27.937+0000 7f8f8e4d7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:05:28.219 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:27 smithi064 conmon[171866]: 2023-06-10T21:05:27.925+0000 7f6d426ce200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-10T21:05:28.220 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:28 smithi064 conmon[171866]: 2023-06-10T21:05:28.211+0000 7f6d426ce200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-10T21:05:28.694 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:28 smithi152 conmon[147575]: 2023-06-10T21:05:28.236+0000 7f8f8e4d7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:05:28.695 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:28 smithi152 conmon[147575]: 2023-06-10T21:05:28.414+0000 7f8f8e4d7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:05:28.957 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 64260 64244 111.443 74.875 0.0139272 0.00896769 2023-06-10T21:05:29.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:28 smithi064 conmon[171866]: 2023-06-10T21:05:28.733+0000 7f6d426ce200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-10T21:05:29.194 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:28 smithi152 conmon[147575]: 2023-06-10T21:05:28.741+0000 7f8f8e4d7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:05:29.960 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 65411 65395 110.37 71.9375 0.0100089 0.00905548 2023-06-10T21:05:30.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:29 smithi064 conmon[171866]: 2023-06-10T21:05:29.987+0000 7f6d426ce200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-10T21:05:30.408 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[171866]: 2023-06-10T21:05:30.233+0000 7f6d426ce200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-10T21:05:30.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: 2023-06-10T21:05:30.000+0000 7f8f8e4d7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:05:30.444 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: [10/Jun/2023:21:05:30] ENGINE Bus STARTING 2023-06-10T21:05:30.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: CherryPy Checker: 2023-06-10T21:05:30.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: The Application mounted at '' has an empty config. 2023-06-10T21:05:30.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: 2023-06-10T21:05:30.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: [10/Jun/2023:21:05:30] ENGINE Serving on http://:::9283 2023-06-10T21:05:30.445 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[147575]: [10/Jun/2023:21:05:30] ENGINE Bus STARTED 2023-06-10T21:05:30.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[171866]: 2023-06-10T21:05:30.619+0000 7f6d426ce200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-10T21:05:30.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[171866]: 2023-06-10T21:05:30.836+0000 7f6d426ce200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-10T21:05:30.962 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 66558 66542 109.347 71.6875 0.0111276 0.00913931 2023-06-10T21:05:30.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: cluster 2023-06-10T21:05:30.015851+0000 2023-06-10T21:05:30.972 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: cluster 2023-06-10T21:05:30.015990+0000 mon.a (mon.0) 12 : cluster [DBG] 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: Standby manager daemon x started 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: audit 2023 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: -06-10T21:05:30.019633+0000 mon.b 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: (mon.1) 117 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: audit 2023 2023-06-10T21:05:30.973 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: -06-10T21:05:30.020292+0000 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: mon.b (mon.1) 118 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: -10T21:05:30.023438+0000 mon.b (mon.1 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: ) 119 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: 05:30.024616+0000 mon.b (mon.1 2023-06-10T21:05:30.974 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:30 smithi152 conmon[97365]: ) 120 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:05:31.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: cluster 2023-06-10T21:05:30.015851+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-06-10T21:05:31.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: cluster 2023- 2023-06-10T21:05:31.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: 06-10T21:05:30.015990+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-06-10T21:05:31.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: audit 2023-06-10T21:05:30.019633+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-10T21:05:31.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: audit 2023-06-10T21:05:30.020292+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-10T21:05:31.280 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: audit 2023-06-10T21:05:30.023438+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-10T21:05:31.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:30 smithi064 conmon[107122]: audit 2023-06-10T21:05:30.024616+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.? 172.21.15.152:0/2583884436' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-10T21:05:31.281 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:31 smithi064 conmon[171866]: 2023-06-10T21:05:31.272+0000 7f6d426ce200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-10T21:05:31.663 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:31 smithi064 conmon[170068]: level=info ts=2023-06-10T21:05:31.443Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2023-06-10T21:05:31.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 67691 67675 108.348 70.8125 0.0254211 0.00922388 2023-06-10T21:05:32.349 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[170068]: level=error ts=2023-06-10T21:05:32.343Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi152:8443/api/prometheus_receiver\": dial tcp 172.21.15.152:8443: connect: connection refused" 2023-06-10T21:05:32.349 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:32.344Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-06-10T21:05:32.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:31 smithi064 conmon[107122]: cluster 2023-06-10T21:05:30.989036 2023-06-10T21:05:32.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:31 smithi064 conmon[107122]: +0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e35: y(active, since 58s), standbys: x 2023-06-10T21:05:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:31 smithi152 conmon[97365]: cluster 2023-06-10T21:05:30.989036+0000 mon.a (mon.0) 13 2023-06-10T21:05:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:31 smithi152 conmon[97365]: : cluster [DBG] mgrmap e35: y(active, since 58s), standbys: x 2023-06-10T21:05:32.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:05:32.963974+0000 min lat: 0.00128113 max lat: 0.342459 avg lat: 0.00930533 2023-06-10T21:05:32.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:05:32.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 68823 68807 107.409 70.75 0.0100228 0.00930533 2023-06-10T21:05:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: debug 2023-06-10T21:05:32.909+0000 7fb762745700 -1 mon.c@2(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-10T21:05:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.902855+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-06-10T21:05:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.903306+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.912263+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.912573+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e36: y(active, starting, since 0.00942144s), standbys: x 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.929552+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.929791+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.929999+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.930147+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:05:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.930534+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.931149+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.931335+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.931494+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.931902+0000 mon.b (mon.1) 129 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.932163+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.932357+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.932529+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:05:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.932684+0000 mon.b (mon.1) 133 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.932843+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.933019+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.933905+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.934170+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: audit 2023-06-10T21:05:32.934756+0000 mon.b (mon.1) 138 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:05:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.957538+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2023-06-10T21:05:33.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:32 smithi064 conmon[171866]: 2023-06-10T21:05:32.888+0000 7f6d426ce200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-10T21:05:33.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: [10/Jun/2023:21:05:33] ENGINE Bus STARTING 2023-06-10T21:05:33.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: debug 2023-06-10T21:05:32.907+0000 7f69a2f47700 -1 mon.b@1(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-10T21:05:33.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: cluster 2023-06-10T21:05:32.902855+0000 mon.a (mon.0) 14 : cluster [INF] 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: Active manager daemon y restarted 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: cluster 2023-06-10T21:05:32.903306+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: cluster 2023- 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 06-10T21 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: :05:32.912263+0000 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: mon.a (mon.0 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: ) 16 : cluster [DBG] 2023-06-10T21:05:33.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: cluster 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 2023-06-10T21:05:32.912573+0000 mon.a ( 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: mon.0) 17 : cluster [DBG] mgrmap e36: y(active, starting, since 0.00942144s), standbys: x 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 10T21:05:32. 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 929552+0000 mon.b (mon.1) 121 2023-06-10T21:05:33.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32.929791+0000 mon.b (mon.1) 122 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: -10T21:05:32.929999+0000 mon.b (mon.1) 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 123 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 05:32.930147+0000 mon.b (mon.1) 124 : 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi064.wuuunt"}]: dispatch 2023-06-10T21:05:33.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: .930534+0000 mon.b (mon.1) 125 : audit 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi152.ciyxdy"}]: dispatch 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32. 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 931149+0000 mon.b (mon.1) 126 : audit [DBG] 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 2023-06-10T21:05:32.931335+0000 mon.b (mon.1 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: ) 127 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32.931494+0000 mon.b ( 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: mon.1) 128 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 21:05:32.931902+0000 mon.b (mon.1) 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 129 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05 2023-06-10T21:05:33.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: :32.932163+0000 mon.b (mon.1) 130 : 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: .932357+0000 mon.b (mon.1) 131 : audit 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 2023-06-10T21:05:32.932529+0000 mon.b (mon.1 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: ) 132 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:05:33.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 05:32.932684+0000 mon.b (mon.1) 133 : 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05: 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 32.932843+0000 mon.b (mon.1) 134 : audit 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: .933019+0000 mon.b (mon.1) 135 : audit 2023-06-10T21:05:33.203 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:32.933905 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: +0000 mon.b (mon.1) 136 : audit [DBG] 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 2023-06-10T21:05:32.934170+0000 mon.b (mon 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: .1) 137 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-10T21:05:33.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:05:33.205 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: T21:05:32.934756+0000 mon.b (mon. 2023-06-10T21:05:33.205 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 1) 138 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-10T21:05:33.205 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: cluster 2023-06-10T 2023-06-10T21:05:33.205 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: 21:05:32.957538+0000 mon.a (mon.0) 18 2023-06-10T21:05:33.205 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:32 smithi152 conmon[97365]: : cluster [INF] Manager daemon y is now available 2023-06-10T21:05:33.463 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.285+0000 7f6ca8d3c700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.285+0000 7f6ca8d3c700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.285+0000 7f6ca8d3c700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.285+0000 7f6ca8d3c700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.285+0000 7f6ca8d3c700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.308+0000 7f6c9fcaa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.308+0000 7f6c9fcaa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.464 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.308+0000 7f6c9fcaa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.308+0000 7f6c9fcaa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.308+0000 7f6c9fcaa700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: CherryPy Checker: 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: The Application mounted at '' has an empty config. 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: [10/Jun/2023:21:05:33] ENGINE Serving on http://:::9283 2023-06-10T21:05:33.465 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: [10/Jun/2023:21:05:33] ENGINE Bus STARTED 2023-06-10T21:05:33.909 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[170068]: level=error ts=2023-06-10T21:05:33.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-06-10T21:05:33.909 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[170068]: level=warn ts=2023-06-10T21:05:33.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi064.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.64:8443: connect: connection refused" 2023-06-10T21:05:33.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 69707 69691 106.137 55.25 0.0127826 0.00941672 2023-06-10T21:05:33.979 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:05:33.979 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (2m) 55s ago 6m 26.8M - 0.23.0 ba2b418f427c 13fd7844ce61 2023-06-10T21:05:33.979 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (84s) 33s ago 6m 48.2M - 8.3.5 dad864ee21e9 95a95736b9e9 2023-06-10T21:05:33.979 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (5m) 55s ago 5m 19.7M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:05:33.979 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (5m) 33s ago 5m 14.7M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443,9283,8765 running (39s) 33s ago 10m 45.4M - 18.0.0-3603-g5937c68a 8fc92d915e8e 0e93fb4f51a4 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (75s) 55s ago 12m 477M - 18.0.0-3603-g5937c68a 8fc92d915e8e 2bda388cec1d 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (12m) 55s ago 12m 128M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 27c52af4dbab 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (10m) 33s ago 10m 95.4M 2048M 16.2.13-257-gd8c5d349 b644dd95899d a46036733d6f 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (10m) 55s ago 10m 98.9M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 22815652c7a9 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (2m) 55s ago 6m 29.9M - 1.3.1 1dbe0e931976 c7179c864e62 2023-06-10T21:05:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (2m) 33s ago 6m 20.2M - 1.3.1 1dbe0e931976 91ff512a05b1 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (10m) 55s ago 10m 2060M 2500M 16.2.13-257-gd8c5d349 b644dd95899d ef1eb4e15c36 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (9m) 55s ago 9m 1228M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 601741de864d 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (9m) 55s ago 9m 2755M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8cca00e2828d 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (9m) 55s ago 9m 861M 2500M 16.2.13-257-gd8c5d349 b644dd95899d c9c0acd987a8 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (8m) 33s ago 8m 2053M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (8m) 33s ago 8m 2199M 2500M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:05:33.981 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (8m) 33s ago 8m 1057M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:05:33.982 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (7m) 33s ago 7m 1609M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:05:33.982 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (44s) 33s ago 7m 53.4M - 2.33.4 514e6a882f6e 2013ef6ec5fb 2023-06-10T21:05:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.912945+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.923285+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: cluster 2023-06-10T21:05:32.926330+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.018788+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.034244+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.055158+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.125764+0000 mon.b (mon.1) 142 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.126360+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.202525+0000 mon.b (mon.1) 143 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.203145+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:05:34.402 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[107122]: cluster 2023-06-10T21:05:33.924561+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e37: y(active, since 1.0214s), standbys: x 2023-06-10T21:05:34.403 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:33 smithi064 conmon[171866]: 2023-06-10T21:05:33.961+0000 7f6cfc057700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-06-10T21:05:34.403 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[170068]: level=info ts=2023-06-10T21:05:34.396Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=2 2023-06-10T21:05:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: cluster 2023-06-10T21:05:32.912945+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: cluster 2023-06-10T21:05:32.923285+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: cluster 2023-06-10T21:05:32.926330+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-10T21:05:34.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.018788+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:34.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.034244+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:34.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.055158+0000 mon.b (mon.1) 141 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:34.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023 2023-06-10T21:05:34.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: -06-10T21:05:33.125764+0000 mon.b (mon.1) 142 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:05:34.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.126360+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-10T21:05:34.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.202525+0000 mon.b (mon.1) 143 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:05:34.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.203145+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-10T21:05:34.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:34 smithi152 conmon[97365]: cluster 2023-06-10T21:05:33.924561+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e37: y(active, since 1.0214s), standbys: x 2023-06-10T21:05:34.658 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:05:34 smithi064 conmon[170068]: level=info ts=2023-06-10T21:05:34.482Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2023-06-10T21:05:34.966 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 70684 70668 105.065 61.0625 0.0140813 0.00951233 2023-06-10T21:05:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:35 smithi064 conmon[107122]: audit 2023-06-10T21:05:33.948573+0000 mgr.y (mgr.34101) 1 : audit [DBG] from='client.34107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:05:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:35 smithi064 conmon[107122]: cluster 2023-06-10T21:05:33.964605+0000 mgr.y (mgr.34101) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:35 smithi152 conmon[97365]: audit 2023-06-10T21:05:33.948573+0000 mgr.y ( 2023-06-10T21:05:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:35 smithi152 conmon[97365]: mgr.34101) 1 : audit [DBG] from='client.34107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:05:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:35 smithi152 conmon[97365]: cluster 2023-06-10T21:05:33.964605+0000 mgr.y (mgr.34101) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:35.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 71612 71596 103.971 58 0.0214813 0.00961294 2023-06-10T21:05:36.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: cluster 2023-06-10T21:05: 2023-06-10T21:05:36.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: 35.030833+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e38: y(active, since 2s), standbys: x 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: cephadm 2023-06- 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: 10T21:05:35.144501+0000 mgr.y (mgr.34101) 4 : cephadm [INF] 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: [10/Jun/2023:21:05:35] ENGINE Bus STARTING 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: cephadm 2023-06-10T21:05: 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: 35.278460+0000 mgr.y (mgr.34101) 5 2023-06-10T21:05:36.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: : cephadm [INF] [10/Jun/2023:21:05:35] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: cephadm 2023- 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: 06-10T21:05:35.383449+0000 mgr.y 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: (mgr.34101) 6 : cephadm [INF] 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: [10/Jun/2023:21:05:35] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:36.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: 2023-06-10T21:05:35.383583+0000 mgr.y (mgr.34101) 7 : cephadm [INF] 2023-06-10T21:05:36.354 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:36 smithi064 conmon[107122]: [10/Jun/2023:21:05:35] ENGINE Bus STARTED 2023-06-10T21:05:36.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade pause' 2023-06-10T21:05:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: cluster 2023-06-10T21:05:35.030833+0000 mon.a (mon.0) 22 : cluster 2023-06-10T21:05:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: [DBG] mgrmap e38: y(active, since 2s), standbys: x 2023-06-10T21:05:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:35.144501+0000 mgr.y (mgr.34101) 4 : cephadm [INF] [10/Jun/2023:21:05:35] ENGINE Bus STARTING 2023-06-10T21:05:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:35.278460+0000 mgr.y (mgr.34101) 5 : cephadm [INF] [10/Jun/2023:21:05:35] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:05:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:35.383449+0000 mgr.y (mgr.34101) 6 : cephadm [INF] [10/Jun/2023:21:05:35] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:05:36.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:36 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:35.383583+0000 mgr.y (mgr.34101) 7 : cephadm [INF] [10/Jun/2023:21:05:35] ENGINE Bus STARTED 2023-06-10T21:05:36.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 72675 72659 103.119 66.4375 0.0261344 0.0096926 2023-06-10T21:05:37.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 73751 73735 102.322 67.25 0.00957144 0.00976814 2023-06-10T21:05:38.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:38 smithi064 conmon[107122]: cluster 2023-06-10T21:05:36.934400+0000 mgr.y (mgr.34101) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:38.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:38 smithi152 conmon[97365]: cluster 2023-06-10T21:05:36.934400+0000 mgr.y (mgr.34101) 8 : 2023-06-10T21:05:38.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:38 smithi152 conmon[97365]: cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:38.967 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 74872 74856 101.621 70.0625 0.0254845 0.00983531 2023-06-10T21:05:39.621 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:05:39.968 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 75849 75833 100.756 61.0625 0.0178877 0.00992032 2023-06-10T21:05:40.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'sleep 60' 2023-06-10T21:05:40.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: cluster 2023-06-10T21:05:38.934840 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 9 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: audit 2023-06-10T21:05:39.555430+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: audit 2023-06-10T21:05:39.569658+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: audit 2023-06-10T21:05:39.594809+0000 mgr.y (mgr.34101) 10 : audit [DBG] from='client.25295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:39.598790+0000 mgr.y (mgr.34101) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:05:40.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:40 smithi064 conmon[107122]: audit 2023-06-10T21:05:39.609864+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: cluster 2023-06-10T21:05:38.934840+0000 mgr.y ( 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: mgr.34101) 9 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: audit 2023-06-10T21:05:39.555430+0000 mon.a (mon.0) 23 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: audit 2023-06-10T21:05:39.569658+0000 mon.a (mon.0) 24 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: audit 2023-06-10T21:05:39.594809+0000 mgr.y (mgr.34101) 10 : audit [DBG] from='client.25295 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:39.598790+0000 mgr.y (mgr.34101) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:05:40.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:40 smithi152 conmon[97365]: audit 2023-06-10T21:05:39.609864+0000 mon.a (mon.0) 25 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:40.968 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 76705 76689 99.7718 53.5 0.00739457 0.0100173 2023-06-10T21:05:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:41 smithi152 conmon[97365]: audit 2023-06-10T21:05:40.554038+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:41 smithi152 conmon[97365]: audit 2023-06-10T21:05:40.575473+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:41 smithi064 conmon[107122]: audit 2023-06-10T21:05:40.554038+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:41.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:41 smithi064 conmon[107122]: audit 2023-06-10T21:05:40.575473+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:41.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 77729 77713 99.0422 64 0.0193155 0.0100904 2023-06-10T21:05:42.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:42 smithi064 conmon[107122]: cluster 2023-06-10T21:05:40.935617+0000 mgr.y (mgr.34101) 12 2023-06-10T21:05:42.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:42 smithi064 conmon[107122]: : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s 2023-06-10T21:05:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:42 smithi152 conmon[97365]: cluster 2023-06-10T21:05:40.935617+0000 mgr.y (mgr.34101) 12 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s 2023-06-10T21:05:42.968 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 78659 78643 98.2243 58.125 0.0155319 0.0101755 2023-06-10T21:05:43.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:05:42] "GET /metrics HTTP/1.1" 200 33497 "" "Prometheus/2.33.4" 2023-06-10T21:05:43.968 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 79670 79654 97.5377 63.1875 0.00572107 0.0102469 2023-06-10T21:05:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:44 smithi152 conmon[97365]: cluster 2023-06-10T21:05:42.936051+0000 mgr.y (mgr.34101) 13 2023-06-10T21:05:44.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:44 smithi152 conmon[97365]: : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s 2023-06-10T21:05:44.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:44 smithi152 conmon[97365]: audit 2023-06-10T21:05:43.437720+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:44.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:44 smithi152 conmon[97365]: audit 2023-06-10T21:05:43.453342+0000 mon.a (mon 2023-06-10T21:05:44.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:44 smithi152 conmon[97365]: .0) 29 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:44.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:44 smithi064 conmon[107122]: cluster 2023-06-10T21:05:42.936051+0000 mgr.y (mgr.34101) 13 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s 2023-06-10T21:05:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:44 smithi064 conmon[107122]: audit 2023-06-10T21:05:43.437720+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:44 smithi064 conmon[107122]: audit 2023-06-10T21:05:43.453342+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:44.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 80405 80389 96.5449 45.9375 0.0109736 0.010353 2023-06-10T21:05:45.585 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:45 smithi152 conmon[97365]: audit 2023-06-10T21:05:44.579105+0000 mon.a (mon.0 2023-06-10T21:05:45.585 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:45 smithi152 conmon[97365]: ) 30 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:45.586 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:45 smithi152 conmon[97365]: audit 2023-06-10T21:05:44.598285+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:45.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:45 smithi064 conmon[107122]: audit 2023-06-10T21:05:44.579105+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:45 smithi064 conmon[107122]: audit 2023-06-10T21:05:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:45 smithi064 conmon[107122]: 2023-06-10T21: 2023-06-10T21:05:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:45 smithi064 conmon[107122]: 05:44.598285+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:45.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 81553 81537 96.0773 71.75 0.0203147 0.0104037 2023-06-10T21:05:46.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:46 smithi064 conmon[107122]: cluster 2023-06-10T21:05:44.936782+0000 mgr.y (mgr.34101) 14 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-06-10T21:05:46.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:46 smithi152 conmon[97365]: cluster 2023-06-10T21:05:44.936782+0000 mgr.y (mgr.34101) 14 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-06-10T21:05:46.969 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 82684 82668 95.6073 70.6875 0.0162182 0.0104542 2023-06-10T21:05:47.972 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 83736 83720 95.0611 65.75 0.0145802 0.0105145 2023-06-10T21:05:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: cluster 2023-06-10T21:05:46.937143+0000 mgr.y (mgr.34101) 15 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-06-10T21:05:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: audit 2023-06-10T21:05:47.211414+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: audit 2023-06-10T21:05:47.220779+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: audit 2023-06-10T21:05:47.229877+0000 mon.b (mon.1) 144 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:48.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: audit 2023-06-10T21:05:47.230997+0000 mon.b (mon.1) 145 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:48.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:47.234676+0000 mgr.y (mgr.34101) 16 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:05:48.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:47.235002+0000 mgr.y (mgr.34101) 17 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:05:48.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:47.667824+0000 mgr.y (mgr.34101) 18 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:05:48.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:48 smithi064 conmon[107122]: audit 2023-06-10T21:05:48.052506+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:48.663 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: cluster 2023-06-10T21:05:46.937143+0000 mgr.y (mgr.34101) 15 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: audit 2023-06-10T21:05:47.211414+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: audit 2023-06-10T21:05:47.220779+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: audit 2023-06-10T21:05:47.229877+0000 mon.b (mon.1) 144 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: audit 2023-06-10T21:05:47.230997+0000 mon.b (mon.1) 145 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:47.234676+0000 mgr.y (mgr.34101) 16 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:47.235002+0000 mgr.y (mgr.34101) 17 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:05:48.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:47.667824+0000 mgr.y (mgr.34101) 18 : cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:05:48.665 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[97365]: audit 2023-06-10T21:05:48.052506+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:05:48.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[146558]: ts=2023-06-10T21:05:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:48.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:48 smithi152 conmon[146558]: ts=2023-06-10T21:05:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:48.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 84748 84732 94.4897 63.25 0.0141597 0.0105785 2023-06-10T21:05:49.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:47.863162 2023-06-10T21:05:49.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 19 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:05:49.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:48.153616 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 20 : cephadm [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: cephadm 2023 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: -06-10T21:05:48.548491+0000 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: mgr.y (mgr.34101) 21 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:49.667 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: 2023-06-10T21:05:48.589275+0000 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: mgr.y (mgr.34101) 22 : cephadm [INF] 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: audit 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: 2023-06-10T21:05:49.033768+0000 mon.a (mon.0) 34 : audit [INF] 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: audit 2023-06-10 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: T21:05:49.057484+0000 mon.a (mon.0) 35 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:49.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:49 smithi064 conmon[107122]: 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:47.863162+0000 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: mgr.y (mgr.34101) 19 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:48.153616+0000 mgr.y (mgr.34101) 20 : cephadm 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: [INF] Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:49.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 2023-06-10T21: 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 05:48.548491+0000 mgr.y (mgr. 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 34101) 21 : cephadm [INF] Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: cephadm 2023- 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 06-10T21:05:48.589275 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: +0000 mgr.y (mgr.34101) 22 : 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: cephadm [INF] Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:05:49.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: audit 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 2023 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: -06-10T21:05:49.033768+0000 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: mon.a (mon.0) 34 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: audit 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: 2023-06-10T21:05:49.057484+0000 2023-06-10T21:05:49.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:49 smithi152 conmon[97365]: mon.a (mon.0) 35 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:49.974 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 85753 85737 93.9343 62.8125 0.0253829 0.0106412 2023-06-10T21:05:50.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: cluster 2023-06-10T21:05:48.937571+0000 mgr.y (mgr.34101) 23 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: cephadm 2023-06-10T21:05:49.220819+0000 mgr.y (mgr.34101) 24 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05:49.807905+0000 mon.a (mon.0) 36 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21: 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: 05:49.822823+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05:49.832869 2023-06-10T21:05:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: +0000 mon.a (mon.0) 38 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: 2023-06-10T21:05:50.047379+0000 mon.b 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: (mon.1) 147 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05: 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: 50.054492+0000 mon.b (mon.1) 148 : audit 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05:50 2023-06-10T21:05:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: .055648+0000 mon.b (mon.1) 149 : audit 2023-06-10T21:05:50.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:50.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: audit 2023-06-10T21:05:50.062237+0000 mon.a ( 2023-06-10T21:05:50.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:50 smithi064 conmon[107122]: mon.0) 39 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: cluster 2023-06-10T21:05:48.937571+0000 mgr.y 2023-06-10T21:05:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: (mgr.34101) 23 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s 2023-06-10T21:05:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: cephadm 2023-06-10T21:05:49.220819+0000 mgr.y (mgr.34101 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: ) 24 : cephadm [INF] Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: -10T21:05:49.807905+0000 mon.a (mon.0) 36 : audit 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: -06-10T21:05:49.822823+0000 2023-06-10T21:05:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: mon.a (mon.0) 37 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: -10T21:05:49.832869+0000 mon.a ( 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: mon.0) 38 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: 06-10T21:05:50.047379+0000 mon.b 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: (mon.1) 147 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: 2023-06-10T21:05:50.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: 2023-06-10T21:05:50.054492+0000 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: mon.b (mon.1) 148 : audit 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023-06-10T21: 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: 05:50.055648+0000 mon.b (mon.1 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: ) 149 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:05:50.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: 10T21:05:50.062237+0000 mon.a (mon.0 2023-06-10T21:05:50.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:50 smithi152 conmon[97365]: ) 39 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:05:50.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 86845 86829 93.4902 68.25 0.010815 0.0106914 2023-06-10T21:05:51.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 87958 87942 93.0848 69.5625 0.00700936 0.0107373 2023-06-10T21:05:52.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:52 smithi064 conmon[107122]: cluster 2023-06-10T21:05:50.938260+0000 mgr.y (mgr.34101) 25 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-06-10T21:05:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:52 smithi152 conmon[97365]: cluster 2023-06-10T21:05:50.938260+0000 2023-06-10T21:05:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:52 smithi152 conmon[97365]: mgr.y (mgr.34101) 25 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-06-10T21:05:52.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:05:52.972870+0000 min lat: 0.00128113 max lat: 0.342459 avg lat: 0.0107855 2023-06-10T21:05:52.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:05:52.976 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 89052 89036 92.6732 68.375 0.00911443 0.0107855 2023-06-10T21:05:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:05:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:05:52] "GET /metrics HTTP/1.1" 200 190046 "" "Prometheus/2.33.4" 2023-06-10T21:05:53.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 90129 90113 92.2576 67.3125 0.00528294 0.0108345 2023-06-10T21:05:54.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:54 smithi064 conmon[107122]: cluster 2023-06-10T21:05:52.938688+0000 mgr.y (mgr.34101) 26 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:05:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:54 smithi152 conmon[97365]: cluster 2023-06-10T21:05:52.938688+0000 mgr.y (mgr.34101) 26 : cluster 2023-06-10T21:05:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:54 smithi152 conmon[97365]: [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:05:54.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 91199 91183 91.8484 66.875 0.00847034 0.0108818 2023-06-10T21:05:55.975 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 92280 92264 91.4631 67.5625 0.0135086 0.0109284 2023-06-10T21:05:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:56 smithi064 conmon[107122]: cluster 2023-06-10T21:05:54.939430 2023-06-10T21:05:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:56 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 27 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:05:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:56 smithi152 conmon[97365]: cluster 2023-06-10T21:05:54.939430+0000 mgr.y (mgr.34101 2023-06-10T21:05:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:56 smithi152 conmon[97365]: ) 27 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:05:56.981 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 93340 93324 91.0661 66.25 0.0166391 0.0109758 2023-06-10T21:05:57.980 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 94410 94394 90.6915 66.875 0.0215138 0.0110213 2023-06-10T21:05:58.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:58 smithi064 conmon[107122]: cluster 2023-06-10T21:05:56.940776+0000 mgr.y (mgr.34101 2023-06-10T21:05:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:05:58 smithi064 conmon[107122]: ) 28 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-06-10T21:05:58.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:58 smithi152 conmon[97365]: cluster 2023-06-10T21:05:56.940776+0000 mgr.y ( 2023-06-10T21:05:58.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:05:58 smithi152 conmon[97365]: mgr.34101) 28 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 871 op/s 2023-06-10T21:05:58.948 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:58 smithi152 conmon[146558]: ts=2023-06-10T21:05:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:58.949 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:05:58 smithi152 conmon[146558]: ts=2023-06-10T21:05:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:05:58.980 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 95451 95435 90.3034 65.0625 0.00491776 0.011069 2023-06-10T21:05:59.980 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 96472 96456 89.9079 63.8125 0.0149942 0.0111176 2023-06-10T21:06:00.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:00 smithi064 conmon[107122]: cluster 2023-06-10T21:05:58.944679+0000 mgr.y (mgr.34101) 29 : cluster 2023-06-10T21:06:00.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:00 smithi064 conmon[107122]: [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-06-10T21:06:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:00 smithi152 conmon[97365]: cluster 2023-06-10T21:05:58.944679+0000 mgr.y (mgr. 2023-06-10T21:06:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:00 smithi152 conmon[97365]: 34101) 29 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-06-10T21:06:00.981 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 97490 97474 89.5209 63.625 0.0130222 0.0111664 2023-06-10T21:06:01.983 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 98503 98487 89.1397 63.3125 0.0170953 0.0112133 2023-06-10T21:06:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:02 smithi064 conmon[107122]: cluster 2023-06-10T21:06:00.945308+0000 mgr.y 2023-06-10T21:06:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:02 smithi064 conmon[107122]: (mgr.34101) 30 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:06:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:02 smithi152 conmon[97365]: cluster 2023-06-10T21:06:00.945308+0000 mgr.y ( 2023-06-10T21:06:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:02 smithi152 conmon[97365]: mgr.34101) 30 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:06:02.982 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 99529 99513 88.7825 64.125 0.0132187 0.0112585 2023-06-10T21:06:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:02] "GET /metrics HTTP/1.1" 200 190046 "" "Prometheus/2.33.4" 2023-06-10T21:06:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:03 smithi064 conmon[107122]: audit 2023-06-10T21:06:03.052827 2023-06-10T21:06:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:03 smithi064 conmon[107122]: +0000 mon.b (mon.1) 150 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:03.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:03 smithi152 conmon[97365]: audit 2023-06-10T21:06:03.052827+0000 mon.b 2023-06-10T21:06:03.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:03 smithi152 conmon[97365]: (mon.1) 150 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:03.982 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 100517 100501 88.402 61.75 0.0198846 0.0113064 2023-06-10T21:06:04.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:04 smithi064 conmon[107122]: cluster 2023-06-10T21:06:02.946778+0000 mgr.y (mgr.34101) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-10T21:06:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:04 smithi152 conmon[97365]: cluster 2023-06-10T21:06:02.946778+0000 2023-06-10T21:06:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:04 smithi152 conmon[97365]: mgr.y (mgr.34101) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-10T21:06:04.982 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 101506 101490 88.0327 61.8125 0.0195267 0.0113537 2023-06-10T21:06:05.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 102487 102471 87.665 61.3125 0.02497 0.0114021 2023-06-10T21:06:06.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:06 smithi064 conmon[107122]: cluster 2023-06-10T21:06:04. 2023-06-10T21:06:06.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:06 smithi064 conmon[107122]: 947474+0000 mgr.y (mgr.34101) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:06:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:06 smithi152 conmon[97365]: cluster 2023-06-10T21:06:04.947474+0000 2023-06-10T21:06:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:06 smithi152 conmon[97365]: mgr.y (mgr.34101) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:06:06.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 103455 103439 87.298 60.5 0.015907 0.0114499 2023-06-10T21:06:07.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 104423 104407 86.9409 60.5 0.0240636 0.0114971 2023-06-10T21:06:08.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:08 smithi064 conmon[107122]: cluster 2023-06-10T21:06:06.947888+0000 mgr.y (mgr.34101) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-06-10T21:06:08.665 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:08 smithi152 conmon[146558]: ts=2023-06-10T21:06:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:08.665 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:08 smithi152 conmon[146558]: ts=2023-06-10T21:06:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:08.665 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:08 smithi152 conmon[97365]: cluster 2023-06-10T21:06:06.947888+0000 mgr.y (mgr.34101) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-06-10T21:06:08.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 105386 105370 86.589 60.1875 0.0185889 0.0115444 2023-06-10T21:06:09.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 106349 106333 86.2463 60.1875 0.00745882 0.0115896 2023-06-10T21:06:10.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:10 smithi064 conmon[107122]: cluster 2023-06-10T21:06:08.948324+0000 mgr.y (mgr.34101) 34 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-06-10T21:06:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:10 smithi152 conmon[97365]: cluster 2023-06-10T21:06:08.948324+0000 mgr.y (mgr.34101) 34 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2023-06-10T21:06:10.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 107295 107279 85.8988 59.125 0.0207996 0.0116365 2023-06-10T21:06:11.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 108251 108235 85.5679 59.75 0.0106908 0.011682 2023-06-10T21:06:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:12 smithi064 conmon[107122]: cluster 2023-06-10T21:06:10.952184 2023-06-10T21:06:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:12 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 35 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:06:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:12 smithi152 conmon[97365]: cluster 2023-06-10T21:06:10.952184+0000 2023-06-10T21:06:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:12 smithi152 conmon[97365]: mgr.y (mgr.34101) 35 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:06:12.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:06:12.982300+0000 min lat: 0.00128113 max lat: 0.342459 avg lat: 0.0117265 2023-06-10T21:06:12.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:06:12.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 109199 109183 85.2391 59.25 0.0220172 0.0117265 2023-06-10T21:06:13.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:12] "GET /metrics HTTP/1.1" 200 190047 "" "Prometheus/2.33.4" 2023-06-10T21:06:13.984 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 110103 110087 84.8844 56.5 0.0168329 0.0117757 2023-06-10T21:06:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:14 smithi064 conmon[107122]: cluster 2023-06-10T21:06:12.952649+0000 2023-06-10T21:06:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:14 smithi064 conmon[107122]: mgr.y (mgr.34101) 36 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-06-10T21:06:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:14 smithi152 conmon[97365]: cluster 2023-06-10T21:06:12.952649+0000 2023-06-10T21:06:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:14 smithi152 conmon[97365]: mgr.y (mgr.34101) 36 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 827 op/s 2023-06-10T21:06:14.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 111017 111001 84.546 57.125 0.0245102 0.0118225 2023-06-10T21:06:15.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 111937 111921 84.2203 57.5 0.0192866 0.0118687 2023-06-10T21:06:16.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:16 smithi152 conmon[97365]: cluster 2023-06-10T21:06:14.953345+0000 mgr.y 2023-06-10T21:06:16.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:16 smithi152 conmon[97365]: (mgr.34101) 37 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:06:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:16 smithi064 conmon[107122]: cluster 2023-06-10T21:06:14.953345+0000 mgr.y ( 2023-06-10T21:06:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:16 smithi064 conmon[107122]: mgr.34101) 37 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:06:16.985 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 112817 112801 83.8726 55 0.0124704 0.0119179 2023-06-10T21:06:17.986 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 113716 113700 83.5464 56.1875 0.0171216 0.0119641 2023-06-10T21:06:18.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:18 smithi152 conmon[146558]: ts=2023-06-10T21:06:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:18.695 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:18 smithi152 conmon[146558]: ts=2023-06-10T21:06:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:18 smithi152 conmon[97365]: cluster 2023-06-10T21:06:16.953778+0000 mgr.y 2023-06-10T21:06:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:18 smithi152 conmon[97365]: (mgr.34101) 38 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-06-10T21:06:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:18 smithi152 conmon[97365]: audit 2023-06-10T21:06:18.055111+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:18 smithi064 conmon[107122]: cluster 2023-06-10T21:06:16.953778+0000 mgr.y (mgr.34101) 38 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-06-10T21:06:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:18 smithi064 conmon[107122]: audit 2023-06-10T21:06:18.055111+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:18.991 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 114616 114600 83.2244 56.25 0.032873 0.0120106 2023-06-10T21:06:19.991 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 115524 115508 82.9193 56.75 0.0182161 0.0120553 2023-06-10T21:06:20.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:20 smithi152 conmon[97365]: cluster 2023-06-10T21:06:18.954181+0000 mgr.y (mgr 2023-06-10T21:06:20.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:20 smithi152 conmon[97365]: .34101) 39 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-06-10T21:06:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:20 smithi064 conmon[107122]: cluster 2023-06-10T21 2023-06-10T21:06:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:20 smithi064 conmon[107122]: :06:18.954181+0000 mgr.y (mgr.34101) 39 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s 2023-06-10T21:06:20.992 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 116410 116394 82.6065 55.375 0.0173868 0.0121005 2023-06-10T21:06:21.992 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 117303 117287 82.3055 55.8125 0.0222178 0.0121447 2023-06-10T21:06:22.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:22 smithi064 conmon[107122]: cluster 2023-06-10T21:06:20.957876 2023-06-10T21:06:22.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:22 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 40 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:06:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:22 smithi152 conmon[97365]: cluster 2023-06-10T21:06:20.957876+0000 mgr.y (mgr.34101) 40 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:06:22.992 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0102 2023-06-10T21:06:22.992 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 118118 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 82.0171 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 35.562 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 212.312 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 45.9375 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1312 2023-06-10T21:06:22.993 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 568.991 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3397 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 735 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0121886 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.00998335 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.342459 2023-06-10T21:06:22.994 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00128113 2023-06-10T21:06:23.000 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:06:23.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:22] "GET /metrics HTTP/1.1" 200 190046 "" "Prometheus/2.33.4" 2023-06-10T21:06:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:24 smithi064 conmon[107122]: cluster 2023-06-10T21:06:22.958267+0000 mgr.y (mgr.34101) 41 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-06-10T21:06:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:24 smithi152 conmon[97365]: cluster 2023-06-10T21:06:22.958267 2023-06-10T21:06:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:24 smithi152 conmon[97365]: +0000 mgr.y (mgr.34101) 41 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2023-06-10T21:06:26.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:26 smithi064 conmon[107122]: cluster 2023-06-10T21:06:24.958925 2023-06-10T21:06:26.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:26 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 42 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.14k op/s 2023-06-10T21:06:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:26 smithi152 conmon[97365]: cluster 2023-06-10T21:06:24.958925+0000 mgr.y (mgr.34101 2023-06-10T21:06:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:26 smithi152 conmon[97365]: ) 42 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 66 MiB/s wr, 1.14k op/s 2023-06-10T21:06:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:28 smithi064 conmon[107122]: cluster 2023-06-10T21:06:26.959338+0000 mgr.y (mgr.34101) 43 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 743 op/s 2023-06-10T21:06:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:28 smithi152 conmon[97365]: cluster 2023-06-10T21:06:26.959338+0000 mgr.y ( 2023-06-10T21:06:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:28 smithi152 conmon[97365]: mgr.34101) 43 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 743 op/s 2023-06-10T21:06:28.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:28 smithi152 conmon[146558]: ts=2023-06-10T21:06:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:28.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:28 smithi152 conmon[146558]: ts=2023-06-10T21:06:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:30.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:30 smithi064 conmon[107122]: cluster 2023-06-10T21:06:28.959768+0000 mgr.y (mgr.34101) 44 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 743 op/s 2023-06-10T21:06:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:30 smithi152 conmon[97365]: cluster 2023-06-10T21:06:28.959768+0000 mgr.y ( 2023-06-10T21:06:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:30 smithi152 conmon[97365]: mgr.34101) 44 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 743 op/s 2023-06-10T21:06:32.883 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:32] "GET /metrics HTTP/1.1" 200 190046 "" "Prometheus/2.33.4" 2023-06-10T21:06:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:32 smithi064 conmon[107122]: cluster 2023-06-10T21:06:30.960478+0000 mgr.y (mgr.34101) 45 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.12k op/s 2023-06-10T21:06:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:32 smithi152 conmon[97365]: cluster 2023-06-10T21:06:30.960478+0000 mgr.y (mgr.34101) 45 2023-06-10T21:06:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:32 smithi152 conmon[97365]: : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.12k op/s 2023-06-10T21:06:33.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:33 smithi064 conmon[107122]: audit 2023-06-10T21:06:33.053442+0000 mon.b (mon 2023-06-10T21:06:33.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:33 smithi064 conmon[107122]: .1) 152 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:33 smithi152 conmon[97365]: audit 2023-06-10T21:06:33.053442+0000 mon.b (mon 2023-06-10T21:06:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:33 smithi152 conmon[97365]: .1) 152 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:34 smithi064 conmon[107122]: cluster 2023-06-10T21:06:32.960845+0000 2023-06-10T21:06:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:34 smithi064 conmon[107122]: mgr.y (mgr.34101) 46 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 743 op/s 2023-06-10T21:06:34.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:34 smithi152 conmon[97365]: cluster 2023-06-10T21: 2023-06-10T21:06:34.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:34 smithi152 conmon[97365]: 06:32.960845+0000 mgr.y (mgr.34101) 46 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 743 op/s 2023-06-10T21:06:36.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:36 smithi064 conmon[107122]: cluster 2023-06-10T21:06:34.961499 2023-06-10T21:06:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:36 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 47 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 1.11k op/s 2023-06-10T21:06:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:36 smithi152 conmon[97365]: cluster 2023-06-10T21:06:34.961499+0000 mgr.y (mgr 2023-06-10T21:06:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:36 smithi152 conmon[97365]: .34101) 47 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 1.11k op/s 2023-06-10T21:06:38.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:38 smithi064 conmon[107122]: cluster 2023-06-10T21:06:36.961901+0000 mgr.y (mgr.34101) 48 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-10T21:06:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:38 smithi152 conmon[97365]: cluster 2023-06-10T21:06:36.961901+0000 mgr.y (mgr.34101 2023-06-10T21:06:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:38 smithi152 conmon[97365]: ) 48 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-10T21:06:38.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:38 smithi152 conmon[146558]: ts=2023-06-10T21:06:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:38.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:38 smithi152 conmon[146558]: ts=2023-06-10T21:06:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:40.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:40 smithi064 conmon[107122]: cluster 2023-06-10T21:06:38.964438+0000 2023-06-10T21:06:40.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:40 smithi064 conmon[107122]: mgr.y (mgr.34101) 49 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-10T21:06:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:40 smithi152 conmon[97365]: cluster 2023-06-10T21:06:38.964438+0000 2023-06-10T21:06:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:40 smithi152 conmon[97365]: mgr.y (mgr.34101) 49 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-10T21:06:42.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:42 smithi064 conmon[107122]: cluster 2023-06-10T21:06:40.966615+0000 mgr.y (mgr.34101) 50 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2023-06-10T21:06:42.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:42] "GET /metrics HTTP/1.1" 200 190095 "" "Prometheus/2.33.4" 2023-06-10T21:06:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:42 smithi152 conmon[97365]: cluster 2023-06-10T21:06:40.966615+0000 mgr.y ( 2023-06-10T21:06:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:42 smithi152 conmon[97365]: mgr.34101) 50 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2023-06-10T21:06:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:44 smithi064 conmon[107122]: cluster 2023-06-10T21:06:42.967025+0000 2023-06-10T21:06:44.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:44 smithi064 conmon[107122]: mgr.y (mgr.34101) 51 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-06-10T21:06:44.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:44 smithi152 conmon[97365]: cluster 2023-06-10T21:06:42.967025+0000 mgr.y (mgr.34101) 51 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-06-10T21:06:45.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade resume' 2023-06-10T21:06:46.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:46 smithi064 conmon[107122]: cluster 2023-06-10T21:06:44.967696+0000 mgr.y (mgr.34101) 52 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-06-10T21:06:46.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:46 smithi152 conmon[97365]: cluster 2023-06-10T21:06:44.967696+0000 mgr.y 2023-06-10T21:06:46.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:46 smithi152 conmon[97365]: (mgr.34101) 52 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2023-06-10T21:06:48.390 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:06:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:48 smithi064 conmon[107122]: cluster 2023-06-10T21:06:46 2023-06-10T21:06:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:48 smithi064 conmon[107122]: .971527+0000 mgr.y (mgr.34101) 53 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:06:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:48 smithi064 conmon[107122]: audit 2023-06-10T21:06:48.053962+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:48 smithi064 conmon[107122]: audit 2023-06-10T21:06:48.372258+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:48 smithi064 conmon[107122]: audit 2023-06-10T21:06:48.385209+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:06:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[97365]: cluster 2023-06-10T21:06:46.971527+0000 mgr.y (mgr.34101) 53 2023-06-10T21:06:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[97365]: : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:06:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[97365]: audit 2023-06-10T21:06:48.053962+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:06:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[97365]: audit 2023-06-10T21:06:48.372258+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[97365]: audit 2023-06-10T21:06:48.385209+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:06:48.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[146558]: ts=2023-06-10T21:06:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:48.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:48 smithi152 conmon[146558]: ts=2023-06-10T21:06:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:49.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'echo wait for majority of mons to upgrade' 2023-06-10T21:06:49.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:49 smithi064 conmon[107122]: audit 2023-06-10T21:06:48.362280+0000 mgr.y (mgr.34101) 54 : audit [DBG] from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:06:49.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:49 smithi064 conmon[107122]: cephadm 2023-06-10T21:06:48.362783+0000 mgr.y (mgr.34101) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:06:49.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:49 smithi152 conmon[97365]: audit 2023-06-10T21:06:48.362280+0000 mgr.y (mgr.34101) 54 : audit [DBG] from='client.25301 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:06:49.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:49 smithi152 conmon[97365]: cephadm 2023-06-10T21:06:48.362783+0000 mgr.y (mgr.34101) 55 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:06:50.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:50 smithi064 conmon[107122]: cluster 2023-06-10T21:06:48.971880 2023-06-10T21:06:50.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:50 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 56 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:06:50.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:50 smithi152 conmon[97365]: cluster 2023-06-10T21:06:48.971880+0000 mgr.y ( 2023-06-10T21:06:50.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:50 smithi152 conmon[97365]: mgr.34101) 56 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:06:52.260 INFO:teuthology.orchestra.run.smithi064.stdout:wait for majority of mons to upgrade 2023-06-10T21:06:52.888 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:06:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:06:52] "GET /metrics HTTP/1.1" 200 190096 "" "Prometheus/2.33.4" 2023-06-10T21:06:52.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:52 smithi064 conmon[107122]: cluster 2023-06-10T21:06:50.972528+0000 mgr.y (mgr.34101) 57 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-06-10T21:06:52.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:52 smithi064 conmon[107122]: audit 2023-06-10T21:06:52.405197+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:52.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:52 smithi064 conmon[107122]: audit 2023-06-10T21:06:52.428405+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: cluster 2023-06-10T21:06:50.972528+0000 mgr.y (mgr. 2023-06-10T21:06:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: 34101) 57 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-06-10T21:06:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: audit 2023-06-10T21:06:52.405197+0000 mon.a (mon.0) 41 : audit [INF] 2023-06-10T21:06:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:06:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: T21:06:52.428405+0000 mon.a (mon.0) 42 : audit [INF] 2023-06-10T21:06:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:52 smithi152 conmon[97365]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:53.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-06-10T21:06:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:54 smithi152 conmon[97365]: cluster 2023-06-10T21:06:52.972911+0000 mgr.y 2023-06-10T21:06:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:54 smithi152 conmon[97365]: (mgr.34101) 58 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:06:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:54 smithi064 conmon[107122]: cluster 2023-06-10T21:06:52.972911+0000 mgr.y (mgr.34101) 58 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:06:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:56 smithi152 conmon[97365]: cluster 2023-06-10T21:06:54.973595+0000 mgr.y (mgr.34101) 59 2023-06-10T21:06:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:56 smithi152 conmon[97365]: : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-06-10T21:06:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:56 smithi152 conmon[97365]: audit 2023-06-10T21:06:56.408375+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:56 smithi152 conmon[97365]: audit 2023-06-10T21:06:56.417865+0000 2023-06-10T21:06:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:56 smithi152 conmon[97365]: mon.a (mon.0) 44 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:57.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:56 smithi064 conmon[107122]: cluster 2023-06-10T21:06:54.973595+0000 mgr.y (mgr.34101) 59 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2023-06-10T21:06:57.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:56 smithi064 conmon[107122]: audit 2023-06-10T21:06:56.408375+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:57.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:56 smithi064 conmon[107122]: audit 2023-06-10T21:06:56.417865+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:58.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:56.843089+0000 mon.b (mon.1) 155 : audit [DBG] from='client.? 172.21.15.64:0/3722376794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:06:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.268861+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:06:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.274599+0000 mon.b (mon.1) 157 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:06:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.281547+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.547160+0000 mon.b (mon.1) 158 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:06:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.560875+0000 mon.b (mon.1) 159 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:06:58.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.565783+0000 mon.b (mon.1) 160 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:06:58.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:57 smithi064 conmon[107122]: audit 2023-06-10T21:06:57.568153+0000 mon.b (mon.1) 161 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-06-10T21:06:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:56.843089+0000 mon.b (mon.1) 155 : audit 2023-06-10T21:06:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: [DBG] from='client.? 172.21.15.64:0/3722376794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.268861+0000 mon.b (mon.1) 156 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.274599+0000 mon.b (mon.1) 157 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.281547+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.547160+0000 mon.b (mon. 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: 1) 158 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:06:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.560875+0000 mon.b ( 2023-06-10T21:06:58.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: mon.1) 159 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:06:58.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06:57.565783 2023-06-10T21:06:58.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: +0000 mon.b (mon.1) 160 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:06:58.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: audit 2023-06-10T21:06: 2023-06-10T21:06:58.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:57 smithi152 conmon[97365]: 57.568153+0000 mon.b (mon.1) 161 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-06-10T21:06:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:58 smithi064 conmon[107122]: cluster 2023-06-10T21:06:56.974001+0000 mgr.y (mgr.34101) 60 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-06-10T21:06:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:58 smithi064 conmon[107122]: cephadm 2023-06-10T21:06:57.569146+0000 mgr.y (mgr.34101) 61 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-06-10T21:06:58.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[146558]: ts=2023-06-10T21:06:58.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:58.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[146558]: ts=2023-06-10T21:06:58.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:06:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[97365]: cluster 2023-06-10T21:06:56. 2023-06-10T21:06:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[97365]: 974001+0000 mgr.y (mgr.34101) 60 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-06-10T21:06:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[97365]: cephadm 2023-06-10T21:06:57.569146+0000 mgr.y (mgr.34101) 61 : cephadm 2023-06-10T21:06:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:58 smithi152 conmon[97365]: [INF] Upgrade: It appears safe to stop mon.c 2023-06-10T21:07:00.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:59 smithi064 conmon[107122]: audit 2023-06-10T21:06:59.545438+0000 mon.c ( 2023-06-10T21:07:00.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:06:59 smithi064 conmon[107122]: mon.2) 129 : audit [DBG] from='client.? 172.21.15.64:0/1469499924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:07:00.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:06:59 smithi152 conmon[97365]: audit 2023-06-10T21:06:59.545438+0000 mon.c (mon.2) 129 : audit [DBG] from='client.? 172.21.15.64:0/1469499924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:07:01.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:00 smithi064 conmon[107122]: cluster 2023-06-10T21:06:58.974434+0000 mgr.y (mgr.34101) 62 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-06-10T21:07:01.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:00 smithi064 conmon[107122]: audit 2023-06-10T21:07:00.257534+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:01.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:00 smithi064 conmon[107122]: audit 2023-06-10T21:07:00.263418+0000 mon.b (mon.1) 162 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:07:01.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:00 smithi064 conmon[107122]: audit 2023-06-10T21:07:00.277385+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:07:01.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:00 smithi064 conmon[107122]: audit 2023-06-10T21:07:00.278360+0000 mon.b (mon.1) 163 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:07:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: cluster 2023-06-10T21:06:58.974434+0000 mgr.y (mgr.34101) 62 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 603 op/s 2023-06-10T21:07:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: audit 2023-06-10T21:07:00.257534+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: audit 2023-06-10T21:07:00.263418+0000 mon.b (mon.1) 162 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:07:01.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:07:01.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: 06-10T21:07:00.277385+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:07:01.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:00 smithi152 conmon[97365]: audit 2023-06-10T21:07:00.278360+0000 mon.b (mon.1) 163 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:07:01.762 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:01 smithi064 conmon[107122]: cephadm 2023-06-10T21:07:00.234910 2023-06-10T21:07:01.762 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:01 smithi064 conmon[107122]: +0000 mgr.y (mgr.34101) 63 : cephadm [INF] Upgrade: Updating mon.c 2023-06-10T21:07:01.762 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:01 smithi064 conmon[107122]: cephadm 2023-06-10T21:07:00.279584+0000 mgr.y (mgr.34101) 64 : cephadm [INF] Deploying daemon mon.c on smithi064 2023-06-10T21:07:02.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:01 smithi152 conmon[97365]: cephadm 2023-06-10T21:07:00.234910+0000 mgr.y (mgr.34101) 63 : cephadm 2023-06-10T21:07:02.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:01 smithi152 conmon[97365]: [INF] Upgrade: Updating mon.c 2023-06-10T21:07:02.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:01 smithi152 conmon[97365]: cephadm 2023-06-10T21:07:00.279584+0000 mgr.y (mgr.34101) 64 : cephadm [INF] Deploying daemon mon.c on smithi064 2023-06-10T21:07:02.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:02 smithi064 systemd[1]: Stopping Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:07:03.093 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:02] "GET /metrics HTTP/1.1" 200 190096 "" "Prometheus/2.33.4" 2023-06-10T21:07:03.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:02 smithi064 conmon[107122]: cluster 2023-06-10T21: 2023-06-10T21:07:03.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:02 smithi064 conmon[107122]: 07:00.975102+0000 mgr.y (mgr.34101) 65 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-06-10T21:07:03.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:02 smithi064 conmon[107122]: audit 2023-06-10T21:07:02.149648+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.64:0/1880016430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:07:03.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:03 smithi064 conmon[107122]: debug 2023-06-10T21:07:03.089+0000 7fb768f52700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-10T21:07:03.094 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:03 smithi064 conmon[107122]: debug 2023-06-10T21:07:03.089+0000 7fb768f52700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-06-10T21:07:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:02 smithi152 conmon[97365]: cluster 2023-06-10T21:07:00.975102+0000 mgr.y (mgr.34101 2023-06-10T21:07:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:02 smithi152 conmon[97365]: ) 65 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-06-10T21:07:03.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:02 smithi152 conmon[97365]: audit 2023-06-10T21:07:02.149648+0000 mon.c (mon.2) 130 : audit [DBG] from='client.? 172.21.15.64:0/1880016430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:07:03.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:03 smithi064 bash[186278]: 22815652c7a963e7e13d14de7e03464901864a8e394ac5692be60df86500820e 2023-06-10T21:07:04.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:04 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.c.service: Succeeded. 2023-06-10T21:07:04.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:04 smithi064 systemd[1]: Stopped Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:07:04.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:04 smithi064 systemd[1]: Starting Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:07:05.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 bash[186643]: 925081a4753eab527f5566e5d35df0ea73454b8eefac3d3468919ea54a89f994 2023-06-10T21:07:05.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: set uid:gid to 167:167 (ceph:ceph) 2023-06-10T21:07:05.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc), process ceph-mon, pid 7 2023-06-10T21:07:05.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: pidfile_write: ignore empty --pid-file 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: load: jerasure load: lrc 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: RocksDB version: 7.9.2 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Git sha 0 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Compile date 2023-06-09 19:19:26 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: DB SUMMARY 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: DB Session ID: IV5GF3L4RQIP5SSBPO86 2023-06-10T21:07:05.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: CURRENT file: CURRENT 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: IDENTITY file: IDENTITY 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: MANIFEST file: MANIFEST-000005 size: 905 Bytes 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 13760639 ; 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.error_if_exists: 0 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.create_if_missing: 0 2023-06-10T21:07:05.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.paranoid_checks: 1 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.flush_verify_memtable_count: 1 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.env: 0x56498a898720 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.fs: PosixFileSystem 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.info_log: 0x56498c5853e0 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_file_opening_threads: 16 2023-06-10T21:07:05.918 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.statistics: (nil) 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.use_fsync: 0 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_log_file_size: 0 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.keep_log_file_num: 1000 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.recycle_log_file_num: 0 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_fallocate: 1 2023-06-10T21:07:05.919 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_mmap_reads: 0 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_mmap_writes: 0 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.use_direct_reads: 0 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.create_missing_column_families: 0 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.db_log_dir: 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.wal_dir: 2023-06-10T21:07:05.920 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.advise_random_on_open: 1 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.db_write_buffer_size: 0 2023-06-10T21:07:05.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.write_buffer_manager: 0x56498b814c80 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.rate_limiter: (nil) 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.wal_recovery_mode: 2 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enable_thread_tracking: 0 2023-06-10T21:07:05.922 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enable_pipelined_write: 0 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.unordered_write: 0 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.row_cache: None 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.wal_filter: None 2023-06-10T21:07:05.923 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T21:07:05.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_ingest_behind: 0 2023-06-10T21:07:05.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.two_write_queues: 0 2023-06-10T21:07:05.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.manual_wal_flush: 0 2023-06-10T21:07:05.924 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.wal_compression: 0 2023-06-10T21:07:05.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.atomic_flush: 0 2023-06-10T21:07:05.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T21:07:05.926 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T21:07:05.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T21:07:05.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.log_readahead_size: 0 2023-06-10T21:07:05.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-10T21:07:05.927 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.best_efforts_recovery: 0 2023-06-10T21:07:05.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-10T21:07:05.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-10T21:07:05.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.allow_data_in_errors: 0 2023-06-10T21:07:05.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.db_host_id: __hostname__ 2023-06-10T21:07:05.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enforce_single_del_contracts: true 2023-06-10T21:07:05.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_background_jobs: 2 2023-06-10T21:07:05.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_background_compactions: -1 2023-06-10T21:07:05.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_subcompactions: 1 2023-06-10T21:07:05.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T21:07:05.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T21:07:05.930 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T21:07:05.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_total_wal_size: 0 2023-06-10T21:07:05.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T21:07:05.931 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T21:07:05.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T21:07:05.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T21:07:05.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_open_files: -1 2023-06-10T21:07:05.932 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bytes_per_sync: 0 2023-06-10T21:07:05.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T21:07:05.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T21:07:05.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_readahead_size: 0 2023-06-10T21:07:05.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_background_flushes: -1 2023-06-10T21:07:05.933 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Compression algorithms supported: 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kZSTD supported: 0 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kXpressCompression supported: 0 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kLZ4HCCompression supported: 1 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kZlibCompression supported: 1 2023-06-10T21:07:05.934 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kSnappyCompression supported: 1 2023-06-10T21:07:05.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kLZ4Compression supported: 1 2023-06-10T21:07:05.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: kBZip2Compression supported: 0 2023-06-10T21:07:05.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T21:07:05.935 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: DMutex implementation: pthread_mutex_t 2023-06-10T21:07:05.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-06-10T21:07:05.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-10T21:07:05.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T21:07:05.936 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.merge_operator: 2023-06-10T21:07:05.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_filter: None 2023-06-10T21:07:05.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_filter_factory: None 2023-06-10T21:07:05.937 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.sst_partitioner_factory: None 2023-06-10T21:07:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T21:07:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T21:07:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56498c585500) 2023-06-10T21:07:05.938 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks: 1 2023-06-10T21:07:05.939 INFO:journalctl@ceph.mon.c.smithi064.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T21:07:05.939 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T21:07:05.939 INFO:journalctl@ceph.mon.c.smithi064.stdout: pin_top_level_index_and_filter: 1 2023-06-10T21:07:05.939 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_type: 0 2023-06-10T21:07:05.939 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_index_type: 0 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_shortening: 1 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_block_hash_table_util_ratio: 0.750000 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: checksum: 4 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: no_block_cache: 0 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache: 0x56498b897350 2023-06-10T21:07:05.940 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_name: BinnedLRUCache 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_options: 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: capacity : 536870912 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_shard_bits : 4 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: strict_capacity_limit : 0 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: high_pri_pool_ratio: 0.000 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_cache_compressed: (nil) 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: persistent_cache: (nil) 2023-06-10T21:07:05.941 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size: 4096 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_size_deviation: 10 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_restart_interval: 16 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: index_block_restart_interval: 1 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_block_size: 4096 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: partition_filters: 0 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: use_delta_encoding: 1 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: filter_policy: bloomfilter 2023-06-10T21:07:05.942 INFO:journalctl@ceph.mon.c.smithi064.stdout: whole_key_filtering: 1 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: verify_compression: 0 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: read_amp_bytes_per_bit: 0 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: format_version: 5 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_index_compression: 1 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: block_align: 0 2023-06-10T21:07:05.943 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_auto_readahead_size: 262144 2023-06-10T21:07:05.944 INFO:journalctl@ceph.mon.c.smithi064.stdout: prepopulate_block_cache: 0 2023-06-10T21:07:05.944 INFO:journalctl@ceph.mon.c.smithi064.stdout: initial_auto_readahead_size: 8192 2023-06-10T21:07:05.944 INFO:journalctl@ceph.mon.c.smithi064.stdout: num_file_reads_for_auto_readahead: 2 2023-06-10T21:07:05.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.write_buffer_size: 33554432 2023-06-10T21:07:05.944 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_write_buffer_number: 2 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression: NoCompression 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression: Disabled 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.prefix_extractor: nullptr 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.num_levels: 7 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T21:07:05.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-10T21:07:05.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.level: 32767 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.strategy: 0 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:07:05.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.enabled: false 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T21:07:05.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.target_file_size_base: 67108864 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T21:07:05.949 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.arena_block_size: 1048576 2023-06-10T21:07:05.950 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.disable_auto_compactions: 0 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T21:07:05.951 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T21:07:05.952 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.table_properties_collectors: 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.inplace_update_support: 0 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.bloom_locality: 0 2023-06-10T21:07:05.953 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.max_successive_merges: 0 2023-06-10T21:07:05.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T21:07:05.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.paranoid_file_checks: 0 2023-06-10T21:07:05.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.force_consistency_checks: 1 2023-06-10T21:07:05.954 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.report_bg_io_stats: 0 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.ttl: 2592000 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enable_blob_files: false 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.min_blob_size: 0 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_file_size: 268435456 2023-06-10T21:07:05.955 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_compression_type: NoCompression 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.enable_blob_garbage_collection: false 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.blob_file_starting_level: 0 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-10T21:07:05.956 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 20.sst 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 12000, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 18 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c03dcc7c-1897-4f8a-850b-87680f02ed11 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431225488295, "job": 1, "event": "recovery_started", "wal_files": [18]} 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #18 mode 2 2023-06-10T21:07:05.957 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 systemd[1]: Started Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431225532829, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 23, "file_size": 8925013, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12006, "largest_seqno": 14234, "table_properties": {"data_size": 8912916, "index_size": 9478, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 12357, "raw_average_key_size": 19, "raw_value_size": 8894382, "raw_average_value_size": 14208, "num_data_blocks": 415, "num_entries": 626, "num_filter_entries": 626, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1686431225, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c03dcc7c-1897-4f8a-850b-87680f02ed11", "db_session_id": "IV5GF3L4RQIP5SSBPO86", "orig_file_number": 23, "seqno_to_time_mapping": "N/A"}} 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431225532971, "job": 1, "event": "recovery_finished"} 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:5047] Creating manifest 25 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000018.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56498b930000 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: DB pointer 0x56498b91e000 2023-06-10T21:07:05.958 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: starting mon.c rank 2 at public addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] at bind addrs [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???) e3 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).mds e7 new map 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).mds e7 print_map 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout: e7 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:07:05.959 INFO:journalctl@ceph.mon.c.smithi064.stdout: legacy client fscid: 1 2023-06-10T21:07:05.960 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.960 INFO:journalctl@ceph.mon.c.smithi064.stdout: Filesystem 'foo' (1) 2023-06-10T21:07:05.960 INFO:journalctl@ceph.mon.c.smithi064.stdout: fs_name foo 2023-06-10T21:07:05.960 INFO:journalctl@ceph.mon.c.smithi064.stdout: epoch 5 2023-06-10T21:07:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-10T21:07:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout: created 2023-06-10T20:59:25.413290+0000 2023-06-10T21:07:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout: modified 2023-06-10T20:59:39.018592+0000 2023-06-10T21:07:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout: tableserver 0 2023-06-10T21:07:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout: root 0 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_timeout 60 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: session_autoclose 300 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_file_size 1099511627776 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: required_client_features {} 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure 0 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: last_failure_osd_epoch 0 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:07:05.962 INFO:journalctl@ceph.mon.c.smithi064.stdout: max_mds 1 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: in 0 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: up {0=24548} 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: failed 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: damaged 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: stopped 2023-06-10T21:07:05.963 INFO:journalctl@ceph.mon.c.smithi064.stdout: data_pools [4] 2023-06-10T21:07:05.964 INFO:journalctl@ceph.mon.c.smithi064.stdout: metadata_pool 3 2023-06-10T21:07:05.964 INFO:journalctl@ceph.mon.c.smithi064.stdout: inline_data disabled 2023-06-10T21:07:05.964 INFO:journalctl@ceph.mon.c.smithi064.stdout: balancer 2023-06-10T21:07:05.964 INFO:journalctl@ceph.mon.c.smithi064.stdout: bal_rank_mask -1 2023-06-10T21:07:05.964 INFO:journalctl@ceph.mon.c.smithi064.stdout: standby_count_wanted 1 2023-06-10T21:07:05.965 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi064.wuuunt{0:24548} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:07:05.965 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.965 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.965 INFO:journalctl@ceph.mon.c.smithi064.stdout: Standby daemons: 2023-06-10T21:07:05.966 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.966 INFO:journalctl@ceph.mon.c.smithi064.stdout: [mds.foo.smithi152.ciyxdy{-1:24553} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:07:05.966 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-10T21:07:05.966 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:07:05.966 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:07:05.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:07:05.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-06-10T21:07:05.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-10T21:07:05.967 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:07:05 smithi064 ceph-mon[186706]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-10T21:07:05.967 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** DB Stats ** 2023-06-10T21:07:05.968 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-06-10T21:07:05.968 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T21:07:05.968 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:07:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:07:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T21:07:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:07:05.969 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:07:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-06-10T21:07:05.970 INFO:journalctl@ceph.mon.c.smithi064.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:07:05.971 INFO:journalctl@ceph.mon.c.smithi064.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-10T21:07:05.971 INFO:journalctl@ceph.mon.c.smithi064.stdout: L0 1/0 8.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.5 0.02 0.00 1 0.019 0 0 0.0 0.0 2023-06-10T21:07:05.971 INFO:journalctl@ceph.mon.c.smithi064.stdout: L6 1/0 24.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-10T21:07:05.971 INFO:journalctl@ceph.mon.c.smithi064.stdout: Sum 2/0 33.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.5 0.02 0.00 1 0.019 0 0 0.0 0.0 2023-06-10T21:07:05.972 INFO:journalctl@ceph.mon.c.smithi064.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.5 0.02 0.00 1 0.019 0 0 0.0 0.0 2023-06-10T21:07:05.972 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.972 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** Compaction Stats [default] ** 2023-06-10T21:07:05.972 INFO:journalctl@ceph.mon.c.smithi064.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:07:05.973 INFO:journalctl@ceph.mon.c.smithi064.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T21:07:05.973 INFO:journalctl@ceph.mon.c.smithi064.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 436.5 0.02 0.00 1 0.019 0 0 0.0 0.0 2023-06-10T21:07:05.973 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: Uptime(secs): 0.1 total, 0.1 interval 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: Flush(GB): cumulative 0.008, interval 0.008 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T21:07:05.974 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T21:07:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T21:07:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout: AddFile(Keys): cumulative 0, interval 0 2023-06-10T21:07:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout: Cumulative compaction: 0.01 GB write, 157.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:07:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout: Interval compaction: 0.01 GB write, 157.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:07:05.975 INFO:journalctl@ceph.mon.c.smithi064.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T21:07:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache BinnedLRUCache@0x56498b897350#7 capacity: 512.00 MB usage: 629.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2023-06-10T21:07:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout: Block cache entry stats(count,size,portion): DataBlock(22,571.03 KB,0.108916%) FilterBlock(2,7.81 KB,0.00149012%) IndexBlock(2,50.16 KB,0.00956655%) Misc(1,0.00 KB,0%) 2023-06-10T21:07:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout: 2023-06-10T21:07:05.976 INFO:journalctl@ceph.mon.c.smithi064.stdout: ** File Read Latency Histogram By Level [default] ** 2023-06-10T21:07:08.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:08 smithi152 conmon[146558]: ts=2023-06-10T21:07:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:08.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:08 smithi152 conmon[146558]: ts=2023-06-10T21:07:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:12.023 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 2 2023-06-10T21:07:13.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:12] "GET /metrics HTTP/1.1" 200 190093 "" "Prometheus/2.33.4" 2023-06-10T21:07:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:05.763062+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-10T21:07:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.786578+0000 mon.b (mon.1) 167 : cluster [INF] mon.b calling monitor election 2023-06-10T21:07:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.795310+0000 mon.a (mon.0) 61 : cluster [INF] mon.a calling monitor election 2023-06-10T21:07:13.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.808413+0000 mon.a (mon.0) 62 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.822745+0000 mon.a (mon.0) 63 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.823473+0000 mon.a (mon.0) 64 : cluster [DBG] fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.823516+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.824042+0000 mon.a (mon.0) 66 : cluster [DBG] mgrmap e38: y(active, since 98s), standbys: x 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.824237+0000 mon.a (mon.0) 67 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.824258+0000 mon.a (mon.0) 68 : cluster [INF] Cluster is now healthy 2023-06-10T21:07:13.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: cluster 2023-06-10T21:07:11.841000+0000 mon.a (mon.0) 69 : cluster [INF] overall HEALTH_OK 2023-06-10T21:07:13.202 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:12 smithi152 conmon[97365]: audit 2023-06-10T21:07:11.991002+0000 mon.a (mon.0) 70 : audit [DBG] from='client.? 172.21.15.64:0/659499534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:07:13.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:07:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:13 smithi152 conmon[97365]: cluster 2023-06-10T21:07:12.979350+0000 2023-06-10T21:07:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:13 smithi152 conmon[97365]: mgr.y (mgr.34101) 71 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 255 B/s wr, 591 op/s 2023-06-10T21:07:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:14 smithi152 conmon[97365]: cluster 2023-06-10T21:07:13.851043+0000 mon.a (mon.0) 71 : 2023-06-10T21:07:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:14 smithi152 conmon[97365]: cluster [DBG] mgrmap e39: y(active, since 100s), standbys: x 2023-06-10T21:07:15.564 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:15 smithi064 conmon[171866]: 2023-06-10T21:07:15.558+0000 7f6cfc057700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-06-10T21:07:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:16 smithi152 conmon[97365]: cluster 2023-06-10T21:07:14.981909+0000 mgr.y (mgr. 2023-06-10T21:07:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:16 smithi152 conmon[97365]: 34101) 72 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-06-10T21:07:17.152 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:07:17.152 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (4m) 20s ago 8m 28.2M - 0.23.0 ba2b418f427c 13fd7844ce61 2023-06-10T21:07:17.152 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (3m) 24s ago 7m 61.6M - 8.3.5 dad864ee21e9 95a95736b9e9 2023-06-10T21:07:17.152 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (7m) 20s ago 7m 20.4M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:07:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (7m) 24s ago 7m 15.1M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:07:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443,9283,8765 running (2m) 24s ago 12m 447M - 18.0.0-3603-g5937c68a 8fc92d915e8e 0e93fb4f51a4 2023-06-10T21:07:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (2m) 20s ago 13m 498M - 18.0.0-3603-g5937c68a 8fc92d915e8e 2bda388cec1d 2023-06-10T21:07:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (117s) 20s ago 14m 56.0M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 6f8d9991dea0 2023-06-10T21:07:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (12m) 24s ago 12m 120M 2048M 16.2.13-257-gd8c5d349 b644dd95899d a46036733d6f 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 starting - - - 2048M 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (4m) 20s ago 8m 31.4M - 1.3.1 1dbe0e931976 c7179c864e62 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (3m) 24s ago 8m 31.6M - 1.3.1 1dbe0e931976 91ff512a05b1 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (11m) 20s ago 11m 2794M 2500M 16.2.13-257-gd8c5d349 b644dd95899d ef1eb4e15c36 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (11m) 20s ago 11m 2097M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 601741de864d 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (11m) 20s ago 11m 3723M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8cca00e2828d 2023-06-10T21:07:17.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (10m) 20s ago 10m 2529M 2500M 16.2.13-257-gd8c5d349 b644dd95899d c9c0acd987a8 2023-06-10T21:07:17.155 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (10m) 24s ago 10m 2892M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:07:17.155 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (10m) 24s ago 10m 3234M 2500M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:07:17.155 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (9m) 24s ago 9m 1544M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:07:17.155 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (9m) 24s ago 9m 2760M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:07:17.155 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (2m) 24s ago 8m 73.8M - 2.33.4 514e6a882f6e 2013ef6ec5fb 2023-06-10T21:07:18.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade pause' 2023-06-10T21:07:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[97365]: cluster 2023-06-10T21:07:16.982333+0000 mgr.y ( 2023-06-10T21:07:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[97365]: mgr.34101) 73 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:07:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[97365]: audit 2023-06-10T21:07:17.113352+0000 mgr.y (mgr.34101) 74 : audit [DBG] from='client.25180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:07:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[97365]: audit 2023-06-10T21:07:18.060596+0000 mon.b (mon.1) 2023-06-10T21:07:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[97365]: 168 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:07:18.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[146558]: ts=2023-06-10T21:07:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:18.947 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:18 smithi152 conmon[146558]: ts=2023-06-10T21:07:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:20 smithi152 conmon[97365]: cluster 2023-06-10T21:07:18.982788+0000 mgr.y (mgr.34101) 75 2023-06-10T21:07:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:20 smithi152 conmon[97365]: : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:07:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:20 smithi152 conmon[97365]: audit 2023-06-10T21:07:19.227879+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:20 smithi152 conmon[97365]: audit 2023-06-10T21:07:19.249540 2023-06-10T21:07:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:20 smithi152 conmon[97365]: +0000 mon.a (mon.0) 73 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:21.654 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:07:22.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'sleep 60' 2023-06-10T21:07:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: cluster 2023-06-10T21:07:20.983347+0000 mgr.y (mgr.34101) 76 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: audit 2023-06-10T21:07:21.277757+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: audit 2023-06-10T21:07:21.303400+0000 mon.a (mon.0 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: ) 75 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: audit 2023-06-10T21 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: :07:21.626335+0000 mgr.y (mgr. 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: 34101) 77 : audit [DBG] from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:07:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: cephadm 2023-06-10T 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: 21:07:21.626811+0000 mgr.y (mgr 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: .34101) 78 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: 06-10T21:07:21.634420+0000 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: mon.a (mon.0) 76 : 2023-06-10T21:07:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:22 smithi152 conmon[97365]: audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:22.902 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:22] "GET /metrics HTTP/1.1" 200 187102 "" "Prometheus/2.33.4" 2023-06-10T21:07:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:24 smithi152 conmon[97365]: cluster 2023-06-10T21:07:22.988171+0000 2023-06-10T21:07:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:24 smithi152 conmon[97365]: mgr.y (mgr.34101) 79 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-06-10T21:07:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:26 smithi152 conmon[97365]: cluster 2023-06-10T21:07:24.988882+0000 mgr.y (mgr 2023-06-10T21:07:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:26 smithi152 conmon[97365]: .34101) 80 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-06-10T21:07:28.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[146558]: ts=2023-06-10T21:07:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:28.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[146558]: ts=2023-06-10T21:07:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: cluster 2023-06-10T21:07:26.990894+0000 mgr.y (mgr 2023-06-10T21:07:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: .34101) 81 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06-10T21:07:27.714628+0000 mon.a (mon.0) 77 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06-10T 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 21:07:27.748323+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: -06-10T21:07:27.751306+0000 mon.b (mon.1) 169 : 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06-10T21:07:27. 2023-06-10T21:07:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 752455+0000 mon.b (mon.1) 170 : audit [INF] 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06-10T21:07:27.767123+0000 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: mon.a (mon.0) 79 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023- 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 06-10T21:07:28.017568+0000 mon.b ( 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: mon.1) 171 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:07:28.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 10T21:07:28.025934+0000 mon.b (mon 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: .1) 172 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 10T21:07:28.031346+0000 mon.b (mon 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: .1) 173 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:07:28.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: T21:07:28.068029+0000 mon.a (mon. 2023-06-10T21:07:28.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:28 smithi152 conmon[97365]: 0) 80 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:07:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:30 smithi152 conmon[97365]: cluster 2023-06-10T21:07:28.992297+0000 mgr.y (mgr 2023-06-10T21:07:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:30 smithi152 conmon[97365]: .34101) 82 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-10T21:07:33.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:32] "GET /metrics HTTP/1.1" 200 187102 "" "Prometheus/2.33.4" 2023-06-10T21:07:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:32 smithi152 conmon[97365]: cluster 2023-06-10T21:07:30.995391+0000 mgr.y (mgr 2023-06-10T21:07:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:32 smithi152 conmon[97365]: .34101) 83 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-10T21:07:34.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:33 smithi152 conmon[97365]: audit 2023-06-10T21:07:33.070713+0000 mon.b (mon 2023-06-10T21:07:34.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:33 smithi152 conmon[97365]: .1) 174 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:07:35.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:34 smithi152 conmon[97365]: cluster 2023-06-10T21:07:32.995764+0000 mgr.y (mgr.34101) 84 : cluster 2023-06-10T21:07:35.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:34 smithi152 conmon[97365]: [DBG] pgmap v63: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-06-10T21:07:37.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:36 smithi152 conmon[97365]: cluster 2023-06-10T21:07:34.996443+0000 2023-06-10T21:07:37.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:36 smithi152 conmon[97365]: mgr.y (mgr.34101) 85 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:07:38.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:37 smithi152 conmon[97365]: cluster 2023-06-10T21:07:36.996993+0000 mgr.y (mgr.34101) 2023-06-10T21:07:38.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:37 smithi152 conmon[97365]: 86 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-06-10T21:07:38.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:38 smithi152 conmon[146558]: ts=2023-06-10T21:07:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:38.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:38 smithi152 conmon[146558]: ts=2023-06-10T21:07:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:40 smithi152 conmon[97365]: cluster 2023-06-10T21:07:38.997493+0000 mgr.y 2023-06-10T21:07:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:40 smithi152 conmon[97365]: (mgr.34101) 87 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-06-10T21:07:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:42 smithi152 conmon[97365]: cluster 2023-06-10T21:07:40.998155+0000 mgr.y (mgr.34101) 88 : cluster 2023-06-10T21:07:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:42 smithi152 conmon[97365]: [DBG] pgmap v67: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:07:43.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:42] "GET /metrics HTTP/1.1" 200 187100 "" "Prometheus/2.33.4" 2023-06-10T21:07:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:44 smithi152 conmon[97365]: cluster 2023-06-10T21:07:42.998575+0000 mgr.y (mgr.34101) 89 : cluster [DBG] 2023-06-10T21:07:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:44 smithi152 conmon[97365]: pgmap v68: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-06-10T21:07:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:46 smithi152 conmon[97365]: cluster 2023-06-10T21:07:44.999252+0000 mgr.y (mgr 2023-06-10T21:07:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:46 smithi152 conmon[97365]: .34101) 90 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:07:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:48 smithi152 conmon[97365]: cluster 2023-06-10T21:07:46.999661+0000 mgr.y (mgr.34101 2023-06-10T21:07:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:48 smithi152 conmon[97365]: ) 91 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:07:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:48 smithi152 conmon[97365]: audit 2023-06-10T21:07:48.068817+0000 mon.b (mon.1) 175 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:07:48.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:48 smithi152 conmon[146558]: ts=2023-06-10T21:07:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:48.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:48 smithi152 conmon[146558]: ts=2023-06-10T21:07:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:50 smithi152 conmon[97365]: cluster 2023-06-10T21:07:49.000150+0000 mgr.y (mgr 2023-06-10T21:07:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:50 smithi152 conmon[97365]: .34101) 92 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-10T21:07:52.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:52 smithi152 conmon[97365]: cluster 2023-06-10T21:07:51.000791+0000 2023-06-10T21:07:52.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:52 smithi152 conmon[97365]: mgr.y (mgr.34101) 93 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-06-10T21:07:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:07:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:07:52] "GET /metrics HTTP/1.1" 200 187117 "" "Prometheus/2.33.4" 2023-06-10T21:07:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:54 smithi152 conmon[97365]: cluster 2023-06-10T21:07:53.001206+0000 2023-06-10T21:07:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:54 smithi152 conmon[97365]: mgr.y (mgr.34101) 94 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-06-10T21:07:56.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:56 smithi152 conmon[97365]: cluster 2023-06-10T21:07:55.001829+0000 mgr.y (mgr.34101) 95 : cluster [DBG] 2023-06-10T21:07:56.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:56 smithi152 conmon[97365]: pgmap v74: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-06-10T21:07:58.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:58 smithi152 conmon[97365]: cluster 2023-06-10T21:07:57.002961+0000 2023-06-10T21:07:58.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:07:58 smithi152 conmon[97365]: mgr.y (mgr.34101) 96 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:07:58.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:58 smithi152 conmon[146558]: ts=2023-06-10T21:07:58.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:07:58.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:07:58 smithi152 conmon[146558]: ts=2023-06-10T21:07:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:08:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:00 smithi152 conmon[97365]: cluster 2023-06-10T21:07:59.003448+0000 mgr.y ( 2023-06-10T21:08:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:00 smithi152 conmon[97365]: mgr.34101) 97 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-06-10T21:08:02.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:02 smithi152 conmon[97365]: cluster 2023-06-10T21:08:01.004046+0000 mgr.y (mgr.34101) 2023-06-10T21:08:02.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:02 smithi152 conmon[97365]: 98 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-06-10T21:08:03.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:02] "GET /metrics HTTP/1.1" 200 187117 "" "Prometheus/2.33.4" 2023-06-10T21:08:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:03 smithi152 conmon[97365]: audit 2023-06-10T21:08:03.069788+0000 mon.b (mon.1) 176 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:04.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:04 smithi152 conmon[97365]: cluster 2023-06-10T21:08:03.009085+0000 mgr.y 2023-06-10T21:08:04.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:04 smithi152 conmon[97365]: (mgr.34101) 99 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-10T21:08:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:06 smithi152 conmon[97365]: cluster 2023-06-10T21:08:05.009768+0000 mgr.y (mgr. 2023-06-10T21:08:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:06 smithi152 conmon[97365]: 34101) 100 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-10T21:08:08.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:08 smithi152 conmon[97365]: cluster 2023-06-10T21:08:07.010191+0000 mgr.y ( 2023-06-10T21:08:08.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:08 smithi152 conmon[97365]: mgr.34101) 101 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-10T21:08:08.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:08:08 smithi152 conmon[146558]: ts=2023-06-10T21:08:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:08:08.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:08:08 smithi152 conmon[146558]: ts=2023-06-10T21:08:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi152.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.64\", device_class=\"ssd\", hostname=\"smithi064\", instance=\"smithi064:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.64\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-10T21:08:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:10 smithi152 conmon[97365]: cluster 2023-06-10T21:08:09.010742+0000 mgr.y (mgr.34101) 102 : cluster [DBG] 2023-06-10T21:08:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:10 smithi152 conmon[97365]: pgmap v81: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-06-10T21:08:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:12 smithi152 conmon[97365]: cluster 2023-06-10T21:08:11.011321+0000 mgr.y ( 2023-06-10T21:08:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:12 smithi152 conmon[97365]: mgr.34101) 103 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-06-10T21:08:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:12] "GET /metrics HTTP/1.1" 200 187120 "" "Prometheus/2.33.4" 2023-06-10T21:08:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:14 smithi152 conmon[97365]: cluster 2023-06-10T21:08:13.011734+0000 mgr.y (mgr.34101) 104 : cluster 2023-06-10T21:08:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:14 smithi152 conmon[97365]: [DBG] pgmap v83: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:08:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:16 smithi152 conmon[97365]: cluster 2023-06-10T21:08:15.012330+0000 2023-06-10T21:08:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:16 smithi152 conmon[97365]: mgr.y (mgr.34101) 105 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2023-06-10T21:08:18.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:18 smithi152 conmon[97365]: cluster 2023-06-10T21:08:17.012816+0000 mgr.y (mgr.34101 2023-06-10T21:08:18.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:18 smithi152 conmon[97365]: ) 106 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-06-10T21:08:18.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:18 smithi152 conmon[97365]: audit 2023-06-10T21:08:18.071950+0000 mon.b (mon.1) 177 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:20 smithi152 conmon[97365]: cluster 2023-06-10T21:08:19.013329+0000 mgr.y (mgr 2023-06-10T21:08:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:20 smithi152 conmon[97365]: .34101) 107 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-06-10T21:08:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:22 smithi152 conmon[97365]: cluster 2023-06-10T21:08:21.014343+0000 mgr.y (mgr.34101) 108 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-06-10T21:08:23.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:22] "GET /metrics HTTP/1.1" 200 187120 "" "Prometheus/2.33.4" 2023-06-10T21:08:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:24 smithi152 conmon[97365]: cluster 2023-06-10T21:08:23.014753 2023-06-10T21:08:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:24 smithi152 conmon[97365]: +0000 mgr.y (mgr.34101) 109 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-10T21:08:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:26 smithi152 conmon[97365]: cluster 2023-06-10T21:08:25.015442+0000 mgr.y (mgr.34101) 110 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:27.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade resume' 2023-06-10T21:08:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:28 smithi152 conmon[97365]: cluster 2023-06-10T21:08:27.015939+0000 mgr.y (mgr. 2023-06-10T21:08:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:28 smithi152 conmon[97365]: 34101) 111 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-10T21:08:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:28 smithi152 conmon[97365]: audit 2023-06-10T21:08:28.326199+0000 mon.b (mon.1) 178 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:30.113 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:30 smithi152 conmon[97365]: cluster 2023-06-10T21:08:29.016443+0000 mgr.y (mgr.34101) 112 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-06-10T21:08:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:30 smithi152 conmon[97365]: audit 2023-06-10T21:08:30.106257+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:31.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'echo wait for all mons to upgrade' 2023-06-10T21:08:31.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:31 smithi152 conmon[97365]: audit 2023-06-10T21:08:30.094205+0000 mgr.y (mgr.34101 2023-06-10T21:08:31.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:31 smithi152 conmon[97365]: ) 113 : audit [DBG] from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:31.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:31 smithi152 conmon[97365]: cephadm 2023-06-10T21:08:30.094667+0000 mgr.y (mgr.34101) 114 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:32 smithi152 conmon[97365]: cluster 2023-06-10T21:08:31.020525+0000 2023-06-10T21:08:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:32 smithi152 conmon[97365]: mgr.y (mgr.34101) 115 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-06-10T21:08:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:32 smithi152 conmon[97365]: audit 2023-06-10T21:08:32.280205+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:32 smithi152 conmon[97365]: audit 2023-06-10T21:08:32.301014+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:33.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:32] "GET /metrics HTTP/1.1" 200 187120 "" "Prometheus/2.33.4" 2023-06-10T21:08:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:33 smithi152 conmon[97365]: audit 2023-06-10T21:08:33.077390+0000 mon.b (mon. 2023-06-10T21:08:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:33 smithi152 conmon[97365]: 1) 179 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:33.951 INFO:teuthology.orchestra.run.smithi064.stdout:wait for all mons to upgrade 2023-06-10T21:08:34.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-06-10T21:08:34.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:34 smithi152 conmon[97365]: cluster 2023-06-10T21:08:33.023682+0000 mgr.y 2023-06-10T21:08:34.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:34 smithi152 conmon[97365]: (mgr.34101) 116 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:08:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:36 smithi152 conmon[97365]: cluster 2023-06-10T21:08:35.024421+0000 mgr.y (mgr.34101) 117 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-10T21:08:38.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: audit 2023-06-10T21:08:36.726586+0000 mon.a (mon.0 2023-06-10T21:08:38.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: ) 84 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: audit 2023-06-10T21:08:36.750813+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: audit 2023-06-10 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: T21:08:37.651801+0000 mon.b (mon.1) 180 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: audit 2023-06 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: -10T21:08:37.653784+0000 mon.b (mon.1) 181 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: audit 2023-06- 2023-06-10T21:08:38.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: 10T21:08:37.678506+0000 mon.a (mon.0) 2023-06-10T21:08:38.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:37 smithi152 conmon[97365]: 86 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:39.043 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: cluster 2023-06-10T21:08:37.024844+0000 mgr.y (mgr.34101) 2023-06-10T21:08:39.043 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: 118 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-06-10T21:08:39.043 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: audit 2023-06-10T21:08:37.930499+0000 mon.b (mon.1) 182 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:39.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: audit 2023-06-10T21:08:37.934706+0000 mon.b (mon.1) 183 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:08:39.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: audit 2023-06-10T21:08:37.936526+0000 mon.b (mon.1) 184 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:08:39.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: audit 2023-06-10T21:08:37.942506+0000 mon.b (mon.1) 185 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-10T21:08:39.044 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:38 smithi152 conmon[97365]: audit 2023-06-10T21:08:38.122008+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.64:0/3928636637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:40.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:39 smithi152 conmon[97365]: cephadm 2023-06-10T21:08:37.943174+0000 2023-06-10T21:08:40.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:39 smithi152 conmon[97365]: mgr.y (mgr.34101) 119 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-06-10T21:08:41.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:40 smithi152 conmon[97365]: cluster 2023-06-10T21:08:39.025338+0000 mgr.y 2023-06-10T21:08:41.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:40 smithi152 conmon[97365]: (mgr.34101) 120 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-06-10T21:08:41.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:40 smithi152 conmon[97365]: audit 2023-06-10T21:08:40.707210+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.64:0/2037630736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: cephadm 2023-06-10T21:08:40.907929+0000 mgr.y (mgr.34101 2023-06-10T21:08:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: ) 121 : cephadm [INF] Upgrade: Updating mon.b 2023-06-10T21:08:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: audit 2023-06-10T21:08:40.926302+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: audit 2023-06-10T21:08:40.939448+0000 mon.b (mon.1) 186 : audit [INF] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: audit 2023-06-10T21:08:40.940709+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: audit 2023-06-10T21:08:40.943284+0000 mon.b (mon.1) 187 : audit [DBG] from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: cephadm 2023-06-10T21:08:40.944431+0000 mgr.y (mgr.34101) 122 : cephadm [INF] Deploying daemon mon.b on smithi152 2023-06-10T21:08:42.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:41 smithi152 conmon[97365]: cluster 2023-06-10T21:08:41.027659+0000 mgr.y (mgr.34101) 123 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 948 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-06-10T21:08:43.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:42] "GET /metrics HTTP/1.1" 200 187121 "" "Prometheus/2.33.4" 2023-06-10T21:08:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:42 smithi152 systemd[1]: Stopping Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:08:43.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:43 smithi152 conmon[97365]: debug 2023-06-10T21:08:43.256+0000 7f69a9754700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-10T21:08:43.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:43 smithi152 conmon[97365]: debug 2023-06-10T21:08:43.256+0000 7f69a9754700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-06-10T21:08:43.578 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:43 smithi152 bash[156418]: a46036733d6f84427973cdcebacc3259be60cf6fed4e24399d79d4ce1c4203a9 2023-06-10T21:08:43.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE Bus STOPPING 2023-06-10T21:08:43.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:08:43.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE Bus STOPPED 2023-06-10T21:08:43.659 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE Bus STARTING 2023-06-10T21:08:43.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE Serving on http://:::9283 2023-06-10T21:08:43.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:43 smithi064 conmon[171866]: [10/Jun/2023:21:08:43] ENGINE Bus STARTED 2023-06-10T21:08:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:44 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.b.service: Succeeded. 2023-06-10T21:08:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:44 smithi152 systemd[1]: Stopped Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:08:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:44 smithi152 systemd[1]: Starting Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: set uid:gid to 167:167 (ceph:ceph) 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc), process ceph-mon, pid 7 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: pidfile_write: ignore empty --pid-file 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 bash[156756]: a70d292c32729fc31c1c53b15b318d4d35a77c73d60836455fbe6fbe7322f26f 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: load: jerasure load: lrc 2023-06-10T21:08:45.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: RocksDB version: 7.9.2 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Git sha 0 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Compile date 2023-06-09 19:19:26 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: DB SUMMARY 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: DB Session ID: R9XL3G7MJLKJHQGBQFYP 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: CURRENT file: CURRENT 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: IDENTITY file: IDENTITY 2023-06-10T21:08:45.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: MANIFEST file: MANIFEST-000005 size: 1092 Bytes 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000023.sst 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000021.log size: 735849 ; 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.error_if_exists: 0 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.create_if_missing: 0 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.paranoid_checks: 1 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.flush_verify_memtable_count: 1 2023-06-10T21:08:45.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-10T21:08:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-10T21:08:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.env: 0x55f96d148720 2023-06-10T21:08:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.fs: PosixFileSystem 2023-06-10T21:08:45.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.info_log: 0x55f96f7953e0 2023-06-10T21:08:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_file_opening_threads: 16 2023-06-10T21:08:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.statistics: (nil) 2023-06-10T21:08:45.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.use_fsync: 0 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_log_file_size: 0 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.log_file_time_to_roll: 0 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.keep_log_file_num: 1000 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.recycle_log_file_num: 0 2023-06-10T21:08:45.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_fallocate: 1 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_mmap_reads: 0 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_mmap_writes: 0 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.use_direct_reads: 0 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.create_missing_column_families: 0 2023-06-10T21:08:45.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.db_log_dir: 2023-06-10T21:08:45.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.wal_dir: 2023-06-10T21:08:45.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.table_cache_numshardbits: 6 2023-06-10T21:08:45.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.WAL_ttl_seconds: 0 2023-06-10T21:08:45.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.WAL_size_limit_MB: 0 2023-06-10T21:08:45.453 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.is_fd_close_on_exec: 1 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.advise_random_on_open: 1 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.db_write_buffer_size: 0 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.write_buffer_manager: 0x55f96ea24c80 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-10T21:08:45.454 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.use_adaptive_mutex: 0 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.rate_limiter: (nil) 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.wal_recovery_mode: 2 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enable_thread_tracking: 0 2023-06-10T21:08:45.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enable_pipelined_write: 0 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.unordered_write: 0 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.row_cache: None 2023-06-10T21:08:45.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.wal_filter: None 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_ingest_behind: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.two_write_queues: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.manual_wal_flush: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.wal_compression: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.atomic_flush: 0 2023-06-10T21:08:45.457 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.persist_stats_to_disk: 0 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.write_dbid_to_manifest: 0 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.log_readahead_size: 0 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.best_efforts_recovery: 0 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-10T21:08:45.458 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.allow_data_in_errors: 0 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.db_host_id: __hostname__ 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enforce_single_del_contracts: true 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_background_jobs: 2 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_background_compactions: -1 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_subcompactions: 1 2023-06-10T21:08:45.459 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-10T21:08:45.460 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-10T21:08:45.460 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.delayed_write_rate : 16777216 2023-06-10T21:08:45.460 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_total_wal_size: 0 2023-06-10T21:08:45.460 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-10T21:08:45.460 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.stats_dump_period_sec: 600 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.stats_persist_period_sec: 600 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_open_files: -1 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bytes_per_sync: 0 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.wal_bytes_per_sync: 0 2023-06-10T21:08:45.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.strict_bytes_per_sync: 0 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_readahead_size: 0 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_background_flushes: -1 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Compression algorithms supported: 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kZSTD supported: 0 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kXpressCompression supported: 0 2023-06-10T21:08:45.462 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kLZ4HCCompression supported: 1 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kZlibCompression supported: 1 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kSnappyCompression supported: 1 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kLZ4Compression supported: 1 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: kBZip2Compression supported: 0 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Fast CRC32 supported: Supported on x86 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: DMutex implementation: pthread_mutex_t 2023-06-10T21:08:45.463 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.merge_operator: 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_filter: None 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_filter_factory: None 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.sst_partitioner_factory: None 2023-06-10T21:08:45.464 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.memtable_factory: SkipListFactory 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.table_factory: BlockBasedTable 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f96f795500) 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks: 1 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-10T21:08:45.465 INFO:journalctl@ceph.mon.b.smithi152.stdout: pin_top_level_index_and_filter: 1 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_type: 0 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_index_type: 0 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_shortening: 1 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: checksum: 4 2023-06-10T21:08:45.466 INFO:journalctl@ceph.mon.b.smithi152.stdout: no_block_cache: 0 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache: 0x55f96eaa71f0 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_name: BinnedLRUCache 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_options: 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: capacity : 536870912 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_shard_bits : 4 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: strict_capacity_limit : 0 2023-06-10T21:08:45.467 INFO:journalctl@ceph.mon.b.smithi152.stdout: high_pri_pool_ratio: 0.000 2023-06-10T21:08:45.468 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_cache_compressed: (nil) 2023-06-10T21:08:45.468 INFO:journalctl@ceph.mon.b.smithi152.stdout: persistent_cache: (nil) 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size: 4096 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_size_deviation: 10 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_restart_interval: 16 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: index_block_restart_interval: 1 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_block_size: 4096 2023-06-10T21:08:45.469 INFO:journalctl@ceph.mon.b.smithi152.stdout: partition_filters: 0 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: use_delta_encoding: 1 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: filter_policy: bloomfilter 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: whole_key_filtering: 1 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: verify_compression: 0 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: read_amp_bytes_per_bit: 0 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: format_version: 5 2023-06-10T21:08:45.470 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_index_compression: 1 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout: block_align: 0 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_auto_readahead_size: 262144 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout: prepopulate_block_cache: 0 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout: initial_auto_readahead_size: 8192 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout: num_file_reads_for_auto_readahead: 2 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.write_buffer_size: 33554432 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_write_buffer_number: 2 2023-06-10T21:08:45.471 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression: NoCompression 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression: Disabled 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.prefix_extractor: nullptr 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.num_levels: 7 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-10T21:08:45.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-10T21:08:45.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.window_bits: -14 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.level: 32767 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.strategy: 0 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-10T21:08:45.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.enabled: false 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.target_file_size_base: 67108864 2023-06-10T21:08:45.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.target_file_size_multiplier: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-10T21:08:45.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.arena_block_size: 1048576 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-10T21:08:45.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.disable_auto_compactions: 0 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-10T21:08:45.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.table_properties_collectors: 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.inplace_update_support: 0 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.inplace_update_num_locks: 10000 2023-06-10T21:08:45.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.memtable_huge_page_size: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.bloom_locality: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.max_successive_merges: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.optimize_filters_for_hits: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.paranoid_file_checks: 0 2023-06-10T21:08:45.480 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.force_consistency_checks: 1 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.report_bg_io_stats: 0 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.ttl: 2592000 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.periodic_compaction_seconds: 0 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enable_blob_files: false 2023-06-10T21:08:45.481 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.min_blob_size: 0 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_file_size: 268435456 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_compression_type: NoCompression 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.enable_blob_garbage_collection: false 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.blob_file_starting_level: 0 2023-06-10T21:08:45.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 15910, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8a0dc8cf-cdd2-4739-b556-d9bbc87d4252 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431325186095, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-06-10T21:08:45.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431325191374, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 490387, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 15915, "largest_seqno": 16051, "table_properties": {"data_size": 488534, "index_size": 649, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 197, "raw_key_size": 1354, "raw_average_key_size": 23, "raw_value_size": 486826, "raw_average_value_size": 8393, "num_data_blocks": 28, "num_entries": 58, "num_filter_entries": 58, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1686431325, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8a0dc8cf-cdd2-4739-b556-d9bbc87d4252", "db_session_id": "R9XL3G7MJLKJHQGBQFYP", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: EVENT_LOG_v1 {"time_micros": 1686431325191484, "job": 1, "event": "recovery_finished"} 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 systemd[1]: Started Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f96eb40000 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: DB pointer 0x55f96eb2e000 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-10T21:08:45.484 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** DB Stats ** 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-10T21:08:45.485 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: L0 1/0 478.89 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 119.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: L6 1/0 31.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-10T21:08:45.486 INFO:journalctl@ceph.mon.b.smithi152.stdout: Sum 2/0 32.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 119.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 119.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** Compaction Stats [default] ** 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 119.9 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.487 INFO:journalctl@ceph.mon.b.smithi152.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2023-06-10T21:08:45.488 INFO:journalctl@ceph.mon.b.smithi152.stdout: Cumulative compaction: 0.00 GB write, 27.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: Interval compaction: 0.00 GB write, 27.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache BinnedLRUCache@0x55f96eaa71f0#7 capacity: 512.00 MB usage: 1.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.23 KB,4.47035e-05%) IndexBlock(1,0.77 KB,0.000146031%) Misc(1,0.00 KB,0%) 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2023-06-10T21:08:45.489 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: starting mon.b rank 1 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???) e3 preinit fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).mds e7 new map 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).mds e7 print_map 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout: e7 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout: legacy client fscid: 1 2023-06-10T21:08:45.490 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: Filesystem 'foo' (1) 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: fs_name foo 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: epoch 5 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: created 2023-06-10T20:59:25.413290+0000 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: modified 2023-06-10T20:59:39.018592+0000 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: tableserver 0 2023-06-10T21:08:45.491 INFO:journalctl@ceph.mon.b.smithi152.stdout: root 0 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_timeout 60 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: session_autoclose 300 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_file_size 1099511627776 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: required_client_features {} 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure 0 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: last_failure_osd_epoch 0 2023-06-10T21:08:45.492 INFO:journalctl@ceph.mon.b.smithi152.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: max_mds 1 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: in 0 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: up {0=24548} 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: failed 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: damaged 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: stopped 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: data_pools [4] 2023-06-10T21:08:45.493 INFO:journalctl@ceph.mon.b.smithi152.stdout: metadata_pool 3 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: inline_data disabled 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: balancer 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: bal_rank_mask -1 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: standby_count_wanted 1 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi064.wuuunt{0:24548} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.64:6834/2338514290,v1:172.21.15.64:6835/2338514290] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.494 INFO:journalctl@ceph.mon.b.smithi152.stdout: Standby daemons: 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout: 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout: [mds.foo.smithi152.ciyxdy{-1:24553} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.152:6832/437872745,v1:172.21.15.152:6833/437872745] compat {c=[1],r=[1],i=[7ff]}] 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-10T21:08:45.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expand map: {default=false} 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta from 'false' to 'false' 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expanded map: {default=false} 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expand map: {default=info} 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta from 'info' to 'info' 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expanded map: {default=info} 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expand map: {default=daemon} 2023-06-10T21:08:45.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta from 'daemon' to 'daemon' 2023-06-10T21:08:45.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expanded map: {default=daemon} 2023-06-10T21:08:45.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expand map: {default=debug} 2023-06-10T21:08:45.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta from 'debug' to 'debug' 2023-06-10T21:08:45.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:45 smithi152 ceph-mon[156819]: expand_channel_meta expanded map: {default=debug} 2023-06-10T21:08:48.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mkfs b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:08:48.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Updating mgr.y 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Deploying daemon mgr.y on smithi064 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Failing over to other MGR 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:08:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Updating mgr.x 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Deploying daemon mgr.x on smithi152 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Setting container_image for all mgr 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: It appears safe to stop mon.a 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Updating mon.a 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Deploying daemon mon.a on smithi064 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:05:35] ENGINE Bus STARTING 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:05:35] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:05:35] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: [10/Jun/2023:21:05:35] ENGINE Bus STARTED 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: It appears safe to stop mon.c 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Updating mon.c 2023-06-10T21:08:48.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Deploying daemon mon.c on smithi064 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1880016430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mgrmap e38: y(active, since 98s), standbys: x 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:08:48.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: overall HEALTH_OK 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/659499534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v53: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 255 B/s wr, 591 op/s 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mgrmap e39: y(active, since 100s), standbys: x 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v54: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v55: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.25180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v56: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v57: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v58: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v59: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v60: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.168 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v61: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v62: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-10T21:08:48.170 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v63: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v64: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v65: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v66: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v67: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v68: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v69: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v70: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v71: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v72: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v73: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v74: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v75: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v76: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v77: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v78: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v79: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v80: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v81: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v82: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-06-10T21:08:48.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v83: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v84: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v85: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v86: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v87: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v88: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v91: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v92: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-06-10T21:08:48.175 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v94: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.176 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-06-10T21:08:48.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.177 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mkfs b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:08:48.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:08:48.177 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Updating mgr.y 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Deploying daemon mgr.y on smithi064 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Failing over to other MGR 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:08:48.178 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.179 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Updating mgr.x 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Deploying daemon mgr.x on smithi152 2023-06-10T21:08:48.180 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Setting container_image for all mgr 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: It appears safe to stop mon.a 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Updating mon.a 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Deploying daemon mon.a on smithi064 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:05:35] ENGINE Bus STARTING 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:05:35] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:05:35] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: [10/Jun/2023:21:05:35] ENGINE Bus STARTED 2023-06-10T21:08:48.181 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.182 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:08:48.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:08:48.183 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: It appears safe to stop mon.b 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3928636637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2037630736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Upgrade: Updating mon.b 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:48.184 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: Deploying daemon mon.b on smithi152 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: pgmap v97: 89 pgs: 89 active+clean; 948 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/486685667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: overall HEALTH_OK 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mon.a calling monitor election 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mon.b calling monitor election 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mon.c calling monitor election 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: mgrmap e39: y(active, since 3m), standbys: x 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: overall HEALTH_OK 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.187 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.188 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: It appears safe to stop mon.c 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Updating mon.c 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Deploying daemon mon.c on smithi064 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1880016430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mgrmap e38: y(active, since 98s), standbys: x 2023-06-10T21:08:48.189 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: overall HEALTH_OK 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/659499534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v53: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 255 B/s wr, 591 op/s 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mgrmap e39: y(active, since 100s), standbys: x 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v54: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-06-10T21:08:48.190 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v55: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.25180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v56: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v57: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.191 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v58: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v59: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v60: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.192 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.193 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v61: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v62: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v63: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v64: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v65: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v66: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-06-10T21:08:48.194 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v67: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v68: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v69: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v70: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v71: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v72: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v73: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-06-10T21:08:48.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v74: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v75: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v76: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v77: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v78: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v79: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-10T21:08:48.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v80: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v81: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v82: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v83: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v84: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v85: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v86: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-06-10T21:08:48.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v87: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v88: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v91: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v92: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v94: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-10T21:08:48.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:08:48.200 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: It appears safe to stop mon.b 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3928636637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2037630736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Upgrade: Updating mon.b 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.201 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: Deploying daemon mon.b on smithi152 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: pgmap v97: 89 pgs: 89 active+clean; 948 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.202 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/486685667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: overall HEALTH_OK 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mon.a calling monitor election 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mon.b calling monitor election 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mon.c calling monitor election 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.203 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: mgrmap e39: y(active, since 3m), standbys: x 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: overall HEALTH_OK 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.204 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.205 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mkfs b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Reconfiguring daemon grafana.a on smithi152 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Updating mgr.y 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Deploying daemon mgr.y on smithi064 2023-06-10T21:08:48.206 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Need to upgrade myself (mgr.x) 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Failing over to other MGR 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:04:36] ENGINE Bus STARTING 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:04:36] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:04:36] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:04:36] ENGINE Bus STARTED 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.207 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.208 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Reconfiguring prometheus.a (dependencies changed)... 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Reconfiguring daemon prometheus.a on smithi152 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Updating mgr.x 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Deploying daemon mgr.x on smithi152 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Setting container_image for all mgr 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: It appears safe to stop mon.a 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Updating mon.a 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Deploying daemon mon.a on smithi064 2023-06-10T21:08:48.209 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:05:35] ENGINE Bus STARTING 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:05:35] ENGINE Serving on https://172.21.15.64:7150 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:05:35] ENGINE Serving on http://172.21.15.64:8765 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: [10/Jun/2023:21:05:35] ENGINE Bus STARTED 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/etc/ceph/ceph.conf 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/etc/ceph/ceph.conf 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.210 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.conf 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/etc/ceph/ceph.client.admin.keyring 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Updating smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/config/ceph.client.admin.keyring 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: It appears safe to stop mon.c 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Updating mon.c 2023-06-10T21:08:48.211 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Deploying daemon mon.c on smithi064 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1880016430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mgrmap e38: y(active, since 98s), standbys: x 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: overall HEALTH_OK 2023-06-10T21:08:48.212 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/659499534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v53: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 255 B/s wr, 591 op/s 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mgrmap e39: y(active, since 100s), standbys: x 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v54: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 255 B/s wr, 890 op/s 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v55: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.25180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.213 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v56: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v57: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.25183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.214 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v58: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 574 op/s 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v59: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v60: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v61: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v62: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v63: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 565 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v64: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v65: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v66: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v67: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-06-10T21:08:48.217 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v68: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v69: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v70: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v71: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v72: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v73: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v74: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-06-10T21:08:48.218 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v75: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v76: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v77: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v78: 89 pgs: 89 active+clean; 2.7 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v79: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v80: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-10T21:08:48.219 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v81: 89 pgs: 89 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 749 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v82: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v83: 89 pgs: 89 active+clean; 2.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v84: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 981 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v85: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v86: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v87: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-06-10T21:08:48.220 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v88: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v89: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v90: 89 pgs: 89 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v91: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.221 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v92: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v93: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v94: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.222 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v95: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-10T21:08:48.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: It appears safe to stop mon.b 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3928636637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v96: 89 pgs: 89 active+clean; 1.1 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2037630736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Upgrade: Updating mon.b 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:48.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: Deploying daemon mon.b on smithi152 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: pgmap v97: 89 pgs: 89 active+clean; 948 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.225 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/486685667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: overall HEALTH_OK 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mon.a calling monitor election 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mon.b calling monitor election 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mon.c calling monitor election 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-10T21:08:48.226 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: monmap e4: 3 mons at {a=[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0],b=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],c=[v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]} removed_ranks: {} 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: fsmap foo:1 {0=foo.smithi064.wuuunt=up:active} 1 up:standby 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: osdmap e77: 8 total, 8 up, 8 in 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: mgrmap e39: y(active, since 3m), standbys: x 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: overall HEALTH_OK 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:48.227 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:08:48.842 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 3 2023-06-10T21:08:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[180297]: pgmap v100: 89 pgs: 89 active+clean; 721 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-10T21:08:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/594123178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[186706]: pgmap v100: 89 pgs: 89 active+clean; 721 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-10T21:08:49.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:49.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/594123178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:48 smithi152 ceph-mon[156819]: pgmap v100: 89 pgs: 89 active+clean; 721 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-10T21:08:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:08:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/594123178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-10T21:08:50.660 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:50 smithi064 conmon[171866]: 2023-06-10T21:08:50.222+0000 7f6cfc057700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-06-10T21:08:50.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:08:51.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:50 smithi064 ceph-mon[180297]: pgmap v101: 89 pgs: 89 active+clean; 690 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-06-10T21:08:51.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:50 smithi064 ceph-mon[186706]: pgmap v101: 89 pgs: 89 active+clean; 690 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-06-10T21:08:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:50 smithi152 ceph-mon[156819]: pgmap v101: 89 pgs: 89 active+clean; 690 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-06-10T21:08:52.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[180297]: pgmap v102: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-10T21:08:52.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:52.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:52.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[186706]: pgmap v102: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-10T21:08:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:52.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:52 smithi152 ceph-mon[156819]: pgmap v102: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-10T21:08:52.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:52.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:08:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:08:52] "GET /metrics HTTP/1.1" 200 183993 "" "Prometheus/2.33.4" 2023-06-10T21:08:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:53.957 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:08:53.957 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (5m) 17s ago 10m 28.6M - 0.23.0 ba2b418f427c 13fd7844ce61 2023-06-10T21:08:53.957 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (4m) 2s ago 9m 58.9M - 8.3.5 dad864ee21e9 95a95736b9e9 2023-06-10T21:08:53.957 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (9m) 17s ago 9m 21.2M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (9m) 2s ago 9m 15.8M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443,9283,8765 running (3m) 2s ago 13m 448M - 18.0.0-3603-g5937c68a 8fc92d915e8e 0e93fb4f51a4 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (4m) 17s ago 15m 503M - 18.0.0-3603-g5937c68a 8fc92d915e8e 2bda388cec1d 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (3m) 17s ago 15m 73.5M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 6f8d9991dea0 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (9s) 2s ago 14m 24.1M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e a70d292c3272 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (108s) 17s ago 14m 60.3M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 925081a4753e 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (6m) 17s ago 10m 29.9M - 1.3.1 1dbe0e931976 c7179c864e62 2023-06-10T21:08:53.958 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (5m) 2s ago 10m 33.3M - 1.3.1 1dbe0e931976 91ff512a05b1 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (13m) 17s ago 13m 3172M 2500M 16.2.13-257-gd8c5d349 b644dd95899d ef1eb4e15c36 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (13m) 17s ago 13m 2522M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 601741de864d 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (12m) 17s ago 12m 4160M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8cca00e2828d 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (12m) 17s ago 12m 3254M 2500M 16.2.13-257-gd8c5d349 b644dd95899d c9c0acd987a8 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (12m) 2s ago 12m 3383M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (11m) 2s ago 11m 3843M 2500M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:08:53.959 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (11m) 2s ago 11m 1800M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:08:53.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (11m) 2s ago 11m 3426M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:08:53.960 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (4m) 2s ago 10m 84.2M - 2.33.4 514e6a882f6e 2013ef6ec5fb 2023-06-10T21:08:54.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:54 smithi064 ceph-mon[180297]: pgmap v103: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-10T21:08:54.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:54 smithi064 ceph-mon[186706]: pgmap v103: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-10T21:08:54.910 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade pause' 2023-06-10T21:08:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:54 smithi152 ceph-mon[156819]: pgmap v103: 89 pgs: 89 active+clean; 492 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-10T21:08:55.676 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:55 smithi152 ceph-mon[156819]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:55.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:55 smithi064 ceph-mon[186706]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:55.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:55 smithi064 ceph-mon[180297]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:56.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:56 smithi064 ceph-mon[186706]: pgmap v104: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-10T21:08:56.789 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:56 smithi064 ceph-mon[180297]: pgmap v104: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-10T21:08:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:56 smithi152 ceph-mon[156819]: pgmap v104: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-10T21:08:57.876 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:58.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: pgmap v105: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:08:58.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: Reconfiguring mon.a (monmap changed)... 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:58.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.801 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: Reconfiguring daemon mon.a on smithi064 2023-06-10T21:08:58.801 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.805 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: pgmap v105: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.806 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: Reconfiguring mon.a (monmap changed)... 2023-06-10T21:08:58.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:58.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:58.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: Reconfiguring daemon mon.a on smithi064 2023-06-10T21:08:58.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'sleep 60' 2023-06-10T21:08:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: pgmap v105: 89 pgs: 89 active+clean; 281 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:08:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:08:58.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:58.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: Reconfiguring mon.a (monmap changed)... 2023-06-10T21:08:58.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:08:58.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:08:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:08:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: Reconfiguring daemon mon.a on smithi064 2023-06-10T21:08:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:08:59.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:59 smithi064 ceph-mon[180297]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:59.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:08:59 smithi064 ceph-mon[180297]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:59.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:59 smithi064 ceph-mon[186706]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:59.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:08:59 smithi064 ceph-mon[186706]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:08:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:59 smithi152 ceph-mon[156819]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:08:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:08:59 smithi152 ceph-mon[156819]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:09:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: pgmap v106: 89 pgs: 89 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-06-10T21:09:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: pgmap v106: 89 pgs: 89 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-06-10T21:09:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: pgmap v106: 89 pgs: 89 active+clean; 213 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2023-06-10T21:09:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:00.976 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 118118 objects 2023-06-10T21:09:00.976 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :157.976 2023-06-10T21:09:01.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:01 smithi064 ceph-mon[186706]: Reconfiguring mgr.y (monmap changed)... 2023-06-10T21:09:01.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:01 smithi064 ceph-mon[186706]: Reconfiguring daemon mgr.y on smithi064 2023-06-10T21:09:01.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:01 smithi064 ceph-mon[180297]: Reconfiguring mgr.y (monmap changed)... 2023-06-10T21:09:01.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:01 smithi064 ceph-mon[180297]: Reconfiguring daemon mgr.y on smithi064 2023-06-10T21:09:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:01 smithi152 ceph-mon[156819]: Reconfiguring mgr.y (monmap changed)... 2023-06-10T21:09:01.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:01 smithi152 ceph-mon[156819]: Reconfiguring daemon mgr.y on smithi064 2023-06-10T21:09:02.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: pgmap v107: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-06-10T21:09:02.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: pgmap v107: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-06-10T21:09:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:02.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:02.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: pgmap v107: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2023-06-10T21:09:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:02.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:02.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:02.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:03.021 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-06-10T21:09:03.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-06-10T21:09:03.051 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:02] "GET /metrics HTTP/1.1" 200 183993 "" "Prometheus/2.33.4" 2023-06-10T21:09:03.808 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:09:03.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[186706]: Reconfiguring mon.c (monmap changed)... 2023-06-10T21:09:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[186706]: Reconfiguring daemon mon.c on smithi064 2023-06-10T21:09:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[180297]: Reconfiguring mon.c (monmap changed)... 2023-06-10T21:09:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[180297]: Reconfiguring daemon mon.c on smithi064 2023-06-10T21:09:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:03 smithi152 ceph-mon[156819]: Reconfiguring mon.c (monmap changed)... 2023-06-10T21:09:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:03 smithi152 ceph-mon[156819]: Reconfiguring daemon mon.c on smithi064 2023-06-10T21:09:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:05.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: pgmap v108: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-06-10T21:09:05.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.136 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.136 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: Reconfiguring osd.0 (monmap changed)... 2023-06-10T21:09:05.136 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:09:05.136 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:05.137 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.0 on smithi064 2023-06-10T21:09:05.137 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: pgmap v108: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-06-10T21:09:05.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: Reconfiguring osd.0 (monmap changed)... 2023-06-10T21:09:05.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:09:05.138 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:05.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:04 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.0 on smithi064 2023-06-10T21:09:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: pgmap v108: 89 pgs: 89 active+clean; 71 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 561 op/s 2023-06-10T21:09:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: Reconfiguring osd.0 (monmap changed)... 2023-06-10T21:09:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:09:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:05.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:04 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.0 on smithi064 2023-06-10T21:09:06.909 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_1' removed 2023-06-10T21:09:07.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: pgmap v109: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:09:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:09:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:09:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: pgmap v109: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:09:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:09:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:07.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:09:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: pgmap v109: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:09:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:09:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:09:07.782 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:09:07.783 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:09:07.784 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-06-10T21:09:07.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_2 16 2023-06-10T21:09:07.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: Reconfiguring osd.1 (monmap changed)... 2023-06-10T21:09:07.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.1 on smithi064 2023-06-10T21:09:07.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: failed to encode map e78 with expected crc 2023-06-10T21:09:07.848 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: failed to encode map e78 with expected crc 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: osdmap e78: 8 total, 8 up, 8 in 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:09:07.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: Reconfiguring osd.1 (monmap changed)... 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.1 on smithi064 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: failed to encode map e78 with expected crc 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: failed to encode map e78 with expected crc 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: osdmap e78: 8 total, 8 up, 8 in 2023-06-10T21:09:07.850 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:07.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:09:07.851 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: Reconfiguring osd.1 (monmap changed)... 2023-06-10T21:09:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.1 on smithi064 2023-06-10T21:09:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: failed to encode map e78 with expected crc 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: failed to encode map e78 with expected crc 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/788965932' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: osdmap e78: 8 total, 8 up, 8 in 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:09:08.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:08.894 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:09:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[186706]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[186706]: Reconfiguring osd.2 (monmap changed)... 2023-06-10T21:09:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.2 on smithi064 2023-06-10T21:09:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[180297]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[180297]: Reconfiguring osd.2 (monmap changed)... 2023-06-10T21:09:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:08 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.2 on smithi064 2023-06-10T21:09:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:08 smithi152 ceph-mon[156819]: pgmap v111: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:08 smithi152 ceph-mon[156819]: Reconfiguring osd.2 (monmap changed)... 2023-06-10T21:09:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:08 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.2 on smithi064 2023-06-10T21:09:10.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.100 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: Reconfiguring osd.3 (monmap changed)... 2023-06-10T21:09:10.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:09:10.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:10.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:10.101 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.3 on smithi064 2023-06-10T21:09:10.101 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: Reconfiguring osd.3 (monmap changed)... 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:10.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:09 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.3 on smithi064 2023-06-10T21:09:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: Reconfiguring osd.3 (monmap changed)... 2023-06-10T21:09:10.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:09:10.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:10.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: pgmap v112: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:10.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:09 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.3 on smithi064 2023-06-10T21:09:11.135 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' created 2023-06-10T21:09:11.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-10T21:09:11.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.290 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:11.291 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-10T21:09:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi064.wuuunt", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:11.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-06-10T21:09:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: Reconfiguring mds.foo.smithi064.wuuunt (monmap changed)... 2023-06-10T21:09:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: Reconfiguring daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[186706]: osdmap e79: 8 total, 8 up, 8 in 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:12.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: Reconfiguring mds.foo.smithi064.wuuunt (monmap changed)... 2023-06-10T21:09:12.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: Reconfiguring daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T21:09:12.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-10T21:09:12.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:12 smithi064 ceph-mon[180297]: osdmap e79: 8 total, 8 up, 8 in 2023-06-10T21:09:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: pgmap v113: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: Reconfiguring mds.foo.smithi064.wuuunt (monmap changed)... 2023-06-10T21:09:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: Reconfiguring daemon mds.foo.smithi064.wuuunt on smithi064 2023-06-10T21:09:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: failed to encode map e79 with expected crc 2023-06-10T21:09:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3462406688' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-10T21:09:12.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:12 smithi152 ceph-mon[156819]: osdmap e79: 8 total, 8 up, 8 in 2023-06-10T21:09:12.783 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:09:12.973 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:12] "GET /metrics HTTP/1.1" 200 184134 "" "Prometheus/2.33.4" 2023-06-10T21:09:13.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: osdmap e80: 8 total, 8 up, 8 in 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: Reconfiguring mon.b (monmap changed)... 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:13.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[186706]: Reconfiguring daemon mon.b on smithi152 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: osdmap e80: 8 total, 8 up, 8 in 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.335 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: Reconfiguring mon.b (monmap changed)... 2023-06-10T21:09:13.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:13.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:13.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:13.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:13 smithi064 ceph-mon[180297]: Reconfiguring daemon mon.b on smithi152 2023-06-10T21:09:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: osdmap e80: 8 total, 8 up, 8 in 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: failed to encode map e80 with expected crc 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: Reconfiguring mon.b (monmap changed)... 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-10T21:09:13.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-10T21:09:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:13 smithi152 ceph-mon[156819]: Reconfiguring daemon mon.b on smithi152 2023-06-10T21:09:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[180297]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[180297]: failed to encode map e81 with expected crc 2023-06-10T21:09:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[180297]: osdmap e81: 8 total, 8 up, 8 in 2023-06-10T21:09:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[180297]: failed to encode map e81 with expected crc 2023-06-10T21:09:14.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[186706]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:14.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[186706]: failed to encode map e81 with expected crc 2023-06-10T21:09:14.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[186706]: osdmap e81: 8 total, 8 up, 8 in 2023-06-10T21:09:14.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:14 smithi064 ceph-mon[186706]: failed to encode map e81 with expected crc 2023-06-10T21:09:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:14 smithi152 ceph-mon[156819]: pgmap v116: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:14 smithi152 ceph-mon[156819]: failed to encode map e81 with expected crc 2023-06-10T21:09:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:14 smithi152 ceph-mon[156819]: osdmap e81: 8 total, 8 up, 8 in 2023-06-10T21:09:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:14 smithi152 ceph-mon[156819]: failed to encode map e81 with expected crc 2023-06-10T21:09:15.152 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-06-10T21:09:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:09:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: Reconfiguring mgr.x (monmap changed)... 2023-06-10T21:09:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:15.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:15.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[180297]: Reconfiguring daemon mgr.x on smithi152 2023-06-10T21:09:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:09:15.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: Reconfiguring mgr.x (monmap changed)... 2023-06-10T21:09:15.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:15.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:15.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:15.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:15 smithi064 ceph-mon[186706]: Reconfiguring daemon mgr.x on smithi152 2023-06-10T21:09:15.418 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:09:15.419 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.419 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:15.420 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: Reconfiguring mgr.x (monmap changed)... 2023-06-10T21:09:15.421 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-10T21:09:15.421 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-10T21:09:15.421 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:15.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:15 smithi152 ceph-mon[156819]: Reconfiguring daemon mgr.x on smithi152 2023-06-10T21:09:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:09:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: osdmap e82: 8 total, 8 up, 8 in 2023-06-10T21:09:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T21:09:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: osdmap e82: 8 total, 8 up, 8 in 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T21:09:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: pgmap v118: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: failed to encode map e82 with expected crc 2023-06-10T21:09:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/432732711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:09:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: osdmap e82: 8 total, 8 up, 8 in 2023-06-10T21:09:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-10T21:09:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:16.783 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-06-10T21:09:16.786 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:09:16.787 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:09:16.841 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:09:16.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:09:16.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_196031 2023-06-10T21:09:16.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:09:16.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:09:17.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:17 smithi064 ceph-mon[186706]: Reconfiguring osd.4 (monmap changed)... 2023-06-10T21:09:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:17 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.4 on smithi152 2023-06-10T21:09:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:17 smithi064 ceph-mon[180297]: Reconfiguring osd.4 (monmap changed)... 2023-06-10T21:09:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:17 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.4 on smithi152 2023-06-10T21:09:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:17 smithi152 ceph-mon[156819]: Reconfiguring osd.4 (monmap changed)... 2023-06-10T21:09:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:17 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.4 on smithi152 2023-06-10T21:09:17.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3520 3504 218.988 219 0.00254474 0.00455463 2023-06-10T21:09:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: Reconfiguring osd.5 (monmap changed)... 2023-06-10T21:09:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T21:09:18.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:18.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.5 on smithi152 2023-06-10T21:09:18.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:18.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:18.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: Reconfiguring osd.5 (monmap changed)... 2023-06-10T21:09:18.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T21:09:18.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:18.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.5 on smithi152 2023-06-10T21:09:18.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-10T21:09:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: Reconfiguring osd.5 (monmap changed)... 2023-06-10T21:09:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-10T21:09:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.5 on smithi152 2023-06-10T21:09:18.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:18.843 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6399 6383 199.45 179.938 0.00214469 0.00468943 2023-06-10T21:09:19.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8715 8699 181.213 144.75 0.00335287 0.00550955 2023-06-10T21:09:20.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 11636 11620 181.544 182.562 0.00773867 0.00550027 2023-06-10T21:09:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: pgmap v121: 89 pgs: 89 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:09:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: Reconfiguring osd.6 (monmap changed)... 2023-06-10T21:09:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T21:09:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.6 on smithi152 2023-06-10T21:09:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: pgmap v121: 89 pgs: 89 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:09:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: Reconfiguring osd.6 (monmap changed)... 2023-06-10T21:09:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T21:09:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:20.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:20 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.6 on smithi152 2023-06-10T21:09:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: pgmap v121: 89 pgs: 89 active+clean; 128 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:09:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: Reconfiguring osd.6 (monmap changed)... 2023-06-10T21:09:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-10T21:09:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:20 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.6 on smithi152 2023-06-10T21:09:21.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13495 13479 168.471 116.188 0.00339892 0.00592826 2023-06-10T21:09:22.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 16153 16137 168.078 166.125 0.00428516 0.00594469 2023-06-10T21:09:22.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[186706]: pgmap v122: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-06-10T21:09:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T21:09:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[180297]: pgmap v122: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-06-10T21:09:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T21:09:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:22.886 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:22] "GET /metrics HTTP/1.1" 200 189902 "" "Prometheus/2.33.4" 2023-06-10T21:09:22.935 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:22 smithi152 ceph-mon[156819]: pgmap v122: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s 2023-06-10T21:09:22.935 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.935 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:22.935 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-10T21:09:22.936 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:23.843 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 18264 18248 162.914 131.938 0.00425214 0.00613148 2023-06-10T21:09:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[186706]: Reconfiguring osd.7 (monmap changed)... 2023-06-10T21:09:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[186706]: Reconfiguring daemon osd.7 on smithi152 2023-06-10T21:09:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[180297]: Reconfiguring osd.7 (monmap changed)... 2023-06-10T21:09:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[180297]: Reconfiguring daemon osd.7 on smithi152 2023-06-10T21:09:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:23 smithi152 ceph-mon[156819]: Reconfiguring osd.7 (monmap changed)... 2023-06-10T21:09:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:23 smithi152 ceph-mon[156819]: Reconfiguring daemon osd.7 on smithi152 2023-06-10T21:09:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.842 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 20634 20618 161.064 148.125 0.0088245 0.00620243 2023-06-10T21:09:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[186706]: pgmap v123: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-06-10T21:09:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[180297]: pgmap v123: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-06-10T21:09:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:24.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:24 smithi152 ceph-mon[156819]: pgmap v123: 89 pgs: 89 active+clean; 417 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s wr, 673 op/s 2023-06-10T21:09:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi152.ciyxdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-10T21:09:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:25.843 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 22739 22723 157.785 131.562 0.00186915 0.00632662 2023-06-10T21:09:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:25 smithi064 ceph-mon[186706]: Reconfiguring mds.foo.smithi152.ciyxdy (monmap changed)... 2023-06-10T21:09:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:25 smithi064 ceph-mon[186706]: Reconfiguring daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T21:09:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:25 smithi064 ceph-mon[180297]: Reconfiguring mds.foo.smithi152.ciyxdy (monmap changed)... 2023-06-10T21:09:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:25 smithi064 ceph-mon[180297]: Reconfiguring daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T21:09:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:25 smithi152 ceph-mon[156819]: Reconfiguring mds.foo.smithi152.ciyxdy (monmap changed)... 2023-06-10T21:09:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:25 smithi152 ceph-mon[156819]: Reconfiguring daemon mds.foo.smithi152.ciyxdy on smithi152 2023-06-10T21:09:26.634 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-06-10T21:09:26.634 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.635 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.843 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 24483 24467 152.894 109 0.00618218 0.00653341 2023-06-10T21:09:26.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-06-10T21:09:26.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:09:26.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:09:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: pgmap v124: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-06-10T21:09:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:09:26.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:26.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:09:26.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:09:26.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:09:26.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:09:26.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:09:27.844 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 26645 26629 151.272 135.125 0.00400642 0.00660399 2023-06-10T21:09:28.844 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 28690 28674 149.316 127.812 0.00453003 0.0066899 2023-06-10T21:09:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:28 smithi064 ceph-mon[186706]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-10T21:09:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:28 smithi064 ceph-mon[180297]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-10T21:09:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:28 smithi152 ceph-mon[156819]: pgmap v125: 89 pgs: 89 active+clean; 1.2 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-10T21:09:29.844 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 30687 30671 147.431 124.812 0.00663452 0.0067776 2023-06-10T21:09:30.844 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 32437 32421 144.712 109.375 0.00441279 0.00690526 2023-06-10T21:09:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:30 smithi064 ceph-mon[180297]: pgmap v126: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-06-10T21:09:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:30 smithi064 ceph-mon[186706]: pgmap v126: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-06-10T21:09:30.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:30 smithi152 ceph-mon[156819]: pgmap v126: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2023-06-10T21:09:31.844 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 34301 34285 142.83 116.5 0.00620159 0.00699551 2023-06-10T21:09:32.845 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 36142 36126 141.084 115.062 0.011528 0.00708289 2023-06-10T21:09:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:32 smithi152 ceph-mon[156819]: pgmap v127: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:09:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:32 smithi064 ceph-mon[186706]: pgmap v127: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:09:33.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:32 smithi064 ceph-mon[180297]: pgmap v127: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:09:33.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:32] "GET /metrics HTTP/1.1" 200 189902 "" "Prometheus/2.33.4" 2023-06-10T21:09:33.845 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 37904 37888 139.262 110.125 0.0126676 0.0071751 2023-06-10T21:09:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:34.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:34.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:34.846 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 39624 39608 137.497 107.5 0.00787729 0.00726694 2023-06-10T21:09:34.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:34 smithi152 ceph-mon[156819]: pgmap v128: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-10T21:09:35.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:34 smithi064 ceph-mon[180297]: pgmap v128: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-10T21:09:35.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:34 smithi064 ceph-mon[186706]: pgmap v128: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-10T21:09:35.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 41332 41316 135.879 106.75 0.0108845 0.00735385 2023-06-10T21:09:36.846 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:09:36.844106+0000 min lat: 0.00111301 max lat: 0.37116 avg lat: 0.0074431 2023-06-10T21:09:36.846 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:09:36.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 42986 42970 134.251 103.375 0.00567144 0.0074431 2023-06-10T21:09:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:36 smithi152 ceph-mon[156819]: pgmap v129: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2023-06-10T21:09:37.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:36 smithi064 ceph-mon[186706]: pgmap v129: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2023-06-10T21:09:37.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:36 smithi064 ceph-mon[180297]: pgmap v129: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 165 MiB/s wr, 2.65k op/s 2023-06-10T21:09:37.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 44619 44603 132.715 102.062 0.00529051 0.00752918 2023-06-10T21:09:38.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 46219 46203 131.228 100 0.010095 0.00761451 2023-06-10T21:09:39.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:38 smithi064 ceph-mon[186706]: pgmap v130: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:09:39.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:38 smithi064 ceph-mon[180297]: pgmap v130: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:09:39.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:38 smithi152 ceph-mon[156819]: pgmap v130: 89 pgs: 89 active+clean; 2.3 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:09:39.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 47765 47749 129.723 96.625 0.0122264 0.00770325 2023-06-10T21:09:40.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 15 49269 49254 128.237 94.0625 0.00881708 0.00779352 2023-06-10T21:09:41.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:40 smithi064 ceph-mon[180297]: pgmap v131: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:09:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:40 smithi064 ceph-mon[186706]: pgmap v131: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:09:41.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:40 smithi152 ceph-mon[156819]: pgmap v131: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:09:41.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 50738 50722 126.777 91.75 0.00860915 0.0078833 2023-06-10T21:09:42.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 52195 52179 125.403 91.0625 0.010429 0.00796816 2023-06-10T21:09:43.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:42 smithi064 ceph-mon[186706]: pgmap v132: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-10T21:09:43.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:42] "GET /metrics HTTP/1.1" 200 189983 "" "Prometheus/2.33.4" 2023-06-10T21:09:43.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:42 smithi064 ceph-mon[180297]: pgmap v132: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-10T21:09:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:42 smithi152 ceph-mon[156819]: pgmap v132: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-10T21:09:43.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 53649 53633 124.125 90.875 0.00982931 0.0080515 2023-06-10T21:09:44.847 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 55063 55047 122.848 88.375 0.0141117 0.00813456 2023-06-10T21:09:45.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:44 smithi064 ceph-mon[186706]: pgmap v133: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-10T21:09:45.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:44 smithi064 ceph-mon[180297]: pgmap v133: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-10T21:09:45.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:44 smithi152 ceph-mon[156819]: pgmap v133: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-10T21:09:45.850 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 56468 56452 121.631 87.8125 0.0122376 0.0082169 2023-06-10T21:09:46.850 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 57841 57825 120.437 85.8125 0.0130512 0.00829805 2023-06-10T21:09:47.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:46 smithi064 ceph-mon[180297]: pgmap v134: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:09:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:46 smithi064 ceph-mon[186706]: pgmap v134: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:09:47.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:46 smithi152 ceph-mon[156819]: pgmap v134: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-10T21:09:47.851 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 59177 59161 119.241 83.5 0.00597667 0.00838109 2023-06-10T21:09:48.851 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 60486 60470 118.07 81.8125 0.00533999 0.00846438 2023-06-10T21:09:49.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:48 smithi064 ceph-mon[180297]: pgmap v135: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:09:49.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:49.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:48 smithi064 ceph-mon[186706]: pgmap v135: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:09:49.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:48 smithi152 ceph-mon[156819]: pgmap v135: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:09:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:09:49.851 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 15 61804 61789 116.99 82.4375 0.0120337 0.00854245 2023-06-10T21:09:50.853 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 63071 63055 115.873 79.125 0.0201125 0.00862546 2023-06-10T21:09:51.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:50 smithi064 ceph-mon[180297]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-10T21:09:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:50 smithi064 ceph-mon[186706]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-10T21:09:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:50 smithi152 ceph-mon[156819]: pgmap v136: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-10T21:09:51.853 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 64290 64274 114.738 76.1875 0.0116665 0.00871003 2023-06-10T21:09:52.853 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 65548 65532 113.735 78.625 0.0174301 0.00878737 2023-06-10T21:09:53.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:52 smithi064 ceph-mon[180297]: pgmap v137: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-06-10T21:09:53.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:09:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:09:52] "GET /metrics HTTP/1.1" 200 189999 "" "Prometheus/2.33.4" 2023-06-10T21:09:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:52 smithi064 ceph-mon[186706]: pgmap v137: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-06-10T21:09:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:52 smithi152 ceph-mon[156819]: pgmap v137: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-06-10T21:09:53.855 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 66780 66764 112.738 77 0.0188882 0.00886542 2023-06-10T21:09:54.856 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 67996 67980 111.766 76 0.0106417 0.00894288 2023-06-10T21:09:55.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:54 smithi064 ceph-mon[180297]: pgmap v138: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:09:55.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:54 smithi064 ceph-mon[186706]: pgmap v138: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:09:55.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:54 smithi152 ceph-mon[156819]: pgmap v138: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:09:55.857 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 69165 69149 110.772 73.0625 0.0129759 0.00902218 2023-06-10T21:09:56.857 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:09:56.855106+0000 min lat: 0.00111301 max lat: 0.37116 avg lat: 0.00910453 2023-06-10T21:09:56.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:09:56.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 15 70304 70289 109.784 71.25 0.0119697 0.00910453 2023-06-10T21:09:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:56 smithi064 ceph-mon[180297]: pgmap v139: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:09:57.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:56 smithi064 ceph-mon[186706]: pgmap v139: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:09:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:56 smithi152 ceph-mon[156819]: pgmap v139: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:09:57.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 71389 71373 108.759 67.75 0.0048171 0.00918937 2023-06-10T21:09:58.857 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 72546 72530 107.891 72.3125 0.00866653 0.0092633 2023-06-10T21:09:59.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:09:58 smithi064 ceph-mon[186706]: pgmap v140: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:09:59.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:09:58 smithi064 ceph-mon[180297]: pgmap v140: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:09:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:09:58 smithi152 ceph-mon[156819]: pgmap v140: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:09:59.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 73702 73686 107.06 72.25 0.0143783 0.00933524 2023-06-10T21:10:00.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 74840 74824 106.244 71.125 0.014376 0.00940726 2023-06-10T21:10:01.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:00 smithi064 ceph-mon[180297]: pgmap v141: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:10:01.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:00 smithi064 ceph-mon[180297]: overall HEALTH_OK 2023-06-10T21:10:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:00 smithi064 ceph-mon[186706]: pgmap v141: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:10:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:00 smithi064 ceph-mon[186706]: overall HEALTH_OK 2023-06-10T21:10:01.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:00 smithi152 ceph-mon[156819]: pgmap v141: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:10:01.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:00 smithi152 ceph-mon[156819]: overall HEALTH_OK 2023-06-10T21:10:01.858 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 75961 75945 105.44 70.0625 0.0164607 0.00947846 2023-06-10T21:10:02.862 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 77033 77017 104.597 67 0.00920997 0.00955531 2023-06-10T21:10:03.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:02 smithi064 ceph-mon[180297]: pgmap v142: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:10:03.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:02] "GET /metrics HTTP/1.1" 200 189999 "" "Prometheus/2.33.4" 2023-06-10T21:10:03.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:02 smithi064 ceph-mon[186706]: pgmap v142: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:10:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:02 smithi152 ceph-mon[156819]: pgmap v142: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-10T21:10:03.862 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 78078 78062 103.761 65.3125 0.00725425 0.00963317 2023-06-10T21:10:04.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade resume' 2023-06-10T21:10:04.134 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:04.134 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:04.863 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 79078 79062 102.901 62.5 0.0202561 0.0097129 2023-06-10T21:10:05.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:04 smithi064 ceph-mon[180297]: pgmap v143: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-06-10T21:10:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:04 smithi064 ceph-mon[186706]: pgmap v143: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-06-10T21:10:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:04 smithi152 ceph-mon[156819]: pgmap v143: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-06-10T21:10:05.862 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 79893 79877 101.841 50.9375 0.0123468 0.00981343 2023-06-10T21:10:06.864 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 80889 80873 101.046 62.25 0.0105961 0.00989142 2023-06-10T21:10:07.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:06 smithi064 ceph-mon[186706]: pgmap v144: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:10:07.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:06 smithi064 ceph-mon[180297]: pgmap v144: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:10:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:06 smithi152 ceph-mon[156819]: pgmap v144: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:10:07.273 INFO:teuthology.orchestra.run.smithi064.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:10:07.864 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 81795 81779 100.175 56.625 0.00752543 0.00997648 2023-06-10T21:10:08.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'echo wait for half of osds to upgrade' 2023-06-10T21:10:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:08 smithi152 ceph-mon[156819]: pgmap v145: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-06-10T21:10:08.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:08 smithi152 ceph-mon[156819]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:10:08.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:08 smithi152 ceph-mon[156819]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:10:08.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:08.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[180297]: pgmap v145: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-06-10T21:10:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[180297]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:10:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[180297]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:10:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[186706]: pgmap v145: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-06-10T21:10:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[186706]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:10:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[186706]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:10:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:08.864 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 82711 82695 99.3498 57.25 0.00923007 0.0100598 2023-06-10T21:10:09.866 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 83566 83550 98.4808 53.4375 0.0138375 0.010149 2023-06-10T21:10:10.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:10 smithi064 ceph-mon[180297]: pgmap v146: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:10:10.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:10 smithi064 ceph-mon[186706]: pgmap v146: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:10:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:10 smithi152 ceph-mon[156819]: pgmap v146: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:10:10.866 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 84410 84394 97.6342 52.75 0.0157339 0.0102366 2023-06-10T21:10:11.055 INFO:teuthology.orchestra.run.smithi064.stdout:wait for half of osds to upgrade 2023-06-10T21:10:11.866 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 85175 85159 96.7286 47.8125 0.0204286 0.0103331 2023-06-10T21:10:12.119 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[186706]: pgmap v147: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:10:12.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.120 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[180297]: pgmap v147: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:10:12.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-06-10T21:10:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:12 smithi152 ceph-mon[156819]: pgmap v147: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:10:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:12.876 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 86037 86021 95.9472 53.875 0.0165646 0.0104177 2023-06-10T21:10:13.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:12] "GET /metrics HTTP/1.1" 200 190002 "" "Prometheus/2.33.4" 2023-06-10T21:10:13.876 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 86919 86903 95.2313 55.125 0.0128462 0.0104963 2023-06-10T21:10:14.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:14 smithi064 ceph-mon[180297]: pgmap v148: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-06-10T21:10:14.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:14 smithi064 ceph-mon[186706]: pgmap v148: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-06-10T21:10:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:14 smithi152 ceph-mon[156819]: pgmap v148: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2023-06-10T21:10:14.876 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 87852 87836 94.595 58.3125 0.0146303 0.0105645 2023-06-10T21:10:15.877 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 88705 88689 93.8956 53.3125 0.0180367 0.0106446 2023-06-10T21:10:16.878 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:10:16.876366+0000 min lat: 0.00111301 max lat: 0.37116 avg lat: 0.0107172 2023-06-10T21:10:16.879 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:10:16.879 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 89599 89583 93.2586 55.875 0.0140466 0.0107172 2023-06-10T21:10:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: pgmap v149: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:10:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2204977222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: pgmap v149: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:10:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2204977222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:17.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: pgmap v149: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:10:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2204977222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.885 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 90477 90461 92.619 54.875 0.00832945 0.0107915 2023-06-10T21:10:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: Upgrade: Setting container_image for all mon 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-10T21:10:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: Upgrade: Setting container_image for all crash 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:17 smithi152 ceph-mon[156819]: Upgrade: osd.0 is safe to restart 2023-06-10T21:10:17.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:17.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: Upgrade: Setting container_image for all mon 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-10T21:10:17.959 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: Upgrade: Setting container_image for all crash 2023-06-10T21:10:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[180297]: Upgrade: osd.0 is safe to restart 2023-06-10T21:10:17.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:17.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: Upgrade: Setting container_image for all mon 2023-06-10T21:10:17.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.962 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:17.963 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: Upgrade: Setting container_image for all crash 2023-06-10T21:10:17.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:17.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-10T21:10:17.964 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:17 smithi064 ceph-mon[186706]: Upgrade: osd.0 is safe to restart 2023-06-10T21:10:18.889 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 91354 91338 92.0074 54.8125 0.0151946 0.0108638 2023-06-10T21:10:19.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[186706]: pgmap v150: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-06-10T21:10:19.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:19.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2529497314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[180297]: pgmap v150: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-06-10T21:10:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2529497314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:18 smithi152 ceph-mon[156819]: pgmap v150: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s 2023-06-10T21:10:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2529497314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:19.892 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 92136 92120 91.3154 48.875 0.0215184 0.0109463 2023-06-10T21:10:20.898 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 92953 92937 90.6827 51.0625 0.011734 0.0110222 2023-06-10T21:10:20.938 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: pgmap v151: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:10:20.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: Upgrade: Updating osd.0 2023-06-10T21:10:20.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:20.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:10:20.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:20.939 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[186706]: Deploying daemon osd.0 on smithi064 2023-06-10T21:10:20.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: pgmap v151: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:10:20.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: Upgrade: Updating osd.0 2023-06-10T21:10:20.945 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:20.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:10:20.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:20.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:20 smithi064 ceph-mon[180297]: Deploying daemon osd.0 on smithi064 2023-06-10T21:10:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: pgmap v151: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:10:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: Upgrade: Updating osd.0 2023-06-10T21:10:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-10T21:10:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:20 smithi152 ceph-mon[156819]: Deploying daemon osd.0 on smithi064 2023-06-10T21:10:21.895 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 93729 93713 90.0341 48.5 0.0167228 0.0111017 2023-06-10T21:10:22.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1548759610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:22.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1548759610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:22.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1548759610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:22.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:22 smithi064 ceph-mon[186706]: pgmap v152: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-10T21:10:22.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:22 smithi064 ceph-mon[180297]: pgmap v152: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-10T21:10:22.898 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 94609 94593 89.4998 55 0.0226691 0.0111674 2023-06-10T21:10:23.081 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:22] "GET /metrics HTTP/1.1" 200 190024 "" "Prometheus/2.33.4" 2023-06-10T21:10:23.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:22 smithi152 ceph-mon[156819]: pgmap v152: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-10T21:10:23.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 95283 95267 88.793 42.125 0.020749 0.0112574 2023-06-10T21:10:24.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3732369571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:24.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3732369571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:24.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3732369571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:24.658 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:24 smithi064 systemd[1]: Stopping Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:10:24.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 96121 96105 88.2578 52.375 0.0190978 0.0113242 2023-06-10T21:10:25.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:24 smithi064 ceph-mon[186706]: pgmap v153: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-06-10T21:10:25.161 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:25 smithi064 conmon[112223]: debug 2023-06-10T21:10:25.022+0000 7f288a04b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:10:25.161 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:25 smithi064 conmon[112223]: debug 2023-06-10T21:10:25.022+0000 7f288a04b700 -1 osd.0 82 *** Got signal Terminated *** 2023-06-10T21:10:25.162 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:25 smithi064 conmon[112223]: debug 2023-06-10T21:10:25.022+0000 7f288a04b700 -1 osd.0 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:10:25.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:24 smithi064 ceph-mon[180297]: pgmap v153: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-06-10T21:10:25.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:24 smithi152 ceph-mon[156819]: pgmap v153: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s 2023-06-10T21:10:25.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 97007 96991 87.7815 55.375 0.0195435 0.0113852 2023-06-10T21:10:26.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:25 smithi064 ceph-mon[180297]: osd.0 marked itself down and dead 2023-06-10T21:10:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:25 smithi064 ceph-mon[186706]: osd.0 marked itself down and dead 2023-06-10T21:10:26.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:25 smithi152 ceph-mon[156819]: osd.0 marked itself down and dead 2023-06-10T21:10:26.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 97026 97010 86.5454 1.1875 0.0116064 0.0114881 2023-06-10T21:10:27.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: pgmap v154: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-06-10T21:10:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:10:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: osdmap e83: 8 total, 7 up, 8 in 2023-06-10T21:10:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2111003944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:27.159 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:26 smithi064 bash[198922]: ef1eb4e15c36a8d93c80a506bf75e7312a6cc6ee2c09cf0d0db5a027dfed21de 2023-06-10T21:10:27.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: pgmap v154: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-06-10T21:10:27.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:10:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: osdmap e83: 8 total, 7 up, 8 in 2023-06-10T21:10:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2111003944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: pgmap v154: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-06-10T21:10:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:10:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: osdmap e83: 8 total, 7 up, 8 in 2023-06-10T21:10:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: failed to encode map e83 with expected crc 2023-06-10T21:10:27.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2111003944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:27.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 98108 98092 86.279 67.625 0.0125388 0.011586 2023-06-10T21:10:28.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: osdmap e84: 8 total, 7 up, 8 in 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[180297]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: osdmap e84: 8 total, 7 up, 8 in 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:27 smithi064 ceph-mon[186706]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: osdmap e84: 8 total, 7 up, 8 in 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:27 smithi152 ceph-mon[156819]: failed to encode map e84 with expected crc 2023-06-10T21:10:28.899 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 99081 99065 85.9255 60.8125 0.0744532 0.0116335 2023-06-10T21:10:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:28 smithi064 ceph-mon[186706]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:10:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3188202068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:28.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:28 smithi064 ceph-mon[180297]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:10:28.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3188202068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:29.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:28 smithi152 ceph-mon[156819]: pgmap v157: 89 pgs: 15 stale+active+clean, 74 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2023-06-10T21:10:29.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3188202068' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:29.901 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 99908 99892 85.4558 51.6875 0.0143394 0.0116973 2023-06-10T21:10:29.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:29 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0.service: Succeeded. 2023-06-10T21:10:29.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:29 smithi064 systemd[1]: Stopped Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:10:29.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:29 smithi064 systemd[1]: Starting Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:10:30.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:29 smithi152 ceph-mon[156819]: pgmap v158: 89 pgs: 8 active+undersized+degraded, 8 active+undersized, 64 active+clean, 9 stale+active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 31438/291705 objects degraded (10.777%) 2023-06-10T21:10:30.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:29 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 31438/291705 objects degraded (10.777%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:29 smithi064 ceph-mon[180297]: pgmap v158: 89 pgs: 8 active+undersized+degraded, 8 active+undersized, 64 active+clean, 9 stale+active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 31438/291705 objects degraded (10.777%) 2023-06-10T21:10:30.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:29 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 31438/291705 objects degraded (10.777%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:30.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:29 smithi064 ceph-mon[186706]: pgmap v158: 89 pgs: 8 active+undersized+degraded, 8 active+undersized, 64 active+clean, 9 stale+active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 31438/291705 objects degraded (10.777%) 2023-06-10T21:10:30.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:29 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 31438/291705 objects degraded (10.777%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:30.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 100930 100914 85.1627 63.875 0.0180717 0.0117371 2023-06-10T21:10:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/395558886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:31.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/395558886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:31.471 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-10T21:10:31.471 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-10T21:10:31.471 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-10T21:10:31.472 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-10T21:10:31.472 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2023-06-10T21:10:31.472 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-10T21:10:31.472 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:31 smithi064 bash[199569]: --> ceph-volume raw activate successful for osd ID: 0 2023-06-10T21:10:31.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/395558886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:31.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 101976 101960 84.899 65.375 0.0168954 0.0117738 2023-06-10T21:10:32.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:32 smithi152 ceph-mon[156819]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:32.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:32 smithi064 ceph-mon[180297]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:32 smithi064 ceph-mon[186706]: pgmap v159: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:32.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 103098 103082 84.7046 70.125 0.00892093 0.0118002 2023-06-10T21:10:33.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:32] "GET /metrics HTTP/1.1" 200 190024 "" "Prometheus/2.33.4" 2023-06-10T21:10:33.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 104083 104067 84.4042 61.5625 0.0131995 0.0118425 2023-06-10T21:10:33.910 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:33 smithi064 bash[200012]: 0499b076219a9cf46ac69734dce2f81c65e031c1c1a976aaa3574d7c15a14f86 2023-06-10T21:10:33.910 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:33 smithi064 ceph-osd[200099]: -- 172.21.15.64:0/3604204469 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5589d40a84e0 con 0x5589d407ac00 2023-06-10T21:10:33.911 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:33 smithi064 systemd[1]: Started Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:10:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3398439732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:34.413 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:33 smithi064 conmon[200073]: 2023-06-10T21:10:33.958+0000 7f5aa18ed640 -1 Falling back to public interface 2023-06-10T21:10:34.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:34.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3398439732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: pgmap v160: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 460 op/s; 49983/297783 objects degraded (16.785%) 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3398439732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:34.902 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 105117 105101 84.1507 64.625 0.0234452 0.0118781 2023-06-10T21:10:35.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:35 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 52521/313149 objects degraded (16.772%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:35.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:35 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 52521/313149 objects degraded (16.772%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:35 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 52521/313149 objects degraded (16.772%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:35.907 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 106094 106078 83.8527 61.0625 0.0093275 0.0119178 2023-06-10T21:10:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:36 smithi152 ceph-mon[156819]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:36.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:36 smithi064 ceph-mon[186706]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:36 smithi064 ceph-mon[180297]: pgmap v161: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:36.908 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:10:36.905987+0000 min lat: 0.00111301 max lat: 1.04868 avg lat: 0.0119725 2023-06-10T21:10:36.908 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:10:36.908 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 106973 106957 83.4909 54.9375 0.0237332 0.0119725 2023-06-10T21:10:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3975752252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:37.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3975752252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:37.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3975752252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:37.908 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 107938 107922 83.2049 60.3125 0.0154615 0.0120136 2023-06-10T21:10:38.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:38 smithi064 ceph-mon[180297]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:38.674 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:38 smithi064 ceph-mon[186706]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:38 smithi152 ceph-mon[156819]: pgmap v162: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 861 op/s; 52521/313149 objects degraded (16.772%) 2023-06-10T21:10:38.912 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 108806 108790 82.8479 54.25 0.0261008 0.0120655 2023-06-10T21:10:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3452667340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:39.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3452667340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:39 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:39 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:39.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3452667340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:39.912 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 109765 109749 82.5719 59.9375 0.00941106 0.0121063 2023-06-10T21:10:40.913 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 110684 110668 82.2729 57.4375 0.00770635 0.0121492 2023-06-10T21:10:41.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[186706]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 54002/319320 objects degraded (16.912%) 2023-06-10T21:10:41.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 54002/319320 objects degraded (16.912%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:41.147 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[180297]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 54002/319320 objects degraded (16.912%) 2023-06-10T21:10:41.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.148 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 54002/319320 objects degraded (16.912%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:41.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:40 smithi152 ceph-mon[156819]: pgmap v163: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 54002/319320 objects degraded (16.912%) 2023-06-10T21:10:41.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:41.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 54002/319320 objects degraded (16.912%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:41.912 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 111657 111641 82.0205 60.8125 0.00978636 0.0121874 2023-06-10T21:10:42.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2930145740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:42.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2930145740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2930145740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:42.913 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 112390 112374 81.5998 45.8125 0.0124269 0.0122503 2023-06-10T21:10:43.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:42 smithi064 ceph-mon[180297]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:43.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:42] "GET /metrics HTTP/1.1" 200 190153 "" "Prometheus/2.33.4" 2023-06-10T21:10:43.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:42 smithi064 ceph-mon[186706]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:42 smithi152 ceph-mon[156819]: pgmap v164: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:43.916 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 113222 113206 81.2573 52 0.0196243 0.0123002 2023-06-10T21:10:44.915 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 114090 114074 80.9506 54.25 0.0101787 0.0123488 2023-06-10T21:10:45.050 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:44 smithi064 ceph-mon[180297]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:45.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2385936520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:45.051 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:44 smithi064 ceph-mon[186706]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:45.052 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2385936520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:45.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:44 smithi152 ceph-mon[156819]: pgmap v165: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s; 54812/327177 objects degraded (16.753%) 2023-06-10T21:10:45.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2385936520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:45.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 114820 114804 80.5539 45.625 0.0170115 0.0124092 2023-06-10T21:10:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 54812/327177 objects degraded (16.753%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 54812/327177 objects degraded (16.753%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:46.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 54812/327177 objects degraded (16.753%), 15 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:46.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:46 smithi064 conmon[171866]: 2023-06-10T21:10:46.182+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:10:46.916 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0092 2023-06-10T21:10:46.916 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 115593 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 80.2647 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 35.6001 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 219 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 1.1875 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1284 2023-06-10T21:10:46.917 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 569.602 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3504 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 19 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0124551 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0164011 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 1.04868 2023-06-10T21:10:46.918 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00111301 2023-06-10T21:10:46.923 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:10:47.157 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:46 smithi064 conmon[200073]: 2023-06-10T21:10:46.858+0000 7f5aa18ed640 -1 osd.0 82 log_to_monitors true 2023-06-10T21:10:47.158 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:10:46 smithi064 conmon[200073]: 2023-06-10T21:10:46.994+0000 7f5a8fd53700 -1 osd.0 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:10:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:47.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1372769222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:47.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:47.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:47.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1372769222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:47.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:46 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:47.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: pgmap v166: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:47.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:10:47.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1372769222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:46 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:10:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:10:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:10:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: osdmap e85: 8 total, 7 up, 8 in 2023-06-10T21:10:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:48.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:48.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:48.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:10:48.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:10:48.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: osdmap e85: 8 total, 7 up, 8 in 2023-06-10T21:10:48.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:48.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:47 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:48.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:10:48.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:10:48.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:10:48.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: osdmap e85: 8 total, 7 up, 8 in 2023-06-10T21:10:48.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: failed to encode map e85 with expected crc 2023-06-10T21:10:48.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: from='osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:48.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:47 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:10:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149] boot 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: osdmap e86: 8 total, 8 up, 8 in 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:49.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:49.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:49.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:10:49.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149] boot 2023-06-10T21:10:49.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: osdmap e86: 8 total, 8 up, 8 in 2023-06-10T21:10:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:10:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:49.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 57030/340314 objects degraded (16.758%) 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: osd.0 [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149] boot 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: osdmap e86: 8 total, 8 up, 8 in 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:10:49.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: failed to encode map e86 with expected crc 2023-06-10T21:10:49.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:10:49.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:10:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: osdmap e87: 8 total, 8 up, 8 in 2023-06-10T21:10:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/699275353' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[186706]: pgmap v171: 89 pgs: 5 remapped+peering, 7 active+undersized+degraded, 15 active+undersized, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 58 MiB/s wr, 1.02k op/s; 21139/342105 objects degraded (6.179%) 2023-06-10T21:10:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: osdmap e87: 8 total, 8 up, 8 in 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/699275353' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:50.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:49 smithi064 ceph-mon[180297]: pgmap v171: 89 pgs: 5 remapped+peering, 7 active+undersized+degraded, 15 active+undersized, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 58 MiB/s wr, 1.02k op/s; 21139/342105 objects degraded (6.179%) 2023-06-10T21:10:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: osdmap e87: 8 total, 8 up, 8 in 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: failed to encode map e87 with expected crc 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/699275353' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:49 smithi152 ceph-mon[156819]: pgmap v171: 89 pgs: 5 remapped+peering, 7 active+undersized+degraded, 15 active+undersized, 62 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 58 MiB/s wr, 1.02k op/s; 21139/342105 objects degraded (6.179%) 2023-06-10T21:10:51.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: osdmap e88: 8 total, 8 up, 8 in 2023-06-10T21:10:51.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21139/342105 objects degraded (6.179%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: osdmap e88: 8 total, 8 up, 8 in 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21139/342105 objects degraded (6.179%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: osdmap e88: 8 total, 8 up, 8 in 2023-06-10T21:10:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: failed to encode map e88 with expected crc 2023-06-10T21:10:51.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21139/342105 objects degraded (6.179%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:52.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:51 smithi064 ceph-mon[186706]: pgmap v173: 89 pgs: 8 remapped+peering, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 B/s rd, 33 MiB/s wr, 797 op/s; 3/343437 objects degraded (0.001%) 2023-06-10T21:10:52.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2574100022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:52.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:51 smithi064 ceph-mon[180297]: pgmap v173: 89 pgs: 8 remapped+peering, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 B/s rd, 33 MiB/s wr, 797 op/s; 3/343437 objects degraded (0.001%) 2023-06-10T21:10:52.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2574100022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:52.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:51 smithi152 ceph-mon[156819]: pgmap v173: 89 pgs: 8 remapped+peering, 2 active+undersized+degraded, 9 active+undersized, 70 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 246 B/s rd, 33 MiB/s wr, 797 op/s; 3/343437 objects degraded (0.001%) 2023-06-10T21:10:52.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2574100022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:10:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:10:52] "GET /metrics HTTP/1.1" 200 207050 "" "Prometheus/2.33.4" 2023-06-10T21:10:53.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:53 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/343437 objects degraded (0.001%), 2 pgs degraded) 2023-06-10T21:10:53.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:53 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:10:53.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:53 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/343437 objects degraded (0.001%), 2 pgs degraded) 2023-06-10T21:10:53.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:53 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:10:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:53 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/343437 objects degraded (0.001%), 2 pgs degraded) 2023-06-10T21:10:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:53 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:10:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:54 smithi064 ceph-mon[186706]: pgmap v174: 89 pgs: 8 remapped+peering, 81 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 553 op/s 2023-06-10T21:10:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3819726264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:54.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:54 smithi064 ceph-mon[180297]: pgmap v174: 89 pgs: 8 remapped+peering, 81 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 553 op/s 2023-06-10T21:10:54.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3819726264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:54 smithi152 ceph-mon[156819]: pgmap v174: 89 pgs: 8 remapped+peering, 81 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 553 op/s 2023-06-10T21:10:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3819726264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:56 smithi152 ceph-mon[156819]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 143 B/s rd, 19 MiB/s wr, 878 op/s; 9492/334572 objects degraded (2.837%); 2.0 MiB/s, 33 objects/s recovering 2023-06-10T21:10:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:56 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 9492/334572 objects degraded (2.837%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:56.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:56 smithi064 ceph-mon[180297]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 143 B/s rd, 19 MiB/s wr, 878 op/s; 9492/334572 objects degraded (2.837%); 2.0 MiB/s, 33 objects/s recovering 2023-06-10T21:10:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:56 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 9492/334572 objects degraded (2.837%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:56 smithi064 ceph-mon[186706]: pgmap v175: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 143 B/s rd, 19 MiB/s wr, 878 op/s; 9492/334572 objects degraded (2.837%); 2.0 MiB/s, 33 objects/s recovering 2023-06-10T21:10:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:56 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 9492/334572 objects degraded (2.837%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:10:57.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1339678531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:57.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1339678531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1339678531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:10:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:10:58 smithi064 ceph-mon[180297]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 125 B/s rd, 17 MiB/s wr, 770 op/s; 9492/334572 objects degraded (2.837%); 1.7 MiB/s, 28 objects/s recovering 2023-06-10T21:10:58.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:10:58 smithi064 ceph-mon[186706]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 125 B/s rd, 17 MiB/s wr, 770 op/s; 9492/334572 objects degraded (2.837%); 1.7 MiB/s, 28 objects/s recovering 2023-06-10T21:10:58.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:10:58 smithi152 ceph-mon[156819]: pgmap v176: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 125 B/s rd, 17 MiB/s wr, 770 op/s; 9492/334572 objects degraded (2.837%); 1.7 MiB/s, 28 objects/s recovering 2023-06-10T21:11:00.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:00 smithi064 ceph-mon[180297]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 MiB/s wr, 649 op/s; 9160/328719 objects degraded (2.787%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:00.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1874825197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:00 smithi064 ceph-mon[186706]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 MiB/s wr, 649 op/s; 9160/328719 objects degraded (2.787%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:00.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1874825197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:00 smithi152 ceph-mon[156819]: pgmap v177: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 6.3 MiB/s wr, 649 op/s; 9160/328719 objects degraded (2.787%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1874825197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:01.663 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:01 smithi064 conmon[171866]: 2023-06-10T21:11:01.208+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:01.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:01.666 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:01.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:02.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[186706]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 9160/322986 objects degraded (2.836%); 3.1 MiB/s, 50 objects/s recovering 2023-06-10T21:11:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9160/322986 objects degraded (2.836%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:02.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2742412713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:02.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[180297]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 9160/322986 objects degraded (2.836%); 3.1 MiB/s, 50 objects/s recovering 2023-06-10T21:11:02.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:02.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:02.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9160/322986 objects degraded (2.836%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:02.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2742412713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:02 smithi152 ceph-mon[156819]: pgmap v178: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 9160/322986 objects degraded (2.836%); 3.1 MiB/s, 50 objects/s recovering 2023-06-10T21:11:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:02 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:02 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:02.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:02 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9160/322986 objects degraded (2.836%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:02.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2742412713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:02] "GET /metrics HTTP/1.1" 200 207050 "" "Prometheus/2.33.4" 2023-06-10T21:11:03.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:03.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:04.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:04 smithi064 ceph-mon[180297]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 9160/322986 objects degraded (2.836%); 2.9 MiB/s, 47 objects/s recovering 2023-06-10T21:11:04.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:04 smithi064 ceph-mon[186706]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 9160/322986 objects degraded (2.836%); 2.9 MiB/s, 47 objects/s recovering 2023-06-10T21:11:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:04 smithi152 ceph-mon[156819]: pgmap v179: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 9160/322986 objects degraded (2.836%); 2.9 MiB/s, 47 objects/s recovering 2023-06-10T21:11:05.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3569667487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:05.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3569667487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3569667487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:06.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:06 smithi064 ceph-mon[186706]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 8811/311574 objects degraded (2.828%); 4.7 MiB/s, 76 objects/s recovering 2023-06-10T21:11:06.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:06 smithi064 ceph-mon[180297]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 8811/311574 objects degraded (2.828%); 4.7 MiB/s, 76 objects/s recovering 2023-06-10T21:11:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:06 smithi152 ceph-mon[156819]: pgmap v180: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 8811/311574 objects degraded (2.828%); 4.7 MiB/s, 76 objects/s recovering 2023-06-10T21:11:07.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3812923888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:07.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3812923888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3812923888' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:08 smithi064 ceph-mon[186706]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 8811/311574 objects degraded (2.828%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:08 smithi064 ceph-mon[180297]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 8811/311574 objects degraded (2.828%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:08 smithi152 ceph-mon[156819]: pgmap v181: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 8811/311574 objects degraded (2.828%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:11:09.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[186706]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[186706]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[186706]: osdmap e89: 8 total, 8 up, 8 in 2023-06-10T21:11:09.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[186706]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3353651343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:09.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[180297]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[180297]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[180297]: osdmap e89: 8 total, 8 up, 8 in 2023-06-10T21:11:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[180297]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3353651343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:09 smithi152 ceph-mon[156819]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:09 smithi152 ceph-mon[156819]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:09 smithi152 ceph-mon[156819]: osdmap e89: 8 total, 8 up, 8 in 2023-06-10T21:11:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:09 smithi152 ceph-mon[156819]: failed to encode map e89 with expected crc 2023-06-10T21:11:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3353651343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: pgmap v183: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8721/306987 objects degraded (2.841%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8721/306987 objects degraded (2.841%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: osdmap e90: 8 total, 8 up, 8 in 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[180297]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: pgmap v183: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8721/306987 objects degraded (2.841%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:11:10.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8721/306987 objects degraded (2.841%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:10.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: osdmap e90: 8 total, 8 up, 8 in 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:10 smithi064 ceph-mon[186706]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: pgmap v183: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 8721/306987 objects degraded (2.841%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:11:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8721/306987 objects degraded (2.841%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: osdmap e90: 8 total, 8 up, 8 in 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:10.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:10 smithi152 ceph-mon[156819]: failed to encode map e90 with expected crc 2023-06-10T21:11:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:12 smithi064 ceph-mon[180297]: pgmap v185: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 8721/302241 objects degraded (2.885%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:11:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2909873462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:12 smithi064 ceph-mon[186706]: pgmap v185: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 8721/302241 objects degraded (2.885%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:11:12.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2909873462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:12.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:12 smithi152 ceph-mon[156819]: pgmap v185: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 8721/302241 objects degraded (2.885%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:11:12.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2909873462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:13.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:12] "GET /metrics HTTP/1.1" 200 207393 "" "Prometheus/2.33.4" 2023-06-10T21:11:14.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:14 smithi064 ceph-mon[186706]: pgmap v186: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8514/301119 objects degraded (2.827%); 1.8 MiB/s, 36 objects/s recovering 2023-06-10T21:11:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:14 smithi064 ceph-mon[180297]: pgmap v186: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8514/301119 objects degraded (2.827%); 1.8 MiB/s, 36 objects/s recovering 2023-06-10T21:11:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:14 smithi152 ceph-mon[156819]: pgmap v186: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 435 op/s; 8514/301119 objects degraded (2.827%); 1.8 MiB/s, 36 objects/s recovering 2023-06-10T21:11:15.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3012131119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:15 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8514/301119 objects degraded (2.827%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3012131119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:15 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8514/301119 objects degraded (2.827%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3012131119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:15 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8514/301119 objects degraded (2.827%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:16.658 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:16 smithi064 conmon[171866]: 2023-06-10T21:11:16.231+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:16 smithi064 ceph-mon[186706]: pgmap v187: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8425/290994 objects degraded (2.895%); 2.5 MiB/s, 47 objects/s recovering 2023-06-10T21:11:16.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:16 smithi064 ceph-mon[180297]: pgmap v187: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8425/290994 objects degraded (2.895%); 2.5 MiB/s, 47 objects/s recovering 2023-06-10T21:11:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:16.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:16 smithi152 ceph-mon[156819]: pgmap v187: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 8425/290994 objects degraded (2.895%); 2.5 MiB/s, 47 objects/s recovering 2023-06-10T21:11:16.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:17.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:17 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:17.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:17 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:17.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3190916043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3190916043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3190916043' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:18 smithi152 ceph-mon[156819]: pgmap v188: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8425/290319 objects degraded (2.902%); 2.3 MiB/s, 43 objects/s recovering 2023-06-10T21:11:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:18.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:18 smithi064 ceph-mon[180297]: pgmap v188: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8425/290319 objects degraded (2.902%); 2.3 MiB/s, 43 objects/s recovering 2023-06-10T21:11:18.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:18 smithi064 ceph-mon[186706]: pgmap v188: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8425/290319 objects degraded (2.902%); 2.3 MiB/s, 43 objects/s recovering 2023-06-10T21:11:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[180297]: pgmap v189: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8336/285387 objects degraded (2.921%); 2.4 MiB/s, 44 objects/s recovering 2023-06-10T21:11:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2118569220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8336/285387 objects degraded (2.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[186706]: pgmap v189: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8336/285387 objects degraded (2.921%); 2.4 MiB/s, 44 objects/s recovering 2023-06-10T21:11:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2118569220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:20 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8336/285387 objects degraded (2.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:20.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:20 smithi152 ceph-mon[156819]: pgmap v189: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8336/285387 objects degraded (2.921%); 2.4 MiB/s, 44 objects/s recovering 2023-06-10T21:11:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2118569220' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:20.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:20 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8336/285387 objects degraded (2.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:22 smithi064 ceph-mon[186706]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 8336/279948 objects degraded (2.978%); 2.0 MiB/s, 37 objects/s recovering 2023-06-10T21:11:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/270745709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:22 smithi064 ceph-mon[180297]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 8336/279948 objects degraded (2.978%); 2.0 MiB/s, 37 objects/s recovering 2023-06-10T21:11:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/270745709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:22.886 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:22] "GET /metrics HTTP/1.1" 200 207490 "" "Prometheus/2.33.4" 2023-06-10T21:11:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:22 smithi152 ceph-mon[156819]: pgmap v190: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 8336/279948 objects degraded (2.978%); 2.0 MiB/s, 37 objects/s recovering 2023-06-10T21:11:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/270745709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:24.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:24 smithi064 ceph-mon[180297]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 8336/279300 objects degraded (2.985%); 2.0 MiB/s, 36 objects/s recovering 2023-06-10T21:11:24.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:24 smithi064 ceph-mon[186706]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 8336/279300 objects degraded (2.985%); 2.0 MiB/s, 36 objects/s recovering 2023-06-10T21:11:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:24 smithi152 ceph-mon[156819]: pgmap v191: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 8336/279300 objects degraded (2.985%); 2.0 MiB/s, 36 objects/s recovering 2023-06-10T21:11:25.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2933395061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:25.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8336/279300 objects degraded (2.985%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2933395061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8336/279300 objects degraded (2.985%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:25.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2933395061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:25.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8336/279300 objects degraded (2.985%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:26 smithi064 ceph-mon[186706]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8250/268932 objects degraded (3.068%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:26.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:26 smithi064 ceph-mon[180297]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8250/268932 objects degraded (3.068%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:26 smithi152 ceph-mon[156819]: pgmap v192: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 8250/268932 objects degraded (3.068%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:27.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1944573275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:27.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1944573275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:27.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1944573275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:28.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:28 smithi064 ceph-mon[180297]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8250/268293 objects degraded (3.075%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:28.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:28 smithi064 ceph-mon[186706]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8250/268293 objects degraded (3.075%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:28 smithi152 ceph-mon[156819]: pgmap v193: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 8250/268293 objects degraded (3.075%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[180297]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8163/263592 objects degraded (3.097%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3181588511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8163/263592 objects degraded (3.097%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:30.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[186706]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8163/263592 objects degraded (3.097%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:30.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3181588511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8163/263592 objects degraded (3.097%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:30 smithi152 ceph-mon[156819]: pgmap v194: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8163/263592 objects degraded (3.097%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3181588511' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8163/263592 objects degraded (3.097%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:31.562 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:31 smithi064 conmon[171866]: 2023-06-10T21:11:31.258+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:31.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:31.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:31.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:32.881 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:32] "GET /metrics HTTP/1.1" 200 207490 "" "Prometheus/2.33.4" 2023-06-10T21:11:32.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[180297]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8163/258051 objects degraded (3.163%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:32.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:32.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:32.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2295279117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[186706]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8163/258051 objects degraded (3.163%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:32.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2295279117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:32 smithi152 ceph-mon[156819]: pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8163/258051 objects degraded (3.163%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:32 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:32 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:32.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2295279117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:33.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:33.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:33.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:34.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:34 smithi064 ceph-mon[186706]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 8163/257361 objects degraded (3.172%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:34 smithi064 ceph-mon[180297]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 8163/257361 objects degraded (3.172%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:34 smithi152 ceph-mon[156819]: pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 8163/257361 objects degraded (3.172%); 922 KiB/s, 14 objects/s recovering 2023-06-10T21:11:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3874688080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:35 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8163/257361 objects degraded (3.172%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3874688080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:35 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8163/257361 objects degraded (3.172%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3874688080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:35 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8163/257361 objects degraded (3.172%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:36.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:36 smithi064 ceph-mon[186706]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8076/247371 objects degraded (3.265%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:36 smithi064 ceph-mon[180297]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8076/247371 objects degraded (3.265%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:36.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:36 smithi152 ceph-mon[156819]: pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8076/247371 objects degraded (3.265%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:37.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2178644990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2178644990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:37.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2178644990' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:38.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:38 smithi064 ceph-mon[186706]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8076/246591 objects degraded (3.275%); 928 KiB/s, 14 objects/s recovering 2023-06-10T21:11:38.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:38 smithi064 ceph-mon[180297]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8076/246591 objects degraded (3.275%); 928 KiB/s, 14 objects/s recovering 2023-06-10T21:11:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:38 smithi152 ceph-mon[156819]: pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 8076/246591 objects degraded (3.275%); 928 KiB/s, 14 objects/s recovering 2023-06-10T21:11:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:40 smithi152 ceph-mon[156819]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 7988/241929 objects degraded (3.302%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2774822820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:40.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7988/241929 objects degraded (3.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:41.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[180297]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 7988/241929 objects degraded (3.302%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:41.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2774822820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:41.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7988/241929 objects degraded (3.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[186706]: pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 7988/241929 objects degraded (3.302%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2774822820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:41.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7988/241929 objects degraded (3.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:42 smithi152 ceph-mon[156819]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 7988/236646 objects degraded (3.376%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:43.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:42 smithi064 ceph-mon[180297]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 7988/236646 objects degraded (3.376%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:43.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:42] "GET /metrics HTTP/1.1" 200 207507 "" "Prometheus/2.33.4" 2023-06-10T21:11:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:42 smithi064 ceph-mon[186706]: pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 7988/236646 objects degraded (3.376%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:44.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1920983087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:44.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1920983087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:44.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1920983087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:45.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:44 smithi064 ceph-mon[180297]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7988/235965 objects degraded (3.385%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:45.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:44 smithi064 ceph-mon[186706]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7988/235965 objects degraded (3.385%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:45.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:44 smithi152 ceph-mon[156819]: pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 7988/235965 objects degraded (3.385%); 933 KiB/s, 14 objects/s recovering 2023-06-10T21:11:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7988/235965 objects degraded (3.385%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4194197949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:46.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7988/235965 objects degraded (3.385%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:46.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4194197949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:46.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7988/235965 objects degraded (3.385%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:46.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4194197949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:46 smithi064 ceph-mon[186706]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7900/226029 objects degraded (3.495%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:47.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:47.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:46 smithi064 ceph-mon[180297]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7900/226029 objects degraded (3.495%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:47.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:46 smithi152 ceph-mon[156819]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 7900/226029 objects degraded (3.495%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:49.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[186706]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 7900/225417 objects degraded (3.505%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:49.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3096719635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:49.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:49.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[180297]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 7900/225417 objects degraded (3.505%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:49.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3096719635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:49.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:48 smithi152 ceph-mon[156819]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 7900/225417 objects degraded (3.505%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3096719635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:11:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[180297]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7812/220899 objects degraded (3.536%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7812/220899 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2369561328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[186706]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7812/220899 objects degraded (3.536%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7812/220899 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2369561328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:50 smithi152 ceph-mon[156819]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7812/220899 objects degraded (3.536%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:11:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7812/220899 objects degraded (3.536%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:11:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2369561328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 7812/216048 objects degraded (3.616%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:11:52] "GET /metrics HTTP/1.1" 200 207440 "" "Prometheus/2.33.4" 2023-06-10T21:11:53.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:11:53 smithi064 conmon[171866]: 2023-06-10T21:11:53.084+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 7812/216048 objects degraded (3.616%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:11:53.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 7812/216048 objects degraded (3.616%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:11:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:11:53.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:11:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1874441897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:54.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1874441897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:54.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:54.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:11:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1874441897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:11:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:55.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:54 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:55.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:54 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:55.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:54 smithi152 ceph-mon[156819]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 7812/215610 objects degraded (3.623%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:55.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:55.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:55.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[180297]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 7812/215610 objects degraded (3.623%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:11:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:11:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:54 smithi064 ceph-mon[186706]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 7812/215610 objects degraded (3.623%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:11:56.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7812/215610 objects degraded (3.623%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:11:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3081325824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:56.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7812/215610 objects degraded (3.623%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:11:56.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3081325824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7812/215610 objects degraded (3.623%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:11:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3081325824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:56 smithi064 ceph-mon[186706]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7723/206370 objects degraded (3.742%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:11:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:56 smithi064 ceph-mon[180297]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7723/206370 objects degraded (3.742%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:11:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:56 smithi152 ceph-mon[156819]: pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7723/206370 objects degraded (3.742%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:11:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:58 smithi064 ceph-mon[186706]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 7723/205692 objects degraded (3.755%); 943 KiB/s, 14 objects/s recovering 2023-06-10T21:11:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:11:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3298067965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:58 smithi064 ceph-mon[180297]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 7723/205692 objects degraded (3.755%); 943 KiB/s, 14 objects/s recovering 2023-06-10T21:11:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:11:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3298067965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:11:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:58 smithi152 ceph-mon[156819]: pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 7723/205692 objects degraded (3.755%); 943 KiB/s, 14 objects/s recovering 2023-06-10T21:11:59.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:11:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3298067965' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:01.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[180297]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7636/201132 objects degraded (3.797%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:12:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7636/201132 objects degraded (3.797%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2152233762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:01.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[186706]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7636/201132 objects degraded (3.797%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:12:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7636/201132 objects degraded (3.797%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2152233762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:00 smithi152 ceph-mon[156819]: pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 7636/201132 objects degraded (3.797%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:12:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7636/201132 objects degraded (3.797%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2152233762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:02 smithi064 ceph-mon[180297]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7636/195897 objects degraded (3.898%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:02.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:02 smithi064 ceph-mon[186706]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7636/195897 objects degraded (3.898%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:02 smithi152 ceph-mon[156819]: pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7636/195897 objects degraded (3.898%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:03.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:03.166 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:02] "GET /metrics HTTP/1.1" 200 207440 "" "Prometheus/2.33.4" 2023-06-10T21:12:03.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:04.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:04 smithi064 ceph-mon[186706]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7636/195228 objects degraded (3.911%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:04.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1448610698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:04.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:04 smithi064 ceph-mon[180297]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7636/195228 objects degraded (3.911%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:04.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1448610698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:04.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:04 smithi152 ceph-mon[156819]: pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 7636/195228 objects degraded (3.911%); 938 KiB/s, 14 objects/s recovering 2023-06-10T21:12:04.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1448610698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7636/195228 objects degraded (3.911%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:05.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7636/195228 objects degraded (3.911%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7636/195228 objects degraded (3.911%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:06.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:06 smithi064 ceph-mon[180297]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 7545/185304 objects degraded (4.072%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:06.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1029109381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:06.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:06 smithi064 ceph-mon[186706]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 7545/185304 objects degraded (4.072%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:06.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1029109381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:06 smithi152 ceph-mon[156819]: pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 7545/185304 objects degraded (4.072%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1029109381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:08.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:08 smithi064 ceph-mon[180297]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7545/184623 objects degraded (4.087%); 949 KiB/s, 14 objects/s recovering 2023-06-10T21:12:08.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:08.411 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:08 smithi064 conmon[171866]: 2023-06-10T21:12:08.106+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:08.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:08 smithi064 ceph-mon[186706]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7545/184623 objects degraded (4.087%); 949 KiB/s, 14 objects/s recovering 2023-06-10T21:12:08.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:08.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:08 smithi152 ceph-mon[156819]: pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7545/184623 objects degraded (4.087%); 949 KiB/s, 14 objects/s recovering 2023-06-10T21:12:08.707 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:09.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:09.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/623211630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:09.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:09.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:09.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/623211630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:09 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:09 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/623211630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:10 smithi064 ceph-mon[180297]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 7457/179958 objects degraded (4.144%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7457/179958 objects degraded (4.144%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:10 smithi064 ceph-mon[186706]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 7457/179958 objects degraded (4.144%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7457/179958 objects degraded (4.144%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:10.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:10 smithi152 ceph-mon[156819]: pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 7457/179958 objects degraded (4.144%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:12:10.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7457/179958 objects degraded (4.144%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2486014302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2486014302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:11.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2486014302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:12 smithi064 ceph-mon[186706]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 7457/174762 objects degraded (4.267%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:12 smithi064 ceph-mon[180297]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 7457/174762 objects degraded (4.267%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:12 smithi152 ceph-mon[156819]: pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 7457/174762 objects degraded (4.267%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:13.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:12] "GET /metrics HTTP/1.1" 200 207504 "" "Prometheus/2.33.4" 2023-06-10T21:12:14.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:14 smithi064 ceph-mon[186706]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7457/174084 objects degraded (4.284%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1788825439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:14.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:14 smithi064 ceph-mon[180297]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7457/174084 objects degraded (4.284%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:14.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1788825439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:14 smithi152 ceph-mon[156819]: pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7457/174084 objects degraded (4.284%); 954 KiB/s, 14 objects/s recovering 2023-06-10T21:12:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1788825439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:15.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:15 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7457/174084 objects degraded (4.284%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:15 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7457/174084 objects degraded (4.284%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:15.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:15 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7457/174084 objects degraded (4.284%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:16.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:16 smithi152 ceph-mon[156819]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 7388/164382 objects degraded (4.494%); 1.3 MiB/s, 22 objects/s recovering 2023-06-10T21:12:16.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2400063584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:16 smithi064 ceph-mon[186706]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 7388/164382 objects degraded (4.494%); 1.3 MiB/s, 22 objects/s recovering 2023-06-10T21:12:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2400063584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:16 smithi064 ceph-mon[180297]: pgmap v217: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 7388/164382 objects degraded (4.494%); 1.3 MiB/s, 22 objects/s recovering 2023-06-10T21:12:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2400063584' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[180297]: osdmap e91: 8 total, 8 up, 8 in 2023-06-10T21:12:17.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[180297]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[180297]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[180297]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[186706]: osdmap e91: 8 total, 8 up, 8 in 2023-06-10T21:12:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[186706]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[186706]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:17 smithi064 ceph-mon[186706]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:17 smithi152 ceph-mon[156819]: osdmap e91: 8 total, 8 up, 8 in 2023-06-10T21:12:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:17 smithi152 ceph-mon[156819]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:17 smithi152 ceph-mon[156819]: failed to encode map e91 with expected crc 2023-06-10T21:12:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:17 smithi152 ceph-mon[156819]: failed to encode map e91 with expected crc 2023-06-10T21:12:18.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: pgmap v219: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7388/163500 objects degraded (4.519%); 1004 KiB/s, 22 objects/s recovering 2023-06-10T21:12:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: osdmap e92: 8 total, 8 up, 8 in 2023-06-10T21:12:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: pgmap v219: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7388/163500 objects degraded (4.519%); 1004 KiB/s, 22 objects/s recovering 2023-06-10T21:12:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: osdmap e92: 8 total, 8 up, 8 in 2023-06-10T21:12:18.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: pgmap v219: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 7388/163500 objects degraded (4.519%); 1004 KiB/s, 22 objects/s recovering 2023-06-10T21:12:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: osdmap e92: 8 total, 8 up, 8 in 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: failed to encode map e92 with expected crc 2023-06-10T21:12:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:19.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/443766500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:19.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/443766500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/443766500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:20 smithi064 ceph-mon[180297]: pgmap v221: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7047/160221 objects degraded (4.398%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:20 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7047/160221 objects degraded (4.398%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:20 smithi064 ceph-mon[186706]: pgmap v221: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7047/160221 objects degraded (4.398%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:20 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7047/160221 objects degraded (4.398%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:20.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:20 smithi152 ceph-mon[156819]: pgmap v221: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 7047/160221 objects degraded (4.398%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:20.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:20 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7047/160221 objects degraded (4.398%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:21.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1630617058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:21.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1630617058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:21.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1630617058' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:22.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:22 smithi064 ceph-mon[180297]: pgmap v222: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 7047/155793 objects degraded (4.523%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:22.894 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:22] "GET /metrics HTTP/1.1" 200 207510 "" "Prometheus/2.33.4" 2023-06-10T21:12:22.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:22 smithi064 ceph-mon[186706]: pgmap v222: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 7047/155793 objects degraded (4.523%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:22 smithi152 ceph-mon[156819]: pgmap v222: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 7047/155793 objects degraded (4.523%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:12:23.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:23 smithi064 conmon[171866]: 2023-06-10T21:12:23.131+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:23.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:23.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[180297]: pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 7047/154611 objects degraded (4.558%); 2.1 MiB/s, 38 objects/s recovering 2023-06-10T21:12:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2649070769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[186706]: pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 7047/154611 objects degraded (4.558%); 2.1 MiB/s, 38 objects/s recovering 2023-06-10T21:12:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2649070769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:24 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:24 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:12:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:24 smithi152 ceph-mon[156819]: pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 406 op/s; 7047/154611 objects degraded (4.558%); 2.1 MiB/s, 38 objects/s recovering 2023-06-10T21:12:24.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2649070769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:25.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7047/154611 objects degraded (4.558%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:25.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7047/154611 objects degraded (4.558%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7047/154611 objects degraded (4.558%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:26.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:26 smithi064 ceph-mon[180297]: pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6770/145707 objects degraded (4.646%); 3.9 MiB/s, 66 objects/s recovering 2023-06-10T21:12:26.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:26 smithi064 ceph-mon[186706]: pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6770/145707 objects degraded (4.646%); 3.9 MiB/s, 66 objects/s recovering 2023-06-10T21:12:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:26 smithi152 ceph-mon[156819]: pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 6770/145707 objects degraded (4.646%); 3.9 MiB/s, 66 objects/s recovering 2023-06-10T21:12:27.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2521677071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:27.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2521677071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:27.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2521677071' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:28.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:28 smithi064 ceph-mon[180297]: pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 6770/144408 objects degraded (4.688%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:12:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:28 smithi064 ceph-mon[186706]: pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 6770/144408 objects degraded (4.688%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:12:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:28 smithi152 ceph-mon[156819]: pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 6770/144408 objects degraded (4.688%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:12:29.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3955593857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:29.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3955593857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3955593857' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:30.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:30 smithi064 ceph-mon[186706]: pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6476/140559 objects degraded (4.607%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:12:30.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 6476/140559 objects degraded (4.607%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:30.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:30 smithi064 ceph-mon[180297]: pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6476/140559 objects degraded (4.607%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:12:30.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 6476/140559 objects degraded (4.607%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:30 smithi152 ceph-mon[156819]: pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 6476/140559 objects degraded (4.607%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:12:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 6476/140559 objects degraded (4.607%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:32.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:32] "GET /metrics HTTP/1.1" 200 207510 "" "Prometheus/2.33.4" 2023-06-10T21:12:32.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:32 smithi064 ceph-mon[186706]: pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6476/135513 objects degraded (4.779%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:32.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4158722154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:32.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:32 smithi064 ceph-mon[180297]: pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6476/135513 objects degraded (4.779%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:32.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4158722154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:32 smithi152 ceph-mon[156819]: pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 6476/135513 objects degraded (4.779%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4158722154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:34.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:34.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:34 smithi152 ceph-mon[156819]: pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6476/134148 objects degraded (4.828%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3374868483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:34 smithi064 ceph-mon[180297]: pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6476/134148 objects degraded (4.828%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:35.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3374868483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:35.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:34 smithi064 ceph-mon[186706]: pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6476/134148 objects degraded (4.828%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:12:35.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3374868483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:36.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:35 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 6476/134148 objects degraded (4.828%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:36.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:35 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 6476/134148 objects degraded (4.828%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:36.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:35 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 6476/134148 objects degraded (4.828%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:37.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:36 smithi064 ceph-mon[186706]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 6181/125268 objects degraded (4.934%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:12:37.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:36 smithi064 ceph-mon[180297]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 6181/125268 objects degraded (4.934%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:12:37.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:36 smithi152 ceph-mon[156819]: pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 6181/125268 objects degraded (4.934%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:12:38.156 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2894839046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:38.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2894839046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:38.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2894839046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:38.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:38 smithi064 conmon[171866]: 2023-06-10T21:12:38.152+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:12:39.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[186706]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6181/124056 objects degraded (4.982%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:12:39.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[186706]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[186706]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[186706]: osdmap e93: 8 total, 8 up, 8 in 2023-06-10T21:12:39.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:39.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[180297]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6181/124056 objects degraded (4.982%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:12:39.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[180297]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[180297]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[180297]: osdmap e93: 8 total, 8 up, 8 in 2023-06-10T21:12:39.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:38 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:39.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:38 smithi152 ceph-mon[156819]: pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 6181/124056 objects degraded (4.982%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:12:39.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:38 smithi152 ceph-mon[156819]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:38 smithi152 ceph-mon[156819]: failed to encode map e93 with expected crc 2023-06-10T21:12:39.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:38 smithi152 ceph-mon[156819]: osdmap e93: 8 total, 8 up, 8 in 2023-06-10T21:12:39.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:38 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:40.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:40.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: osdmap e94: 8 total, 8 up, 8 in 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2437235482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:40.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: osdmap e94: 8 total, 8 up, 8 in 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2437235482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:40.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:12:40.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:12:40.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: osdmap e94: 8 total, 8 up, 8 in 2023-06-10T21:12:40.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: failed to encode map e94 with expected crc 2023-06-10T21:12:40.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2437235482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:41.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:40 smithi064 ceph-mon[180297]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6081/120873 objects degraded (5.031%); 3.1 MiB/s, 58 objects/s recovering 2023-06-10T21:12:41.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 6081/120873 objects degraded (5.031%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:40 smithi064 ceph-mon[186706]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6081/120873 objects degraded (5.031%); 3.1 MiB/s, 58 objects/s recovering 2023-06-10T21:12:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 6081/120873 objects degraded (5.031%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:41.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:40 smithi152 ceph-mon[156819]: pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 6081/120873 objects degraded (5.031%); 3.1 MiB/s, 58 objects/s recovering 2023-06-10T21:12:41.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 6081/120873 objects degraded (5.031%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:42] "GET /metrics HTTP/1.1" 200 207538 "" "Prometheus/2.33.4" 2023-06-10T21:12:43.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:42 smithi064 ceph-mon[180297]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5745/116655 objects degraded (4.925%); 5.1 MiB/s, 90 objects/s recovering 2023-06-10T21:12:43.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:42 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1021157613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:42 smithi064 ceph-mon[186706]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5745/116655 objects degraded (4.925%); 5.1 MiB/s, 90 objects/s recovering 2023-06-10T21:12:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:42 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1021157613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:42 smithi152 ceph-mon[156819]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 5745/116655 objects degraded (4.925%); 5.1 MiB/s, 90 objects/s recovering 2023-06-10T21:12:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:42 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1021157613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:45.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:44 smithi064 ceph-mon[186706]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5745/115650 objects degraded (4.968%); 2.8 MiB/s, 53 objects/s recovering 2023-06-10T21:12:45.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2143441048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:45.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:44 smithi064 ceph-mon[180297]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5745/115650 objects degraded (4.968%); 2.8 MiB/s, 53 objects/s recovering 2023-06-10T21:12:45.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2143441048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:44 smithi152 ceph-mon[156819]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 400 op/s; 5745/115650 objects degraded (4.968%); 2.8 MiB/s, 53 objects/s recovering 2023-06-10T21:12:45.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2143441048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5745/115650 objects degraded (4.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:46.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5745/115650 objects degraded (4.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:46.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5745/115650 objects degraded (4.968%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:47.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:46 smithi064 ceph-mon[186706]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5487/107025 objects degraded (5.127%); 4.8 MiB/s, 86 objects/s recovering 2023-06-10T21:12:47.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:46 smithi064 ceph-mon[180297]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5487/107025 objects degraded (5.127%); 4.8 MiB/s, 86 objects/s recovering 2023-06-10T21:12:47.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:46 smithi152 ceph-mon[156819]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5487/107025 objects degraded (5.127%); 4.8 MiB/s, 86 objects/s recovering 2023-06-10T21:12:48.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1337212737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:48.171 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1337212737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:48.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1337212737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:48 smithi064 ceph-mon[180297]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5487/105672 objects degraded (5.192%); 4.1 MiB/s, 73 objects/s recovering 2023-06-10T21:12:49.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:49.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:48 smithi064 ceph-mon[186706]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5487/105672 objects degraded (5.192%); 4.1 MiB/s, 73 objects/s recovering 2023-06-10T21:12:49.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:48 smithi152 ceph-mon[156819]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 5487/105672 objects degraded (5.192%); 4.1 MiB/s, 73 objects/s recovering 2023-06-10T21:12:49.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:12:50.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3277579561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:50.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3277579561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:50.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3277579561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:50 smithi064 ceph-mon[186706]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 5487/102045 objects degraded (5.377%); 3.7 MiB/s, 66 objects/s recovering 2023-06-10T21:12:51.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5487/102045 objects degraded (5.377%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:51.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:50 smithi064 ceph-mon[180297]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 5487/102045 objects degraded (5.377%); 3.7 MiB/s, 66 objects/s recovering 2023-06-10T21:12:51.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5487/102045 objects degraded (5.377%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:50 smithi152 ceph-mon[156819]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 702 op/s; 5487/102045 objects degraded (5.377%); 3.7 MiB/s, 66 objects/s recovering 2023-06-10T21:12:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5487/102045 objects degraded (5.377%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:52 smithi064 ceph-mon[180297]: pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5211/97011 objects degraded (5.372%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:12:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/626295703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:53.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:12:52] "GET /metrics HTTP/1.1" 200 207537 "" "Prometheus/2.33.4" 2023-06-10T21:12:53.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:52 smithi064 ceph-mon[186706]: pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5211/97011 objects degraded (5.372%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:12:53.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/626295703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:52 smithi152 ceph-mon[156819]: pgmap v239: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 5211/97011 objects degraded (5.372%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:12:53.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/626295703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:54.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:12:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:12:54.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:12:55.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:54 smithi152 ceph-mon[156819]: pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 5211/95742 objects degraded (5.443%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:12:55.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:54 smithi064 ceph-mon[180297]: pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 5211/95742 objects degraded (5.443%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:12:55.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:54 smithi064 ceph-mon[186706]: pgmap v240: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 5211/95742 objects degraded (5.443%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:12:56.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1691506161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:56.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5211/95742 objects degraded (5.443%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:56.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1691506161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:56.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5211/95742 objects degraded (5.443%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1691506161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:56.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5211/95742 objects degraded (5.443%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:12:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:56 smithi064 ceph-mon[186706]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4936/87117 objects degraded (5.666%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:12:57.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:56 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:56 smithi064 ceph-mon[180297]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4936/87117 objects degraded (5.666%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:12:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:56 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:56 smithi152 ceph-mon[156819]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 4936/87117 objects degraded (5.666%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:12:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:56 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:58.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:58.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: osdmap e95: 8 total, 8 up, 8 in 2023-06-10T21:12:58.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/859777022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:58.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: osdmap e95: 8 total, 8 up, 8 in 2023-06-10T21:12:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/859777022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: osdmap e95: 8 total, 8 up, 8 in 2023-06-10T21:12:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: failed to encode map e95 with expected crc 2023-06-10T21:12:58.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/859777022' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:12:59.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 4936/87117 objects degraded (5.666%); 3.4 MiB/s, 55 objects/s recovering 2023-06-10T21:12:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: osdmap e96: 8 total, 8 up, 8 in 2023-06-10T21:12:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 4936/87117 objects degraded (5.666%); 3.4 MiB/s, 55 objects/s recovering 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: osdmap e96: 8 total, 8 up, 8 in 2023-06-10T21:12:59.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: pgmap v243: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 4936/87117 objects degraded (5.666%); 3.4 MiB/s, 55 objects/s recovering 2023-06-10T21:12:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: osdmap e96: 8 total, 8 up, 8 in 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: failed to encode map e96 with expected crc 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:12:59.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:12:59 smithi064 conmon[171866]: 2023-06-10T21:12:59.427+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:00.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3982885486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:00.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:13:00.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:13:00.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:00.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:13:00.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:00.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:12:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3982885486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:13:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:13:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:12:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3982885486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:01.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[180297]: pgmap v245: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 4879/82698 objects degraded (5.900%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:13:01.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:01.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:13:01.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4879/82698 objects degraded (5.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:01.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[186706]: pgmap v245: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 4879/82698 objects degraded (5.900%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:13:01.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:01.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:13:01.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4879/82698 objects degraded (5.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:01.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:00 smithi152 ceph-mon[156819]: pgmap v245: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 4879/82698 objects degraded (5.900%); 2.6 MiB/s, 50 objects/s recovering 2023-06-10T21:13:01.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:00 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:01.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:00 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-10T21:13:01.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4879/82698 objects degraded (5.900%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:03.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:02 smithi064 ceph-mon[180297]: pgmap v246: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4774/79299 objects degraded (6.020%); 3.0 MiB/s, 56 objects/s recovering 2023-06-10T21:13:03.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2375481889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:02] "GET /metrics HTTP/1.1" 200 207537 "" "Prometheus/2.33.4" 2023-06-10T21:13:03.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:02 smithi064 ceph-mon[186706]: pgmap v246: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4774/79299 objects degraded (6.020%); 3.0 MiB/s, 56 objects/s recovering 2023-06-10T21:13:03.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2375481889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:02 smithi152 ceph-mon[156819]: pgmap v246: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4774/79299 objects degraded (6.020%); 3.0 MiB/s, 56 objects/s recovering 2023-06-10T21:13:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2375481889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:04.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:04.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:05.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:05 smithi064 ceph-mon[186706]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4774/77688 objects degraded (6.145%); 823 KiB/s, 22 objects/s recovering 2023-06-10T21:13:05.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:05 smithi064 ceph-mon[180297]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4774/77688 objects degraded (6.145%); 823 KiB/s, 22 objects/s recovering 2023-06-10T21:13:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:05 smithi152 ceph-mon[156819]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 4774/77688 objects degraded (6.145%); 823 KiB/s, 22 objects/s recovering 2023-06-10T21:13:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/673122943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:06.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4774/77688 objects degraded (6.145%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:06.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/673122943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:06.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4774/77688 objects degraded (6.145%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/673122943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4774/77688 objects degraded (6.145%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:07.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:07 smithi064 ceph-mon[180297]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 4701/69813 objects degraded (6.734%); 1.3 MiB/s, 30 objects/s recovering 2023-06-10T21:13:07.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:07 smithi064 ceph-mon[186706]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 4701/69813 objects degraded (6.734%); 1.3 MiB/s, 30 objects/s recovering 2023-06-10T21:13:07.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:07 smithi152 ceph-mon[156819]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 4701/69813 objects degraded (6.734%); 1.3 MiB/s, 30 objects/s recovering 2023-06-10T21:13:08.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:08 smithi064 ceph-mon[180297]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 4701/69813 objects degraded (6.734%); 1.1 MiB/s, 25 objects/s recovering 2023-06-10T21:13:08.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3459436073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:08.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:08 smithi064 ceph-mon[186706]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 4701/69813 objects degraded (6.734%); 1.1 MiB/s, 25 objects/s recovering 2023-06-10T21:13:08.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3459436073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:08.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:08 smithi152 ceph-mon[156819]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 4701/69813 objects degraded (6.734%); 1.1 MiB/s, 25 objects/s recovering 2023-06-10T21:13:08.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3459436073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:10.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:10 smithi064 ceph-mon[180297]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4701/64323 objects degraded (7.308%); 678 KiB/s, 10 objects/s recovering 2023-06-10T21:13:10.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4701/64323 objects degraded (7.308%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:10.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:10 smithi064 ceph-mon[186706]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4701/64323 objects degraded (7.308%); 678 KiB/s, 10 objects/s recovering 2023-06-10T21:13:10.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4701/64323 objects degraded (7.308%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:10 smithi152 ceph-mon[156819]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 4701/64323 objects degraded (7.308%); 678 KiB/s, 10 objects/s recovering 2023-06-10T21:13:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4701/64323 objects degraded (7.308%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1665460867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:11.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1665460867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1665460867' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:12 smithi064 ceph-mon[180297]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4625/59775 objects degraded (7.737%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:13:12.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:12 smithi064 ceph-mon[186706]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4625/59775 objects degraded (7.737%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:13:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:12 smithi152 ceph-mon[156819]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 4625/59775 objects degraded (7.737%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:13:13.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:12] "GET /metrics HTTP/1.1" 200 207569 "" "Prometheus/2.33.4" 2023-06-10T21:13:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1811347303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:13.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1811347303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:13.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1811347303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:14 smithi064 ceph-mon[186706]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 4625/57966 objects degraded (7.979%); 793 KiB/s, 12 objects/s recovering 2023-06-10T21:13:14.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:14 smithi064 ceph-mon[180297]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 4625/57966 objects degraded (7.979%); 793 KiB/s, 12 objects/s recovering 2023-06-10T21:13:14.663 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:14 smithi064 conmon[171866]: 2023-06-10T21:13:14.448+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:14 smithi152 ceph-mon[156819]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 4625/57966 objects degraded (7.979%); 793 KiB/s, 12 objects/s recovering 2023-06-10T21:13:15.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4625/57966 objects degraded (7.979%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:15 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4625/57966 objects degraded (7.979%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:15 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:15 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:15 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4625/57966 objects degraded (7.979%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:16 smithi064 ceph-mon[186706]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 4549/50172 objects degraded (9.067%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:13:16.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1824578989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:16 smithi064 ceph-mon[180297]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 4549/50172 objects degraded (9.067%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:13:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1824578989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:16 smithi152 ceph-mon[156819]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 4549/50172 objects degraded (9.067%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:13:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1824578989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:18.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[186706]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4549/50172 objects degraded (9.067%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:18.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2503657463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:18.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:18.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[180297]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4549/50172 objects degraded (9.067%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:18.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2503657463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:18.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:18.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:18 smithi152 ceph-mon[156819]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4549/50172 objects degraded (9.067%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:18.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2503657463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:18.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:20 smithi152 ceph-mon[156819]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 929 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4549/44616 objects degraded (10.196%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:20 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4549/44616 objects degraded (10.196%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:20 smithi064 ceph-mon[186706]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 929 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4549/44616 objects degraded (10.196%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:20 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4549/44616 objects degraded (10.196%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:20 smithi064 ceph-mon[180297]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 929 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 4549/44616 objects degraded (10.196%); 810 KiB/s, 12 objects/s recovering 2023-06-10T21:13:20.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:20 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4549/44616 objects degraded (10.196%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2214017288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:21.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2214017288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:21.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2214017288' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:22 smithi152 ceph-mon[156819]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 841 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4472/40416 objects degraded (11.065%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:22.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:22 smithi064 ceph-mon[186706]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 841 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4472/40416 objects degraded (11.065%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:22.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:22 smithi064 ceph-mon[180297]: pgmap v256: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 841 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4472/40416 objects degraded (11.065%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:22] "GET /metrics HTTP/1.1" 200 207577 "" "Prometheus/2.33.4" 2023-06-10T21:13:23.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1091412035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1091412035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1091412035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:24 smithi064 ceph-mon[180297]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 804 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4472/38634 objects degraded (11.575%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:24.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:24 smithi064 ceph-mon[186706]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 804 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4472/38634 objects degraded (11.575%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:24 smithi152 ceph-mon[156819]: pgmap v257: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 804 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s; 4472/38634 objects degraded (11.575%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4472/38634 objects degraded (11.575%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:25.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4472/38634 objects degraded (11.575%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4472/38634 objects degraded (11.575%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:26.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:26 smithi064 ceph-mon[180297]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4396/30606 objects degraded (14.363%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:26.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3164676998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:26.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:26 smithi064 ceph-mon[186706]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4396/30606 objects degraded (14.363%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:26.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3164676998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:26 smithi152 ceph-mon[156819]: pgmap v258: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 4396/30606 objects degraded (14.363%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3164676998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:28 smithi064 ceph-mon[180297]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4396/30606 objects degraded (14.363%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:28.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2944512185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:28 smithi064 ceph-mon[186706]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4396/30606 objects degraded (14.363%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2944512185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:28 smithi152 ceph-mon[156819]: pgmap v259: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 637 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 4396/30606 objects degraded (14.363%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2944512185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:29.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:29.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:29.915 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:29 smithi064 conmon[171866]: 2023-06-10T21:13:29.482+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[180297]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4396/25158 objects degraded (17.474%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4396/25158 objects degraded (17.474%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[186706]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4396/25158 objects degraded (17.474%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4396/25158 objects degraded (17.474%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:30 smithi152 ceph-mon[156819]: pgmap v260: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 523 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 4396/25158 objects degraded (17.474%); 815 KiB/s, 12 objects/s recovering 2023-06-10T21:13:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:30 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:30.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:30 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:30.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4396/25158 objects degraded (17.474%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:31.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1063584781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:31.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1063584781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1063584781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:32.882 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:32] "GET /metrics HTTP/1.1" 200 207577 "" "Prometheus/2.33.4" 2023-06-10T21:13:32.882 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:32 smithi064 ceph-mon[186706]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 433 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4317/20808 objects degraded (20.747%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:32.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:32 smithi064 ceph-mon[180297]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 433 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4317/20808 objects degraded (20.747%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:32 smithi152 ceph-mon[156819]: pgmap v261: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 433 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 4317/20808 objects degraded (20.747%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:33.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1079521622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:33.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:33.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1079521622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1079521622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:34.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:34 smithi064 ceph-mon[186706]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 396 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 4317/19071 objects degraded (22.636%); 826 KiB/s, 12 objects/s recovering 2023-06-10T21:13:34.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:34 smithi064 ceph-mon[180297]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 396 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 4317/19071 objects degraded (22.636%); 826 KiB/s, 12 objects/s recovering 2023-06-10T21:13:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:34 smithi152 ceph-mon[156819]: pgmap v262: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 396 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 4317/19071 objects degraded (22.636%); 826 KiB/s, 12 objects/s recovering 2023-06-10T21:13:35.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:35 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4317/19071 objects degraded (22.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:35.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:35 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4317/19071 objects degraded (22.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:35 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4317/19071 objects degraded (22.636%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:36.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:36 smithi064 ceph-mon[186706]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4238/11070 objects degraded (38.284%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:36.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/797017322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:36.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:36 smithi064 ceph-mon[180297]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4238/11070 objects degraded (38.284%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:36.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/797017322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:36 smithi152 ceph-mon[156819]: pgmap v263: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 4238/11070 objects degraded (38.284%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/797017322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:38.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:38 smithi064 ceph-mon[180297]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 4238/11070 objects degraded (38.284%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:38.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:38 smithi064 ceph-mon[186706]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 4238/11070 objects degraded (38.284%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:38 smithi152 ceph-mon[156819]: pgmap v264: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 230 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 4238/11070 objects degraded (38.284%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:39.567 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 115593 objects 2023-06-10T21:13:39.567 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :172.645 2023-06-10T21:13:39.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3043211243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:40.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3043211243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:40.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3043211243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:40 smithi152 ceph-mon[156819]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 114 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4238/5499 objects degraded (77.069%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4238/5499 objects degraded (77.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:41.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:40 smithi064 ceph-mon[186706]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 114 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4238/5499 objects degraded (77.069%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4238/5499 objects degraded (77.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:41.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:40 smithi064 ceph-mon[180297]: pgmap v265: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 114 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 4238/5499 objects degraded (77.069%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:41.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4238/5499 objects degraded (77.069%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:41.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2638548189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:42.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2638548189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2638548189' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:42 smithi152 ceph-mon[156819]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 26 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4159/1293 objects degraded (321.655%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:42] "GET /metrics HTTP/1.1" 200 207578 "" "Prometheus/2.33.4" 2023-06-10T21:13:43.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:42 smithi064 ceph-mon[186706]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 26 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4159/1293 objects degraded (321.655%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:43.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:42 smithi064 ceph-mon[180297]: pgmap v266: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 26 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 4159/1293 objects degraded (321.655%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:13:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2491081182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:44.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2491081182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:44.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2491081182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:44.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:44 smithi064 conmon[171866]: 2023-06-10T21:13:44.513+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:44.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:44 smithi064 ceph-mon[186706]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 8.3 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4159/444 objects degraded (936.712%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:44 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:44 smithi064 ceph-mon[180297]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 8.3 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4159/444 objects degraded (936.712%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:44.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:44 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:44.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:44 smithi152 ceph-mon[156819]: pgmap v267: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 8.3 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 4159/444 objects degraded (936.712%); 842 KiB/s, 13 objects/s recovering 2023-06-10T21:13:44.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:44 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:45.244 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-06-10T21:13:45.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-06-10T21:13:45.644 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:13:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:45 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:45 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4159/444 objects degraded (936.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:46.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:46.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:46.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4159/444 objects degraded (936.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:13:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-10T21:13:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4159/444 objects degraded (936.712%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:46 smithi152 ceph-mon[156819]: pgmap v268: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 4102/72 objects degraded (5697.222%); 842 KiB/s, 22 objects/s recovering 2023-06-10T21:13:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3937164460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:47.070 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:46 smithi064 ceph-mon[180297]: pgmap v268: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 4102/72 objects degraded (5697.222%); 842 KiB/s, 22 objects/s recovering 2023-06-10T21:13:47.071 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3937164460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:47.071 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:46 smithi064 ceph-mon[186706]: pgmap v268: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 4102/72 objects degraded (5697.222%); 842 KiB/s, 22 objects/s recovering 2023-06-10T21:13:47.071 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3937164460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:48 smithi152 ceph-mon[156819]: pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4255209715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:49.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[186706]: pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:49.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4255209715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[180297]: pgmap v269: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:13:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4255209715' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:50 smithi152 ceph-mon[156819]: pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4102/72 objects degraded (5697.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:50.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:13:50.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/324716525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[180297]: pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4102/72 objects degraded (5697.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:51.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:13:51.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/324716525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[186706]: pgmap v270: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 305 op/s; 4102/72 objects degraded (5697.222%); 421 KiB/s, 16 objects/s recovering 2023-06-10T21:13:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4102/72 objects degraded (5697.222%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:13:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:13:51.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/324716525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:51.295 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_2' removed 2023-06-10T21:13:52.020 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:13:52.021 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:13:52.022 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-06-10T21:13:52.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_3 16 2023-06-10T21:13:52.383 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:13:52.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 3985/72 objects degraded (5534.722%); 421 KiB/s, 25 objects/s recovering 2023-06-10T21:13:52.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:13:52.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: osdmap e97: 8 total, 8 up, 8 in 2023-06-10T21:13:52.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[180297]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 3985/72 objects degraded (5534.722%); 421 KiB/s, 25 objects/s recovering 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: osdmap e97: 8 total, 8 up, 8 in 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.587 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:52 smithi064 ceph-mon[186706]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: pgmap v271: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 150 op/s; 3985/72 objects degraded (5534.722%); 421 KiB/s, 25 objects/s recovering 2023-06-10T21:13:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/958080236' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:13:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: osdmap e97: 8 total, 8 up, 8 in 2023-06-10T21:13:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: failed to encode map e97 with expected crc 2023-06-10T21:13:52.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:52 smithi152 ceph-mon[156819]: failed to encode map e97 with expected crc 2023-06-10T21:13:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:13:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:13:52] "GET /metrics HTTP/1.1" 200 207544 "" "Prometheus/2.33.4" 2023-06-10T21:13:53.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2794719264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:53.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2794719264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2794719264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:54.460 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' created 2023-06-10T21:13:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3985/72 objects degraded (5534.722%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:13:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:13:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2708712633' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3985/72 objects degraded (5534.722%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:13:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:13:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2708712633' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:54 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3985/72 objects degraded (5534.722%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:13:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:54 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:13:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2708712633' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:55.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-06-10T21:13:55.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:55.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:55.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-10T21:13:55.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: osdmap e98: 8 total, 8 up, 8 in 2023-06-10T21:13:55.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/723133326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:55.492 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:13:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-10T21:13:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: osdmap e98: 8 total, 8 up, 8 in 2023-06-10T21:13:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/723133326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: pgmap v273: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: failed to encode map e98 with expected crc 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: osdmap e98: 8 total, 8 up, 8 in 2023-06-10T21:13:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/723133326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:56.802 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: pgmap v275: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:56.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: osdmap e99: 8 total, 8 up, 8 in 2023-06-10T21:13:56.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[186706]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: pgmap v275: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: osdmap e99: 8 total, 8 up, 8 in 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.804 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.805 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:56 smithi064 ceph-mon[180297]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: pgmap v275: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: osdmap e99: 8 total, 8 up, 8 in 2023-06-10T21:13:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:56 smithi152 ceph-mon[156819]: failed to encode map e99 with expected crc 2023-06-10T21:13:57.479 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-06-10T21:13:57.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[186706]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[186706]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[186706]: osdmap e100: 8 total, 8 up, 8 in 2023-06-10T21:13:57.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:13:57.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2936358185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:57.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[180297]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[180297]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[180297]: osdmap e100: 8 total, 8 up, 8 in 2023-06-10T21:13:57.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:13:57.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2936358185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:57 smithi152 ceph-mon[156819]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:57 smithi152 ceph-mon[156819]: failed to encode map e100 with expected crc 2023-06-10T21:13:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:57 smithi152 ceph-mon[156819]: osdmap e100: 8 total, 8 up, 8 in 2023-06-10T21:13:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:13:57.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2936358185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:13:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: pgmap v278: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:13:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: osdmap e101: 8 total, 8 up, 8 in 2023-06-10T21:13:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: pgmap v278: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:13:58.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: osdmap e101: 8 total, 8 up, 8 in 2023-06-10T21:13:58.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: pgmap v278: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:13:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3854293227' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: osdmap e101: 8 total, 8 up, 8 in 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: failed to encode map e101 with expected crc 2023-06-10T21:13:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:13:59.141 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-06-10T21:13:59.144 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:13:59.145 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:13:59.198 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:13:59.199 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:13:59.199 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_208298 2023-06-10T21:13:59.199 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:13:59.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:13:59.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:13:59 smithi064 ceph-mon[186706]: failed to encode map e101 with expected crc 2023-06-10T21:13:59.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:13:59 smithi064 ceph-mon[180297]: failed to encode map e101 with expected crc 2023-06-10T21:13:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:13:59 smithi152 ceph-mon[156819]: failed to encode map e101 with expected crc 2023-06-10T21:14:00.199 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 15 3199 3184 198.978 199 0.0066192 0.00500875 2023-06-10T21:14:00.789 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[180297]: pgmap v280: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:14:00.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:00.790 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1392265567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:00.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[186706]: pgmap v280: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:14:00.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:00.790 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1392265567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:00 smithi152 ceph-mon[156819]: pgmap v280: 89 pgs: 3 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:14:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1392265567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:01.199 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6301 6285 196.347 193.812 0.00362648 0.00507865 2023-06-10T21:14:02.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9109 9093 189.368 175.5 0.00559707 0.00526836 2023-06-10T21:14:02.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:02 smithi064 ceph-mon[180297]: pgmap v281: 89 pgs: 89 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:14:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:02 smithi064 ceph-mon[186706]: pgmap v281: 89 pgs: 89 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:14:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:02 smithi152 ceph-mon[156819]: pgmap v281: 89 pgs: 89 active+clean; 29 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:14:03.143 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:02] "GET /metrics HTTP/1.1" 200 207544 "" "Prometheus/2.33.4" 2023-06-10T21:14:03.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 11654 11638 181.789 159.062 0.0026546 0.00549019 2023-06-10T21:14:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1774638885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1774638885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:03.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:03.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1774638885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:03.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:03.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:04.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 14228 14212 177.604 160.875 0.00264836 0.0056227 2023-06-10T21:14:04.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[180297]: pgmap v282: 89 pgs: 89 active+clean; 139 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-06-10T21:14:04.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:04.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:04.711 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[186706]: pgmap v282: 89 pgs: 89 active+clean; 139 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-06-10T21:14:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:04 smithi152 ceph-mon[156819]: pgmap v282: 89 pgs: 89 active+clean; 139 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 285 op/s 2023-06-10T21:14:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:05.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 16476 16460 171.419 140.5 0.00458853 0.00582474 2023-06-10T21:14:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:06.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:06.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:06.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2415412716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:06.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:06.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:06.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:06.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:06.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2415412716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:06.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:06.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:06.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2415412716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:06.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:06.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 18852 18836 168.143 148.5 0.00811986 0.00594075 2023-06-10T21:14:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[180297]: pgmap v283: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:14:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[180297]: Upgrade: osd.1 is safe to restart 2023-06-10T21:14:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[186706]: pgmap v283: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:14:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:06.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:06 smithi064 ceph-mon[186706]: Upgrade: osd.1 is safe to restart 2023-06-10T21:14:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:06 smithi152 ceph-mon[156819]: pgmap v283: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-06-10T21:14:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:06 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-10T21:14:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:06 smithi152 ceph-mon[156819]: Upgrade: osd.1 is safe to restart 2023-06-10T21:14:07.200 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 21223 21207 165.648 148.188 0.00374225 0.00603134 2023-06-10T21:14:08.202 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 23482 23466 162.907 141.188 0.008141 0.00613142 2023-06-10T21:14:09.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[180297]: pgmap v284: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-10T21:14:09.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1515190735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:09.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:09.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[186706]: pgmap v284: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-10T21:14:09.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1515190735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:09.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:08 smithi152 ceph-mon[156819]: pgmap v284: 89 pgs: 89 active+clean; 849 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-10T21:14:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1515190735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:09.202 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 25771 25755 160.92 143.062 0.005086 0.00620797 2023-06-10T21:14:10.205 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 27900 27884 158.349 133.062 0.010313 0.00630885 2023-06-10T21:14:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:10 smithi064 ceph-mon[180297]: pgmap v285: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:14:11.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3015538771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:10 smithi064 ceph-mon[186706]: pgmap v285: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:14:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3015538771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:10 smithi152 ceph-mon[156819]: pgmap v285: 89 pgs: 89 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:14:11.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3015538771' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:11.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 29995 29979 156.065 130.938 0.00855146 0.00640222 2023-06-10T21:14:12.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 31893 31877 153.157 118.625 0.00718695 0.0065222 2023-06-10T21:14:12.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: pgmap v286: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-06-10T21:14:12.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: Upgrade: Updating osd.1 2023-06-10T21:14:12.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:12.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:14:12.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:12.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[186706]: Deploying daemon osd.1 on smithi064 2023-06-10T21:14:12.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: pgmap v286: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-06-10T21:14:12.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: Upgrade: Updating osd.1 2023-06-10T21:14:12.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:12.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:14:12.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:12.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:12 smithi064 ceph-mon[180297]: Deploying daemon osd.1 on smithi064 2023-06-10T21:14:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: pgmap v286: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-06-10T21:14:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: Upgrade: Updating osd.1 2023-06-10T21:14:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-10T21:14:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:12.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:12 smithi152 ceph-mon[156819]: Deploying daemon osd.1 on smithi064 2023-06-10T21:14:12.886 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:12] "GET /metrics HTTP/1.1" 200 207549 "" "Prometheus/2.33.4" 2023-06-10T21:14:13.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 33710 33694 150.33 113.562 0.00755969 0.0066476 2023-06-10T21:14:13.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2404454534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:13.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2404454534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2404454534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:14.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 35409 35393 147.342 106.188 0.00464489 0.00678288 2023-06-10T21:14:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:14 smithi064 ceph-mon[180297]: pgmap v287: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:14:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:14 smithi064 ceph-mon[186706]: pgmap v287: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:14:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:14 smithi152 ceph-mon[156819]: pgmap v287: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:14:15.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 37096 37080 144.724 105.438 0.0057194 0.00690426 2023-06-10T21:14:16.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 38333 38317 140.761 77.3125 0.0160675 0.00709859 2023-06-10T21:14:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:16 smithi064 ceph-mon[180297]: pgmap v288: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s 2023-06-10T21:14:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/714002587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:16 smithi064 ceph-mon[186706]: pgmap v288: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s 2023-06-10T21:14:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/714002587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:16 smithi152 ceph-mon[156819]: pgmap v288: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s 2023-06-10T21:14:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/714002587' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:16.908 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:16 smithi064 systemd[1]: Stopping Ceph osd.1 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:14:17.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 39942 39926 138.529 100.562 0.0113355 0.00721284 2023-06-10T21:14:17.909 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:17 smithi064 conmon[118035]: debug 2023-06-10T21:14:17.436+0000 7f43f844b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:14:17.910 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:17 smithi064 conmon[118035]: debug 2023-06-10T21:14:17.436+0000 7f43f844b700 -1 osd.1 101 *** Got signal Terminated *** 2023-06-10T21:14:17.910 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:17 smithi064 conmon[118035]: debug 2023-06-10T21:14:17.436+0000 7f43f844b700 -1 osd.1 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:14:18.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 41599 41583 136.689 103.562 0.00383643 0.00731047 2023-06-10T21:14:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:18 smithi152 ceph-mon[156819]: pgmap v289: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-10T21:14:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:18 smithi152 ceph-mon[156819]: osd.1 marked itself down and dead 2023-06-10T21:14:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2780622875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:18.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[180297]: pgmap v289: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-10T21:14:18.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[180297]: osd.1 marked itself down and dead 2023-06-10T21:14:18.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:18.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2780622875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:18.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[186706]: pgmap v289: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-10T21:14:18.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[186706]: osd.1 marked itself down and dead 2023-06-10T21:14:18.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:18.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2780622875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:18.777 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:18 smithi064 bash[210349]: 601741de864d5e571c93c7bada56c0abaf4f1a4e275d0d0ac5c5fc75d50fdfee 2023-06-10T21:14:19.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:14:19.210687+0000 min lat: 0.0013329 max lat: 0.252877 avg lat: 0.00732651 2023-06-10T21:14:19.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:14:19.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 41995 41979 131.095 24.75 0.00233433 0.00732651 2023-06-10T21:14:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:19 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:14:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:19 smithi152 ceph-mon[156819]: osdmap e102: 8 total, 7 up, 8 in 2023-06-10T21:14:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:19 smithi152 ceph-mon[156819]: failed to encode map e102 with expected crc 2023-06-10T21:14:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:19 smithi152 ceph-mon[156819]: failed to encode map e102 with expected crc 2023-06-10T21:14:19.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:14:19.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[186706]: osdmap e102: 8 total, 7 up, 8 in 2023-06-10T21:14:19.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[186706]: failed to encode map e102 with expected crc 2023-06-10T21:14:19.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[186706]: failed to encode map e102 with expected crc 2023-06-10T21:14:19.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:14:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[180297]: osdmap e102: 8 total, 7 up, 8 in 2023-06-10T21:14:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[180297]: failed to encode map e102 with expected crc 2023-06-10T21:14:19.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:19 smithi064 ceph-mon[180297]: failed to encode map e102 with expected crc 2023-06-10T21:14:20.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 43208 43192 128.464 75.8125 0.00627876 0.00777806 2023-06-10T21:14:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: pgmap v291: 89 pgs: 18 peering, 9 stale+active+clean, 62 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: osdmap e103: 8 total, 7 up, 8 in 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:20 smithi152 ceph-mon[156819]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: pgmap v291: 89 pgs: 18 peering, 9 stale+active+clean, 62 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-06-10T21:14:20.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: osdmap e103: 8 total, 7 up, 8 in 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.894 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[180297]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: pgmap v291: 89 pgs: 18 peering, 9 stale+active+clean, 62 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2023-06-10T21:14:20.894 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: osdmap e103: 8 total, 7 up, 8 in 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:20.895 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:20 smithi064 ceph-mon[186706]: failed to encode map e103 with expected crc 2023-06-10T21:14:21.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 45026 45010 127.789 113.625 0.00862381 0.0078209 2023-06-10T21:14:21.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/836560615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:21.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/836560615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/836560615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:22.213 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 46942 46926 127.439 119.75 0.00646515 0.00784222 2023-06-10T21:14:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:22 smithi152 ceph-mon[156819]: pgmap v293: 89 pgs: 5 active+undersized+degraded, 18 peering, 2 stale+active+clean, 11 active+undersized, 53 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 13144/123783 objects degraded (10.619%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:22 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 13144/123783 objects degraded (10.619%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:22.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:22 smithi064 ceph-mon[186706]: pgmap v293: 89 pgs: 5 active+undersized+degraded, 18 peering, 2 stale+active+clean, 11 active+undersized, 53 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 13144/123783 objects degraded (10.619%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:22.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:22 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 13144/123783 objects degraded (10.619%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:22.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:22 smithi064 ceph-mon[180297]: pgmap v293: 89 pgs: 5 active+undersized+degraded, 18 peering, 2 stale+active+clean, 11 active+undersized, 53 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s; 13144/123783 objects degraded (10.619%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:22.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:22 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 13144/123783 objects degraded (10.619%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:23.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:22] "GET /metrics HTTP/1.1" 200 207585 "" "Prometheus/2.33.4" 2023-06-10T21:14:23.163 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:22 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.1.service: Succeeded. 2023-06-10T21:14:23.163 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:22 smithi064 systemd[1]: Stopped Ceph osd.1 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:14:23.163 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:23 smithi064 systemd[1]: Starting Ceph osd.1 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:14:23.214 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 48690 48674 126.674 109.25 0.00756272 0.00788937 2023-06-10T21:14:23.775 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3155877938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:23.776 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3155877938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:23.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3155877938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:24.216 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 50441 50425 125.977 109.438 0.0120831 0.00793302 2023-06-10T21:14:24.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:24 smithi064 ceph-mon[186706]: pgmap v294: 89 pgs: 7 active+undersized+degraded, 18 peering, 14 active+undersized, 50 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 16090/130035 objects degraded (12.374%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:24.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:24 smithi064 ceph-mon[180297]: pgmap v294: 89 pgs: 7 active+undersized+degraded, 18 peering, 14 active+undersized, 50 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 16090/130035 objects degraded (12.374%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:24.773 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-06-10T21:14:24.774 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:24 smithi064 bash[211032]: --> ceph-volume raw activate successful for osd ID: 1 2023-06-10T21:14:24.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:24 smithi152 ceph-mon[156819]: pgmap v294: 89 pgs: 7 active+undersized+degraded, 18 peering, 14 active+undersized, 50 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 16090/130035 objects degraded (12.374%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:25.216 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 52206 52190 125.374 110.312 0.00718678 0.00797145 2023-06-10T21:14:26.216 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 53856 53840 124.55 103.125 0.00929997 0.00802363 2023-06-10T21:14:26.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[180297]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s; 21904/149271 objects degraded (14.674%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:26.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:14:26.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1959843748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:26.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[186706]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s; 21904/149271 objects degraded (14.674%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:26.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:14:26.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1959843748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:26 smithi152 ceph-mon[156819]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s; 21904/149271 objects degraded (14.674%); 8.0 KiB/s, 0 objects/s recovering 2023-06-10T21:14:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:26 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:14:26.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1959843748' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:27.218 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 55524 55508 123.819 104.25 0.0100738 0.00807145 2023-06-10T21:14:27.252 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:26 smithi064 bash[211480]: b6d15648d91ceceda6d37fea5d47f956729c2fed4fb584bb94e9f000770a37f1 2023-06-10T21:14:27.252 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:26 smithi064 systemd[1]: Started Ceph osd.1 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:14:27.252 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:26 smithi064 ceph-osd[211543]: -- 172.21.15.64:0/825260350 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cee64184e0 con 0x55cee63eac00 2023-06-10T21:14:27.658 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:27 smithi064 conmon[211518]: 2023-06-10T21:14:27.242+0000 7f8424dac640 -1 Falling back to public interface 2023-06-10T21:14:28.218 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 57055 57039 122.85 95.6875 0.00319453 0.00813401 2023-06-10T21:14:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:28.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:28.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:29.218 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 58578 58562 121.928 95.1875 0.0162341 0.00819695 2023-06-10T21:14:29.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:29 smithi064 ceph-mon[180297]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21904/149271 objects degraded (14.674%); 7.3 KiB/s, 0 objects/s recovering 2023-06-10T21:14:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3679360796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:29 smithi064 ceph-mon[186706]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21904/149271 objects degraded (14.674%); 7.3 KiB/s, 0 objects/s recovering 2023-06-10T21:14:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3679360796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:29 smithi152 ceph-mon[156819]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s; 21904/149271 objects degraded (14.674%); 7.3 KiB/s, 0 objects/s recovering 2023-06-10T21:14:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3679360796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:30.219 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 60057 60041 120.971 92.4375 0.00897058 0.00826193 2023-06-10T21:14:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:30 smithi064 ceph-mon[180297]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 22959/166446 objects degraded (13.794%); 6.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:30.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22959/166446 objects degraded (13.794%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:30 smithi064 ceph-mon[186706]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 22959/166446 objects degraded (13.794%); 6.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22959/166446 objects degraded (13.794%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:30 smithi152 ceph-mon[156819]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s; 22959/166446 objects degraded (13.794%); 6.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22959/166446 objects degraded (13.794%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:31.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 61437 61421 119.881 86.25 0.00956944 0.0083366 2023-06-10T21:14:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1814320623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1814320623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1814320623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:32.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 62938 62922 119.091 93.8125 0.021835 0.00839293 2023-06-10T21:14:33.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:32] "GET /metrics HTTP/1.1" 200 207585 "" "Prometheus/2.33.4" 2023-06-10T21:14:33.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[186706]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 25449/173916 objects degraded (14.633%); 5.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[180297]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 25449/173916 objects degraded (14.633%); 5.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:33.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:32 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:32 smithi152 ceph-mon[156819]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 25449/173916 objects degraded (14.633%); 5.4 KiB/s, 0 objects/s recovering 2023-06-10T21:14:33.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:32 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:32 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:33.229 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 64310 64294 118.101 85.75 0.0159212 0.00846249 2023-06-10T21:14:34.224 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 65712 65696 117.23 87.625 0.0146154 0.00852562 2023-06-10T21:14:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 25914/179334 objects degraded (14.450%) 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/610953214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 25914/179334 objects degraded (14.450%) 2023-06-10T21:14:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/610953214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s; 25914/179334 objects degraded (14.450%) 2023-06-10T21:14:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/610953214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:35.224 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 67167 67151 116.5 90.9375 0.00363888 0.00857904 2023-06-10T21:14:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:35 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25914/179334 objects degraded (14.450%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:35 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25914/179334 objects degraded (14.450%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:35 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25914/179334 objects degraded (14.450%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:36.226 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 68577 68561 115.728 88.125 0.0151698 0.00863642 2023-06-10T21:14:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:36 smithi064 ceph-mon[180297]: pgmap v300: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4271493487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:36 smithi064 ceph-mon[186706]: pgmap v300: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4271493487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:36 smithi152 ceph-mon[156819]: pgmap v300: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4271493487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:37.226 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 69923 69907 114.897 84.125 0.0092572 0.00869836 2023-06-10T21:14:38.227 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 71194 71178 113.988 79.4375 0.0125849 0.00876825 2023-06-10T21:14:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:38 smithi064 ceph-mon[180297]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:38 smithi064 ceph-mon[186706]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:38.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:38 smithi152 ceph-mon[156819]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 28573/195567 objects degraded (14.610%) 2023-06-10T21:14:39.226 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:14:39.224308+0000 min lat: 0.0013329 max lat: 1.03783 avg lat: 0.00884431 2023-06-10T21:14:39.227 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:14:39.227 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 72386 72370 113.001 74.5 0.0113096 0.00884431 2023-06-10T21:14:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2530822728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2530822728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2530822728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:40.158 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:39 smithi064 conmon[211518]: 2023-06-10T21:14:39.858+0000 7f8424dac640 -1 osd.1 101 log_to_monitors true 2023-06-10T21:14:40.230 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 73597 73581 112.081 75.6875 0.00451888 0.00891739 2023-06-10T21:14:40.658 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:14:40 smithi064 conmon[211518]: 2023-06-10T21:14:40.402+0000 7f8413212700 -1 osd.1 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:14:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 29431/209364 objects degraded (14.057%) 2023-06-10T21:14:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T21:14:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 29431/209364 objects degraded (14.057%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 29431/209364 objects degraded (14.057%) 2023-06-10T21:14:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T21:14:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 29431/209364 objects degraded (14.057%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: pgmap v302: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 29431/209364 objects degraded (14.057%) 2023-06-10T21:14:40.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-10T21:14:40.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 29431/209364 objects degraded (14.057%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:40.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:40.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:14:41.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:40 smithi064 conmon[171866]: 2023-06-10T21:14:40.667+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:14:41.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 74934 74918 111.402 83.5625 0.00810658 0.00897217 2023-06-10T21:14:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T21:14:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: osdmap e104: 8 total, 7 up, 8 in 2023-06-10T21:14:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:14:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:14:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3592472918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:41.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:41.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: osdmap e104: 8 total, 7 up, 8 in 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:14:41.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3592472918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: failed to encode map e104 with expected crc 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: osdmap e104: 8 total, 7 up, 8 in 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:14:41.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3592472918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:42.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 75245 75229 109.264 19.4375 0.00346679 0.00898149 2023-06-10T21:14:42.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 31486/215529 objects degraded (14.609%) 2023-06-10T21:14:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:14:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888] boot 2023-06-10T21:14:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: osdmap e105: 8 total, 8 up, 8 in 2023-06-10T21:14:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: failed to encode map e105 with expected crc 2023-06-10T21:14:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 31486/215529 objects degraded (14.609%) 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888] boot 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: osdmap e105: 8 total, 8 up, 8 in 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: failed to encode map e105 with expected crc 2023-06-10T21:14:42.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:14:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 31486/215529 objects degraded (14.609%) 2023-06-10T21:14:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:14:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: osd.1 [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888] boot 2023-06-10T21:14:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: osdmap e105: 8 total, 8 up, 8 in 2023-06-10T21:14:42.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: failed to encode map e105 with expected crc 2023-06-10T21:14:42.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-10T21:14:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:42] "GET /metrics HTTP/1.1" 200 207557 "" "Prometheus/2.33.4" 2023-06-10T21:14:43.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 75245 75229 106.783 0 - 0.00898149 2023-06-10T21:14:43.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: osdmap e106: 8 total, 8 up, 8 in 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:43 smithi152 ceph-mon[156819]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: osdmap e106: 8 total, 8 up, 8 in 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[186706]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: osdmap e106: 8 total, 8 up, 8 in 2023-06-10T21:14:43.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:43.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:43 smithi064 ceph-mon[180297]: failed to encode map e106 with expected crc 2023-06-10T21:14:44.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 75908 75892 105.331 20.7188 0.00838451 0.00948835 2023-06-10T21:14:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: pgmap v307: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 31485/218619 objects degraded (14.402%) 2023-06-10T21:14:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: osdmap e107: 8 total, 8 up, 8 in 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/550880090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:44.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: pgmap v307: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 31485/218619 objects degraded (14.402%) 2023-06-10T21:14:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: osdmap e107: 8 total, 8 up, 8 in 2023-06-10T21:14:44.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/550880090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:44.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: pgmap v307: 89 pgs: 3 peering, 9 active+undersized+degraded, 27 active+undersized, 50 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 31485/218619 objects degraded (14.402%) 2023-06-10T21:14:44.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: osdmap e107: 8 total, 8 up, 8 in 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: failed to encode map e107 with expected crc 2023-06-10T21:14:44.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/550880090' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:45.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 76940 76924 104.444 64.5 0.021005 0.00956997 2023-06-10T21:14:45.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 31485/218619 objects degraded (14.402%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 31485/218619 objects degraded (14.402%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 31485/218619 objects degraded (14.402%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:46.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 77923 77907 103.529 61.4375 0.0146206 0.00965486 2023-06-10T21:14:46.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:46 smithi064 ceph-mon[180297]: pgmap v309: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 16734/226620 objects degraded (7.384%); 210 KiB/s, 3 objects/s recovering 2023-06-10T21:14:46.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/781434949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:46.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:46 smithi064 ceph-mon[186706]: pgmap v309: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 16734/226620 objects degraded (7.384%); 210 KiB/s, 3 objects/s recovering 2023-06-10T21:14:46.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/781434949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:46 smithi152 ceph-mon[156819]: pgmap v309: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 9 active+undersized, 70 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 16734/226620 objects degraded (7.384%); 210 KiB/s, 3 objects/s recovering 2023-06-10T21:14:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/781434949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:47.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 78909 78893 102.657 61.625 0.0124796 0.00973559 2023-06-10T21:14:48.232 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 79908 79892 101.836 62.4375 0.0230364 0.00981401 2023-06-10T21:14:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[186706]: pgmap v310: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 16734/226620 objects degraded (7.384%); 171 KiB/s, 2 objects/s recovering 2023-06-10T21:14:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[180297]: pgmap v310: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 16734/226620 objects degraded (7.384%); 171 KiB/s, 2 objects/s recovering 2023-06-10T21:14:48.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:48.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:48 smithi152 ceph-mon[156819]: pgmap v310: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 79 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 16734/226620 objects degraded (7.384%); 171 KiB/s, 2 objects/s recovering 2023-06-10T21:14:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:14:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:14:49.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 80858 80842 100.987 59.375 0.0155662 0.00989769 2023-06-10T21:14:49.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2742326586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:49.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2742326586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:49.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2742326586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:50.231 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 81841 81825 100.212 61.4375 0.0160615 0.00997333 2023-06-10T21:14:50.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:50 smithi064 ceph-mon[180297]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 14309/236901 objects degraded (6.040%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:14:50.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14309/236901 objects degraded (6.040%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:50.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:50 smithi064 ceph-mon[186706]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 14309/236901 objects degraded (6.040%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:14:50.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14309/236901 objects degraded (6.040%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:50 smithi152 ceph-mon[156819]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s; 14309/236901 objects degraded (6.040%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:14:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14309/236901 objects degraded (6.040%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:51.232 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 82764 82748 99.3931 57.6875 0.013655 0.0100554 2023-06-10T21:14:51.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2487698452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:51.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2487698452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:51.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2487698452' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:52.232 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 15 83737 83722 98.6666 60.875 0.0114326 0.0101302 2023-06-10T21:14:52.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:52 smithi064 ceph-mon[180297]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 14309/241434 objects degraded (5.927%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:14:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:52 smithi064 ceph-mon[186706]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 14309/241434 objects degraded (5.927%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:14:52.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:52 smithi152 ceph-mon[156819]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s; 14309/241434 objects degraded (5.927%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:14:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:14:52] "GET /metrics HTTP/1.1" 200 212749 "" "Prometheus/2.33.4" 2023-06-10T21:14:53.233 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 84717 84701 97.9728 61.1875 0.0160798 0.0102025 2023-06-10T21:14:54.232 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 85631 85615 97.2304 57.125 0.0113801 0.0102798 2023-06-10T21:14:54.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:54 smithi064 ceph-mon[186706]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14309/245160 objects degraded (5.837%); 857 KiB/s, 13 objects/s recovering 2023-06-10T21:14:54.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2944149955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:54.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:54 smithi064 ceph-mon[180297]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14309/245160 objects degraded (5.837%); 857 KiB/s, 13 objects/s recovering 2023-06-10T21:14:54.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2944149955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:54.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:54 smithi152 ceph-mon[156819]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14309/245160 objects degraded (5.837%); 857 KiB/s, 13 objects/s recovering 2023-06-10T21:14:54.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2944149955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:55.234 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 86589 86573 96.5613 59.875 0.0169134 0.0103507 2023-06-10T21:14:55.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:14:55 smithi064 conmon[171866]: 2023-06-10T21:14:55.683+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:14:55.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14309/245160 objects degraded (5.837%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:55.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14309/245160 objects degraded (5.837%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14309/245160 objects degraded (5.837%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:14:56.240 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 87508 87492 95.8648 57.4375 0.00674796 0.0104255 2023-06-10T21:14:56.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[186706]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 14192/255873 objects degraded (5.547%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:14:56.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:14:56.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3831606984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:56.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[180297]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 14192/255873 objects degraded (5.547%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:14:56.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:14:56.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3831606984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:56 smithi152 ceph-mon[156819]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 14192/255873 objects degraded (5.547%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:14:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:56 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:56 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:14:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:56 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:14:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3831606984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:57.241 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 88467 88451 95.2456 59.9375 0.00845732 0.0104941 2023-06-10T21:14:58.243 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 89417 89401 94.6335 59.375 0.0195575 0.0105616 2023-06-10T21:14:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:58 smithi064 ceph-mon[186706]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 14192/255873 objects degraded (5.547%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:14:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:58 smithi064 ceph-mon[180297]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 14192/255873 objects degraded (5.547%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:14:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:58 smithi152 ceph-mon[156819]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 14192/255873 objects degraded (5.547%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:14:59.244 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:14:59.242238+0000 min lat: 0.0013329 max lat: 2.21601 avg lat: 0.0106383 2023-06-10T21:14:59.245 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:14:59.245 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 90282 90266 93.9564 54.0625 0.0190346 0.0106383 2023-06-10T21:14:59.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:14:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2253056778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:59.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:14:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2253056778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:14:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:14:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2253056778' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:00.244 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 91213 91197 93.3702 58.1875 0.0221686 0.0107053 2023-06-10T21:15:00.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:00 smithi064 ceph-mon[180297]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14075/265443 objects degraded (5.302%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:00.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14075/265443 objects degraded (5.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:00 smithi064 ceph-mon[186706]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14075/265443 objects degraded (5.302%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14075/265443 objects degraded (5.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:00.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:00 smithi152 ceph-mon[156819]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 14075/265443 objects degraded (5.302%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:00.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14075/265443 objects degraded (5.302%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:01.246 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 92116 92100 92.7722 56.4375 0.0179501 0.0107734 2023-06-10T21:15:01.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/780553427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/780553427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/780553427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:02.246 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 93006 92990 92.1829 55.625 0.00625085 0.0108424 2023-06-10T21:15:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:02 smithi064 ceph-mon[180297]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 14075/269751 objects degraded (5.218%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:02.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:02 smithi064 ceph-mon[186706]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 14075/269751 objects degraded (5.218%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:02 smithi152 ceph-mon[156819]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 14075/269751 objects degraded (5.218%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:02] "GET /metrics HTTP/1.1" 200 212749 "" "Prometheus/2.33.4" 2023-06-10T21:15:03.246 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 93919 93903 91.6344 57.0625 0.0174027 0.0109084 2023-06-10T21:15:03.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:03.921 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:04.247 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 94751 94735 91.0237 52 0.0221688 0.0109811 2023-06-10T21:15:04.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:04 smithi064 ceph-mon[186706]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 14075/273132 objects degraded (5.153%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:04.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3310180006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:04 smithi064 ceph-mon[180297]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 14075/273132 objects degraded (5.153%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:04.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3310180006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:04 smithi152 ceph-mon[156819]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 880 op/s; 14075/273132 objects degraded (5.153%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:04.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3310180006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:05.247 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 95636 95620 90.4829 55.3125 0.0241526 0.0110463 2023-06-10T21:15:05.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14075/273132 objects degraded (5.153%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14075/273132 objects degraded (5.153%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:06.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14075/273132 objects degraded (5.153%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:06.249 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 96516 96500 89.9536 55 0.0243738 0.0111114 2023-06-10T21:15:06.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:06 smithi152 ceph-mon[156819]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 13959/283227 objects degraded (4.929%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:07.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:06 smithi064 ceph-mon[186706]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 13959/283227 objects degraded (4.929%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:06 smithi064 ceph-mon[180297]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 13959/283227 objects degraded (4.929%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:07.250 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 97352 97336 89.3959 52.25 0.0122436 0.0111813 2023-06-10T21:15:08.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1940420620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:08.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1940420620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:08.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1940420620' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:08.250 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 98229 98213 88.895 54.8125 0.0176186 0.0112439 2023-06-10T21:15:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:08 smithi152 ceph-mon[156819]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 13959/283227 objects degraded (4.929%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:09.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:08 smithi064 ceph-mon[180297]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 13959/283227 objects degraded (4.929%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:09.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:08 smithi064 ceph-mon[186706]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 13959/283227 objects degraded (4.929%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:09.250 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 99049 99033 88.3574 51.25 0.0116922 0.0113132 2023-06-10T21:15:10.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/5117102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/5117102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/5117102' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:10.251 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 99891 99875 87.8544 52.625 0.00503923 0.0113757 2023-06-10T21:15:11.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:10 smithi064 ceph-mon[186706]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13843/292119 objects degraded (4.739%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:11.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13843/292119 objects degraded (4.739%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:11.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:10 smithi064 ceph-mon[180297]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13843/292119 objects degraded (4.739%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:11.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13843/292119 objects degraded (4.739%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:11.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:10 smithi064 conmon[171866]: 2023-06-10T21:15:10.708+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:10 smithi152 ceph-mon[156819]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 13843/292119 objects degraded (4.739%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13843/292119 objects degraded (4.739%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:11.253 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 100741 100725 87.3692 53.125 0.0170257 0.0114407 2023-06-10T21:15:12.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:11 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:12.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:11 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:12.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:11 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:12.253 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 101547 101531 86.8627 50.375 0.0109576 0.0115079 2023-06-10T21:15:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:12] "GET /metrics HTTP/1.1" 200 212784 "" "Prometheus/2.33.4" 2023-06-10T21:15:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:12 smithi064 ceph-mon[180297]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 13843/296118 objects degraded (4.675%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2353030352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:12 smithi064 ceph-mon[186706]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 13843/296118 objects degraded (4.675%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:13.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2353030352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:12 smithi152 ceph-mon[156819]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 13843/296118 objects degraded (4.675%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2353030352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:13.256 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 102423 102407 86.4255 54.75 0.0144948 0.0115653 2023-06-10T21:15:14.263 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 103251 103235 85.96 51.75 0.00977687 0.011628 2023-06-10T21:15:15.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:14 smithi064 ceph-mon[180297]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 13843/299319 objects degraded (4.625%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:15.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2018983543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:15.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:14 smithi064 ceph-mon[186706]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 13843/299319 objects degraded (4.625%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:15.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2018983543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:14 smithi152 ceph-mon[156819]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 13843/299319 objects degraded (4.625%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2018983543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:15.263 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 104092 104076 85.5171 52.5625 0.021034 0.0116883 2023-06-10T21:15:16.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:15 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13843/299319 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:16.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:15 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13843/299319 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:15 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13843/299319 objects degraded (4.625%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:16.263 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 104932 104916 85.0886 52.5 0.0204443 0.0117475 2023-06-10T21:15:17.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:16 smithi064 ceph-mon[180297]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13726/308802 objects degraded (4.445%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:17.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:16 smithi064 ceph-mon[186706]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13726/308802 objects degraded (4.445%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:17.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:16 smithi152 ceph-mon[156819]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 13726/308802 objects degraded (4.445%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:17.264 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 105707 105691 84.6176 48.4375 0.0149088 0.0118127 2023-06-10T21:15:18.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3768389240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:18.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3768389240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:18.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3768389240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:18.264 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 106535 106519 84.2018 51.75 0.0192417 0.0118705 2023-06-10T21:15:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:18 smithi064 ceph-mon[180297]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 13726/308802 objects degraded (4.445%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:19.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:19.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:18 smithi064 ceph-mon[186706]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 13726/308802 objects degraded (4.445%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:19.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:18 smithi152 ceph-mon[156819]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s; 13726/308802 objects degraded (4.445%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:19.264 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:15:19.262459+0000 min lat: 0.0013329 max lat: 2.21601 avg lat: 0.0119304 2023-06-10T21:15:19.265 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:15:19.265 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 107332 107316 83.7722 49.8125 0.00970228 0.0119304 2023-06-10T21:15:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1265556195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:20.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1265556195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:20.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1265556195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:20.271 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 108153 108137 83.37 51.3125 0.00975857 0.011989 2023-06-10T21:15:21.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:20 smithi064 ceph-mon[180297]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 13610/317175 objects degraded (4.291%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:21.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:20 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13610/317175 objects degraded (4.291%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:21.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:20 smithi064 ceph-mon[186706]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 13610/317175 objects degraded (4.291%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:21.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:20 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13610/317175 objects degraded (4.291%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:21.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:20 smithi152 ceph-mon[156819]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 13610/317175 objects degraded (4.291%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:21.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:20 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13610/317175 objects degraded (4.291%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:21.266 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 108970 108954 82.9763 51.0625 0.0151744 0.0120468 2023-06-10T21:15:22.266 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 109754 109738 82.5672 49 0.0158724 0.0121058 2023-06-10T21:15:23.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:22] "GET /metrics HTTP/1.1" 200 212823 "" "Prometheus/2.33.4" 2023-06-10T21:15:23.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:22 smithi064 ceph-mon[186706]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 13610/321105 objects degraded (4.238%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:23.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1989260181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:23.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:22 smithi064 ceph-mon[180297]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 13610/321105 objects degraded (4.238%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:23.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1989260181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:23.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:22 smithi152 ceph-mon[156819]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 13610/321105 objects degraded (4.238%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:23.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1989260181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:23.266 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 110571 110555 82.1923 51.0625 0.0120511 0.012161 2023-06-10T21:15:24.269 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 111359 111343 81.8028 49.25 0.0188675 0.0122189 2023-06-10T21:15:25.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:24 smithi064 ceph-mon[186706]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 13610/324075 objects degraded (4.200%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:25.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3719366812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:24 smithi064 ceph-mon[180297]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 13610/324075 objects degraded (4.200%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:25.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3719366812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:25.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:24 smithi152 ceph-mon[156819]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 776 op/s; 13610/324075 objects degraded (4.200%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:25.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3719366812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:25.271 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 112160 112144 81.4319 50.0625 0.0280713 0.012276 2023-06-10T21:15:26.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:25 smithi064 conmon[171866]: 2023-06-10T21:15:25.738+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13610/324075 objects degraded (4.200%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:26.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:26.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13610/324075 objects degraded (4.200%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:26.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:26.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13610/324075 objects degraded (4.200%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:26.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:26.272 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 112974 112958 81.0803 50.875 0.00837459 0.0123282 2023-06-10T21:15:27.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[186706]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13494/333168 objects degraded (4.050%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:27.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:27.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:27.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[180297]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13494/333168 objects degraded (4.050%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:27.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:27.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:26 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:26 smithi152 ceph-mon[156819]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13494/333168 objects degraded (4.050%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:26 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:27.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:26 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:27.272 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 113717 113701 80.6867 46.4375 0.0193569 0.0123878 2023-06-10T21:15:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1506147876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:28.272 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 114521 114505 80.3449 50.25 0.0175322 0.0124408 2023-06-10T21:15:28.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1506147876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:28.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1506147876' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:29.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:28 smithi152 ceph-mon[156819]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 676 op/s; 13494/333168 objects degraded (4.050%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:29.272 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0113 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 115263 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 80.0337 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 38.8987 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 199 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-06-10T21:15:29.273 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1280 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 622.388 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3184 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0124902 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0292052 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.21601 2023-06-10T21:15:29.274 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.0013329 2023-06-10T21:15:29.279 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:15:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:28 smithi064 ceph-mon[186706]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 676 op/s; 13494/333168 objects degraded (4.050%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:29.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:28 smithi064 ceph-mon[180297]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 676 op/s; 13494/333168 objects degraded (4.050%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4290364278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4290364278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4290364278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:30 smithi152 ceph-mon[156819]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 13377/341151 objects degraded (3.921%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:31.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13377/341151 objects degraded (3.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:31.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:30 smithi064 ceph-mon[186706]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 13377/341151 objects degraded (3.921%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:31.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13377/341151 objects degraded (3.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:30 smithi064 ceph-mon[180297]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s; 13377/341151 objects degraded (3.921%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:31.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13377/341151 objects degraded (3.921%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:33.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:32 smithi064 ceph-mon[180297]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 13377/344415 objects degraded (3.884%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2988814204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:33.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:32] "GET /metrics HTTP/1.1" 200 212823 "" "Prometheus/2.33.4" 2023-06-10T21:15:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:32 smithi064 ceph-mon[186706]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 13377/344415 objects degraded (3.884%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2988814204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:32 smithi152 ceph-mon[156819]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 13377/344415 objects degraded (3.884%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2988814204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:34.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:34 smithi064 ceph-mon[180297]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 745 op/s; 13377/343698 objects degraded (3.892%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4050929376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:34 smithi064 ceph-mon[186706]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 745 op/s; 13377/343698 objects degraded (3.892%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:35.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4050929376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:34 smithi152 ceph-mon[156819]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 43 MiB/s wr, 745 op/s; 13377/343698 objects degraded (3.892%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4050929376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:36.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13377/343698 objects degraded (3.892%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:36.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13377/343698 objects degraded (3.892%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13377/343698 objects degraded (3.892%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:37.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:37 smithi064 ceph-mon[186706]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 914 op/s; 13261/334716 objects degraded (3.962%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:37.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:37 smithi064 ceph-mon[180297]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 914 op/s; 13261/334716 objects degraded (3.962%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:37.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:37 smithi152 ceph-mon[156819]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 914 op/s; 13261/334716 objects degraded (3.962%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:38.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2987766610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:38.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2987766610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:38.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2987766610' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:39.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:39 smithi064 ceph-mon[186706]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 662 op/s; 13261/334716 objects degraded (3.962%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:39.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:39 smithi064 ceph-mon[180297]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 662 op/s; 13261/334716 objects degraded (3.962%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:39 smithi152 ceph-mon[156819]: pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 662 op/s; 13261/334716 objects degraded (3.962%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/696348255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/696348255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/696348255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:41.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[186706]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 888 op/s; 13145/326547 objects degraded (4.025%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:41.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13145/326547 objects degraded (4.025%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:41.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:41.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[180297]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 888 op/s; 13145/326547 objects degraded (4.025%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:41.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13145/326547 objects degraded (4.025%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:41.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:41 smithi152 ceph-mon[156819]: pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 888 op/s; 13145/326547 objects degraded (4.025%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13145/326547 objects degraded (4.025%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:15:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:42.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[180297]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.2 MiB/s wr, 765 op/s; 13145/322995 objects degraded (4.070%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:42.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:42.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[186706]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.2 MiB/s wr, 765 op/s; 13145/322995 objects degraded (4.070%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:42.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:42 smithi152 ceph-mon[156819]: pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 8.2 MiB/s wr, 765 op/s; 13145/322995 objects degraded (4.070%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:43.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:42] "GET /metrics HTTP/1.1" 200 212840 "" "Prometheus/2.33.4" 2023-06-10T21:15:43.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2883973441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2883973441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:43.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2883973441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:44.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:44 smithi064 ceph-mon[186706]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.1 MiB/s wr, 737 op/s; 13145/320274 objects degraded (4.104%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:44.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:44 smithi064 ceph-mon[180297]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.1 MiB/s wr, 737 op/s; 13145/320274 objects degraded (4.104%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:44 smithi152 ceph-mon[156819]: pgmap v338: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 2.1 MiB/s wr, 737 op/s; 13145/320274 objects degraded (4.104%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:45.344 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/668263348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:45.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/668263348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:45.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/668263348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[186706]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 75 KiB/s wr, 865 op/s; 13028/312630 objects degraded (4.167%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13028/312630 objects degraded (4.167%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[180297]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 75 KiB/s wr, 865 op/s; 13028/312630 objects degraded (4.167%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13028/312630 objects degraded (4.167%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:46 smithi152 ceph-mon[156819]: pgmap v339: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 75 KiB/s wr, 865 op/s; 13028/312630 objects degraded (4.167%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13028/312630 objects degraded (4.167%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:15:47.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:15:47.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.916 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:47 smithi064 conmon[171866]: 2023-06-10T21:15:47.754+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:47.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:15:47.916 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:15:47.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:15:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:15:47.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:15:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13028/312630 objects degraded (4.167%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1671071302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13028/312630 objects degraded (4.167%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1671071302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:48.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 13028/312630 objects degraded (4.167%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1671071302' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:15:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:15:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:15:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:15:50.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[180297]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12913/305598 objects degraded (4.225%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:15:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2538782005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12913/305598 objects degraded (4.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:50.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[186706]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12913/305598 objects degraded (4.225%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:15:50.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2538782005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:50.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12913/305598 objects degraded (4.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:50 smithi152 ceph-mon[156819]: pgmap v341: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12913/305598 objects degraded (4.225%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:15:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2538782005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12913/305598 objects degraded (4.225%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:52 smithi152 ceph-mon[156819]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 12913/302190 objects degraded (4.273%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:52 smithi064 ceph-mon[180297]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 12913/302190 objects degraded (4.273%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:53.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:15:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:15:52] "GET /metrics HTTP/1.1" 200 212836 "" "Prometheus/2.33.4" 2023-06-10T21:15:53.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:52 smithi064 ceph-mon[186706]: pgmap v342: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 12913/302190 objects degraded (4.273%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:53.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4101040884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:54.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4101040884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:54.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4101040884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:54 smithi152 ceph-mon[156819]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 12913/299316 objects degraded (4.314%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:55.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:54 smithi064 ceph-mon[186706]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 12913/299316 objects degraded (4.314%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:54 smithi064 ceph-mon[180297]: pgmap v343: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 12913/299316 objects degraded (4.314%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:56.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/39276370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:56.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12913/299316 objects degraded (4.314%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/39276370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12913/299316 objects degraded (4.314%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:56.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/39276370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:56.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12913/299316 objects degraded (4.314%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:15:57.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:56 smithi064 ceph-mon[180297]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12796/291072 objects degraded (4.396%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:56 smithi064 ceph-mon[186706]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12796/291072 objects degraded (4.396%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:56 smithi152 ceph-mon[156819]: pgmap v344: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 12796/291072 objects degraded (4.396%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:15:59.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:58 smithi064 ceph-mon[186706]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12796/291072 objects degraded (4.396%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:59.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:15:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1225213931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:59.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:58 smithi064 ceph-mon[180297]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12796/291072 objects degraded (4.396%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:59.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:15:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1225213931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:15:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:58 smithi152 ceph-mon[156819]: pgmap v345: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12796/291072 objects degraded (4.396%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:15:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:15:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1225213931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[180297]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12679/283545 objects degraded (4.472%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:16:01.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12679/283545 objects degraded (4.472%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:01.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2778345325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:01.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[186706]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12679/283545 objects degraded (4.472%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:16:01.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12679/283545 objects degraded (4.472%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:01.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2778345325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:01.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:00 smithi152 ceph-mon[156819]: pgmap v346: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 12679/283545 objects degraded (4.472%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:16:01.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12679/283545 objects degraded (4.472%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:01.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2778345325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:03.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:02 smithi064 ceph-mon[180297]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12679/280128 objects degraded (4.526%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:02 smithi064 conmon[171866]: 2023-06-10T21:16:02.775+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:02] "GET /metrics HTTP/1.1" 200 212836 "" "Prometheus/2.33.4" 2023-06-10T21:16:03.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:02 smithi064 ceph-mon[186706]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12679/280128 objects degraded (4.526%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:02 smithi152 ceph-mon[156819]: pgmap v347: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 12679/280128 objects degraded (4.526%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:04.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:04.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/143341156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:04.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:04.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:04.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/143341156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:04.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:03 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:03 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/143341156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:04 smithi064 ceph-mon[186706]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12679/277326 objects degraded (4.572%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:05.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:04 smithi064 ceph-mon[180297]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12679/277326 objects degraded (4.572%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:04 smithi152 ceph-mon[156819]: pgmap v348: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12679/277326 objects degraded (4.572%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:06.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12679/277326 objects degraded (4.572%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:06.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1682734201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:06.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12679/277326 objects degraded (4.572%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:06.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1682734201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:06.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12679/277326 objects degraded (4.572%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:06.204 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1682734201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:06 smithi064 ceph-mon[186706]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12566/269502 objects degraded (4.663%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:07.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:06 smithi064 ceph-mon[180297]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12566/269502 objects degraded (4.663%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:07.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:06 smithi152 ceph-mon[156819]: pgmap v349: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 12566/269502 objects degraded (4.663%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:09.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:08 smithi064 ceph-mon[186706]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12566/269502 objects degraded (4.663%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:09.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2281510497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:09.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:08 smithi064 ceph-mon[180297]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12566/269502 objects degraded (4.663%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:09.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2281510497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:08 smithi152 ceph-mon[156819]: pgmap v350: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 12566/269502 objects degraded (4.663%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2281510497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[180297]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12450/262233 objects degraded (4.748%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12450/262233 objects degraded (4.748%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:11.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3816254764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[186706]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12450/262233 objects degraded (4.748%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12450/262233 objects degraded (4.748%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:11.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3816254764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:10 smithi152 ceph-mon[156819]: pgmap v351: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 12450/262233 objects degraded (4.748%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12450/262233 objects degraded (4.748%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3816254764' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:12 smithi064 ceph-mon[180297]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 12450/258825 objects degraded (4.810%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:13.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:12] "GET /metrics HTTP/1.1" 200 212841 "" "Prometheus/2.33.4" 2023-06-10T21:16:13.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:12 smithi064 ceph-mon[186706]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 12450/258825 objects degraded (4.810%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:13.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:12 smithi152 ceph-mon[156819]: pgmap v352: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 12450/258825 objects degraded (4.810%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/990572787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/990572787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/990572787' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:15.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[186706]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12450/256284 objects degraded (4.858%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[186706]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[186706]: osdmap e108: 8 total, 8 up, 8 in 2023-06-10T21:16:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[186706]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[186706]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[180297]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12450/256284 objects degraded (4.858%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:15.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[180297]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[180297]: osdmap e108: 8 total, 8 up, 8 in 2023-06-10T21:16:15.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[180297]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:14 smithi064 ceph-mon[180297]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:14 smithi152 ceph-mon[156819]: pgmap v353: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 12450/256284 objects degraded (4.858%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:14 smithi152 ceph-mon[156819]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:14 smithi152 ceph-mon[156819]: osdmap e108: 8 total, 8 up, 8 in 2023-06-10T21:16:15.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:14 smithi152 ceph-mon[156819]: failed to encode map e108 with expected crc 2023-06-10T21:16:15.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:14 smithi152 ceph-mon[156819]: failed to encode map e108 with expected crc 2023-06-10T21:16:16.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: osdmap e109: 8 total, 8 up, 8 in 2023-06-10T21:16:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12450/256284 objects degraded (4.858%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2159067534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: osdmap e109: 8 total, 8 up, 8 in 2023-06-10T21:16:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12450/256284 objects degraded (4.858%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:16.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2159067534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: osdmap e109: 8 total, 8 up, 8 in 2023-06-10T21:16:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: failed to encode map e109 with expected crc 2023-06-10T21:16:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12450/256284 objects degraded (4.858%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:16.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2159067534' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:16 smithi064 ceph-mon[180297]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12417/249570 objects degraded (4.975%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:16:17.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:16 smithi064 ceph-mon[186706]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12417/249570 objects degraded (4.975%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:16:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:16 smithi152 ceph-mon[156819]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12417/249570 objects degraded (4.975%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:16:18.165 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:18.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:18.172 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:17 smithi064 conmon[171866]: 2023-06-10T21:16:17.803+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[180297]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 12323/248859 objects degraded (4.952%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1683706162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[186706]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 12323/248859 objects degraded (4.952%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1683706162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:19 smithi152 ceph-mon[156819]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 12323/248859 objects degraded (4.952%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:19 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:19 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1683706162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[180297]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12323/243303 objects degraded (5.065%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12323/243303 objects degraded (5.065%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2931313074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[186706]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12323/243303 objects degraded (5.065%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12323/243303 objects degraded (5.065%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2931313074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:21 smithi152 ceph-mon[156819]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12323/243303 objects degraded (5.065%); 1016 KiB/s, 15 objects/s recovering 2023-06-10T21:16:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12323/243303 objects degraded (5.065%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2931313074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:23.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:23 smithi064 ceph-mon[180297]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 12208/239790 objects degraded (5.091%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:16:23.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:22] "GET /metrics HTTP/1.1" 200 212899 "" "Prometheus/2.33.4" 2023-06-10T21:16:23.169 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:23 smithi064 ceph-mon[186706]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 12208/239790 objects degraded (5.091%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:16:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:23 smithi152 ceph-mon[156819]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 12208/239790 objects degraded (5.091%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:16:24.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3937017556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:24.418 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3937017556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:24.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3937017556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:25.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:25 smithi064 ceph-mon[186706]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12208/237294 objects degraded (5.145%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:16:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:25 smithi064 ceph-mon[180297]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12208/237294 objects degraded (5.145%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:16:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:25 smithi152 ceph-mon[156819]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12208/237294 objects degraded (5.145%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:16:26.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12208/237294 objects degraded (5.145%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:26.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12208/237294 objects degraded (5.145%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12208/237294 objects degraded (5.145%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:27.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:27 smithi064 ceph-mon[186706]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12093/230493 objects degraded (5.247%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:16:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3825263942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:27.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:27 smithi064 ceph-mon[180297]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12093/230493 objects degraded (5.247%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:16:27.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3825263942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:27 smithi152 ceph-mon[156819]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 12093/230493 objects degraded (5.247%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:16:27.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3825263942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:29.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:29 smithi064 ceph-mon[180297]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12093/229911 objects degraded (5.260%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:16:29.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3460491411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:29.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:29 smithi064 ceph-mon[186706]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12093/229911 objects degraded (5.260%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:16:29.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3460491411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:29 smithi152 ceph-mon[156819]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12093/229911 objects degraded (5.260%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:16:29.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3460491411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:31.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:31 smithi064 ceph-mon[186706]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 12093/223776 objects degraded (5.404%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12093/223776 objects degraded (5.404%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:31.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:31 smithi064 ceph-mon[180297]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 12093/223776 objects degraded (5.404%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:31.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12093/223776 objects degraded (5.404%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:31 smithi152 ceph-mon[156819]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 12093/223776 objects degraded (5.404%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12093/223776 objects degraded (5.404%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:32.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/425343345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:32.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/425343345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/425343345' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:33.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[180297]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11978/220008 objects degraded (5.444%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:33.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:33.141 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:33.141 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:32 smithi064 conmon[171866]: 2023-06-10T21:16:32.826+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:33.142 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:32] "GET /metrics HTTP/1.1" 200 212899 "" "Prometheus/2.33.4" 2023-06-10T21:16:33.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[186706]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11978/220008 objects degraded (5.444%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:33.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:33.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:33 smithi152 ceph-mon[156819]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11978/220008 objects degraded (5.444%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:34 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:34 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/468742858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:34.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:34.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:34.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/468742858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/468742858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:35 smithi152 ceph-mon[156819]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11978/217437 objects degraded (5.509%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:35 smithi064 ceph-mon[186706]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11978/217437 objects degraded (5.509%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:35 smithi064 ceph-mon[180297]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 11978/217437 objects degraded (5.509%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:36 smithi064 ceph-mon[186706]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11862/210606 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:36.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11978/217437 objects degraded (5.509%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:36 smithi064 ceph-mon[180297]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11862/210606 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:36.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11978/217437 objects degraded (5.509%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:36.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:36 smithi152 ceph-mon[156819]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11862/210606 objects degraded (5.632%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:36.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11978/217437 objects degraded (5.509%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:37.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/777181721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:37.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/777181721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/777181721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:38.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:38 smithi064 ceph-mon[186706]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11862/210048 objects degraded (5.647%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:38 smithi064 ceph-mon[180297]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11862/210048 objects degraded (5.647%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:38 smithi152 ceph-mon[156819]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 11862/210048 objects degraded (5.647%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:39.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4278734779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4278734779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4278734779' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:40 smithi064 ceph-mon[180297]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11862/203757 objects degraded (5.822%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:40.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:40 smithi064 ceph-mon[186706]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11862/203757 objects degraded (5.822%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:40 smithi152 ceph-mon[156819]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11862/203757 objects degraded (5.822%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:41.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11862/203757 objects degraded (5.822%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:41.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11862/203757 objects degraded (5.822%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:41.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11862/203757 objects degraded (5.822%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:42.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:42 smithi064 ceph-mon[186706]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11746/200016 objects degraded (5.873%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:42.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:42 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1097969105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:42.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:42 smithi064 ceph-mon[180297]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11746/200016 objects degraded (5.873%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:42.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:42 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1097969105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:42 smithi152 ceph-mon[156819]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 11746/200016 objects degraded (5.873%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:42 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1097969105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:42] "GET /metrics HTTP/1.1" 200 212921 "" "Prometheus/2.33.4" 2023-06-10T21:16:44.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:44 smithi064 ceph-mon[186706]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11746/197577 objects degraded (5.945%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:44.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2161767223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:44.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:44 smithi064 ceph-mon[180297]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11746/197577 objects degraded (5.945%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:44.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2161767223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:44 smithi152 ceph-mon[156819]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11746/197577 objects degraded (5.945%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2161767223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:45.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11632/190914 objects degraded (6.093%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11632/190914 objects degraded (6.093%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11632/190914 objects degraded (6.093%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:46.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:46 smithi064 ceph-mon[186706]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11632/190914 objects degraded (6.093%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:46.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:46 smithi064 ceph-mon[180297]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11632/190914 objects degraded (6.093%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:46 smithi152 ceph-mon[156819]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11632/190914 objects degraded (6.093%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:47.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1164041997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1164041997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:47.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1164041997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[186706]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 11632/190323 objects degraded (6.112%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:48.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:48.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[180297]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 11632/190323 objects degraded (6.112%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:48.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:48.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:48.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:48 smithi152 ceph-mon[156819]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 11632/190323 objects degraded (6.112%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:48.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:16:49.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3978799178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:49.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3978799178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:49.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3978799178' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:50.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:50 smithi064 ceph-mon[180297]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 11632/186027 objects degraded (6.253%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:50.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11632/186027 objects degraded (6.253%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:50.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:50 smithi064 ceph-mon[186706]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 11632/186027 objects degraded (6.253%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:50.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11632/186027 objects degraded (6.253%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:50.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:50 smithi152 ceph-mon[156819]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 11632/186027 objects degraded (6.253%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:50.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11632/186027 objects degraded (6.253%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[180297]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11516/180477 objects degraded (6.381%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1107892616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:52.884 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:16:52] "GET /metrics HTTP/1.1" 200 212953 "" "Prometheus/2.33.4" 2023-06-10T21:16:52.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[186706]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11516/180477 objects degraded (6.381%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:52.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1107892616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:52.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:52 smithi152 ceph-mon[156819]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 11516/180477 objects degraded (6.381%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:52.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1107892616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:52.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:52.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 11516/178281 objects degraded (6.459%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/366133802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:16:55.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:16:55.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:16:54 smithi064 conmon[171866]: 2023-06-10T21:16:54.968+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:55.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 11516/178281 objects degraded (6.459%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/366133802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:16:55.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 11516/178281 objects degraded (6.459%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:55.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.200 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:55.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/366133802' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:55.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:16:55.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:16:55.201 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:16:56.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11516/178281 objects degraded (6.459%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:56.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11516/178281 objects degraded (6.459%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:16:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:16:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:16:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:16:56.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11516/178281 objects degraded (6.459%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:16:57.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:56 smithi064 ceph-mon[180297]: pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11403/172062 objects degraded (6.627%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:57.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:56 smithi064 ceph-mon[186706]: pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11403/172062 objects degraded (6.627%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:56 smithi152 ceph-mon[156819]: pgmap v376: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 11403/172062 objects degraded (6.627%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:16:58.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2292432414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:58.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2292432414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2292432414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:16:59.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:58 smithi064 ceph-mon[186706]: pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11403/171480 objects degraded (6.650%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:59.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:58 smithi064 ceph-mon[180297]: pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11403/171480 objects degraded (6.650%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:16:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:58 smithi152 ceph-mon[156819]: pgmap v377: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 11403/171480 objects degraded (6.650%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:00.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:16:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/272686565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:16:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/272686565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:00.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:16:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/272686565' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:01.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:00 smithi064 ceph-mon[186706]: pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11403/165411 objects degraded (6.894%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:01.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11403/165411 objects degraded (6.894%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:00 smithi064 ceph-mon[180297]: pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11403/165411 objects degraded (6.894%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:01.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11403/165411 objects degraded (6.894%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:01.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:00 smithi152 ceph-mon[156819]: pgmap v378: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11403/165411 objects degraded (6.894%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:01.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11403/165411 objects degraded (6.894%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:02] "GET /metrics HTTP/1.1" 200 212953 "" "Prometheus/2.33.4" 2023-06-10T21:17:03.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:02 smithi064 ceph-mon[180297]: pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 11287/161658 objects degraded (6.982%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:03.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3876123228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:03.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:02 smithi064 ceph-mon[186706]: pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 11287/161658 objects degraded (6.982%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:03.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3876123228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:02 smithi152 ceph-mon[156819]: pgmap v379: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 11287/161658 objects degraded (6.982%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:03.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3876123228' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:04.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:04.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:05.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:04 smithi064 ceph-mon[180297]: pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11287/159189 objects degraded (7.090%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:05.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/408994221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:04 smithi064 ceph-mon[186706]: pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11287/159189 objects degraded (7.090%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/408994221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:04 smithi152 ceph-mon[156819]: pgmap v380: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 11287/159189 objects degraded (7.090%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:05.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/408994221' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:06.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11287/159189 objects degraded (7.090%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:06.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11287/159189 objects degraded (7.090%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:06.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11287/159189 objects degraded (7.090%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:07.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:06 smithi064 ceph-mon[186706]: pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11170/152103 objects degraded (7.344%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:06 smithi064 ceph-mon[180297]: pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11170/152103 objects degraded (7.344%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:06 smithi152 ceph-mon[156819]: pgmap v381: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 11170/152103 objects degraded (7.344%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:08.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/979108412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:08.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/979108412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:08.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/979108412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:09.198 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:08 smithi152 ceph-mon[156819]: pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 11170/151398 objects degraded (7.378%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:09.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:08 smithi064 ceph-mon[180297]: pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 11170/151398 objects degraded (7.378%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:09.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:08 smithi064 ceph-mon[186706]: pgmap v382: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 11170/151398 objects degraded (7.378%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:10.407 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:09 smithi064 conmon[171866]: 2023-06-10T21:17:09.985+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:11.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 11170/145185 objects degraded (7.694%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:11.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2702296394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:11.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:11.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11170/145185 objects degraded (7.694%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:11.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 11170/145185 objects degraded (7.694%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2702296394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11170/145185 objects degraded (7.694%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: pgmap v383: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 11170/145185 objects degraded (7.694%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2702296394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11170/145185 objects degraded (7.694%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:13.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:12 smithi064 ceph-mon[180297]: pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11054/141276 objects degraded (7.824%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1662441530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:12] "GET /metrics HTTP/1.1" 200 212959 "" "Prometheus/2.33.4" 2023-06-10T21:17:13.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:12 smithi064 ceph-mon[186706]: pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11054/141276 objects degraded (7.824%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1662441530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:13.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:12 smithi152 ceph-mon[156819]: pgmap v384: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 11054/141276 objects degraded (7.824%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:13.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1662441530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:15.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:14 smithi064 ceph-mon[180297]: pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 11054/138690 objects degraded (7.970%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:15.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:14 smithi064 ceph-mon[186706]: pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 11054/138690 objects degraded (7.970%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:14 smithi152 ceph-mon[156819]: pgmap v385: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 11054/138690 objects degraded (7.970%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:16.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/341001544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:16.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11054/138690 objects degraded (7.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/341001544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:16.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11054/138690 objects degraded (7.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/341001544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11054/138690 objects degraded (7.970%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:17.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:17 smithi064 ceph-mon[180297]: pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10937/132003 objects degraded (8.285%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:17.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:17 smithi064 ceph-mon[186706]: pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10937/132003 objects degraded (8.285%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:17 smithi152 ceph-mon[156819]: pgmap v386: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 10937/132003 objects degraded (8.285%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:17:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/166414804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/166414804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/166414804' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:19 smithi064 ceph-mon[186706]: pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10937/131433 objects degraded (8.321%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:19 smithi064 ceph-mon[180297]: pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10937/131433 objects degraded (8.321%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:19.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:19 smithi152 ceph-mon[156819]: pgmap v387: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 10937/131433 objects degraded (8.321%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:19.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:21.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[186706]: pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 10937/125097 objects degraded (8.743%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:21.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/939463062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:21.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10937/125097 objects degraded (8.743%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:21.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[180297]: pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 10937/125097 objects degraded (8.743%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:21.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/939463062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:21.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10937/125097 objects degraded (8.743%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:21 smithi152 ceph-mon[156819]: pgmap v388: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 10937/125097 objects degraded (8.743%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/939463062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10937/125097 objects degraded (8.743%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:23.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:23 smithi064 ceph-mon[180297]: pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10824/121464 objects degraded (8.911%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:23.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/75758863' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:23.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:23 smithi064 ceph-mon[186706]: pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10824/121464 objects degraded (8.911%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:23.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/75758863' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:23.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:22] "GET /metrics HTTP/1.1" 200 212954 "" "Prometheus/2.33.4" 2023-06-10T21:17:23.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:23 smithi152 ceph-mon[156819]: pgmap v389: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10824/121464 objects degraded (8.911%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:17:23.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/75758863' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:25 smithi064 ceph-mon[180297]: pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10824/118947 objects degraded (9.100%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:25.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:25 smithi064 ceph-mon[186706]: pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10824/118947 objects degraded (9.100%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:25.414 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:25 smithi064 conmon[171866]: 2023-06-10T21:17:25.005+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:25 smithi152 ceph-mon[156819]: pgmap v390: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10824/118947 objects degraded (9.100%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:17:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:26.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:26.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:26.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4105462810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:26.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10824/118947 objects degraded (9.100%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:26.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:26.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:26.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4105462810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:26.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10824/118947 objects degraded (9.100%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:26 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:26 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4105462810' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:26.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10824/118947 objects degraded (9.100%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:27.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:27 smithi064 ceph-mon[180297]: pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10709/112101 objects degraded (9.553%); 1.2 MiB/s, 28 objects/s recovering 2023-06-10T21:17:27.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:27 smithi064 ceph-mon[186706]: pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10709/112101 objects degraded (9.553%); 1.2 MiB/s, 28 objects/s recovering 2023-06-10T21:17:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:27 smithi152 ceph-mon[156819]: pgmap v391: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10709/112101 objects degraded (9.553%); 1.2 MiB/s, 28 objects/s recovering 2023-06-10T21:17:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3985633917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:28.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3985633917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:28.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3985633917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:29 smithi152 ceph-mon[156819]: pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 10709/111474 objects degraded (9.607%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:29.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:29 smithi064 ceph-mon[180297]: pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 10709/111474 objects degraded (9.607%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:29 smithi064 ceph-mon[186706]: pgmap v392: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 570 op/s; 10709/111474 objects degraded (9.607%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:31 smithi152 ceph-mon[156819]: pgmap v393: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 10709/107016 objects degraded (10.007%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10709/107016 objects degraded (10.007%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1303453696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[180297]: pgmap v393: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 10709/107016 objects degraded (10.007%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10709/107016 objects degraded (10.007%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1303453696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[186706]: pgmap v393: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 10709/107016 objects degraded (10.007%); 586 KiB/s, 18 objects/s recovering 2023-06-10T21:17:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10709/107016 objects degraded (10.007%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1303453696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:32] "GET /metrics HTTP/1.1" 200 212954 "" "Prometheus/2.33.4" 2023-06-10T21:17:33.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:33 smithi152 ceph-mon[156819]: pgmap v394: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10593/101385 objects degraded (10.448%); 586 KiB/s, 28 objects/s recovering 2023-06-10T21:17:33.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:33.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/992958061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[186706]: pgmap v394: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10593/101385 objects degraded (10.448%); 586 KiB/s, 28 objects/s recovering 2023-06-10T21:17:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/992958061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[180297]: pgmap v394: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10593/101385 objects degraded (10.448%); 586 KiB/s, 28 objects/s recovering 2023-06-10T21:17:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:33.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/992958061' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:35 smithi064 ceph-mon[186706]: pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10593/98784 objects degraded (10.723%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:35 smithi064 ceph-mon[180297]: pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10593/98784 objects degraded (10.723%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:35.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:35 smithi152 ceph-mon[156819]: pgmap v395: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10593/98784 objects degraded (10.723%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10593/98784 objects degraded (10.723%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1373669251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10593/98784 objects degraded (10.723%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1373669251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10593/98784 objects degraded (10.723%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1373669251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:37.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:37 smithi064 ceph-mon[186706]: pgmap v396: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 10477/92004 objects degraded (11.388%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:37.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:37 smithi064 ceph-mon[180297]: pgmap v396: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 10477/92004 objects degraded (11.388%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:37.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:37 smithi152 ceph-mon[156819]: pgmap v396: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 10477/92004 objects degraded (11.388%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:38.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:38 smithi064 ceph-mon[180297]: pgmap v397: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 10477/91338 objects degraded (11.471%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:38.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:38 smithi064 ceph-mon[186706]: pgmap v397: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 10477/91338 objects degraded (11.471%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:38 smithi152 ceph-mon[156819]: pgmap v397: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 10477/91338 objects degraded (11.471%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1349630414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:39.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1349630414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1349630414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[180297]: pgmap v398: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 10477/86967 objects degraded (12.047%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[186706]: pgmap v398: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 10477/86967 objects degraded (12.047%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:40 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:40.412 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:40 smithi064 conmon[171866]: 2023-06-10T21:17:40.028+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:40 smithi152 ceph-mon[156819]: pgmap v398: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 10477/86967 objects degraded (12.047%); 0 B/s, 19 objects/s recovering 2023-06-10T21:17:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:40 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:17:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:40 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:17:41.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10477/86967 objects degraded (12.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/513225036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:41.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10477/86967 objects degraded (12.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:41.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/513225036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10477/86967 objects degraded (12.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/513225036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:42 smithi064 ceph-mon[180297]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10361/81153 objects degraded (12.767%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:42 smithi064 ceph-mon[186706]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10361/81153 objects degraded (12.767%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:42 smithi152 ceph-mon[156819]: pgmap v399: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10361/81153 objects degraded (12.767%); 0 B/s, 28 objects/s recovering 2023-06-10T21:17:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:42] "GET /metrics HTTP/1.1" 200 212876 "" "Prometheus/2.33.4" 2023-06-10T21:17:43.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[180297]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[180297]: osdmap e110: 8 total, 8 up, 8 in 2023-06-10T21:17:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[180297]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[180297]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2453125841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[186706]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[186706]: osdmap e110: 8 total, 8 up, 8 in 2023-06-10T21:17:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[186706]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[186706]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2453125841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:43 smithi152 ceph-mon[156819]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:43 smithi152 ceph-mon[156819]: osdmap e110: 8 total, 8 up, 8 in 2023-06-10T21:17:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:43 smithi152 ceph-mon[156819]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:43 smithi152 ceph-mon[156819]: failed to encode map e110 with expected crc 2023-06-10T21:17:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2453125841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:44.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: osdmap e111: 8 total, 8 up, 8 in 2023-06-10T21:17:44.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 10329/78603 objects degraded (13.141%); 0 B/s, 18 objects/s recovering 2023-06-10T21:17:44.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[186706]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: osdmap e111: 8 total, 8 up, 8 in 2023-06-10T21:17:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 10329/78603 objects degraded (13.141%); 0 B/s, 18 objects/s recovering 2023-06-10T21:17:44.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:44 smithi064 ceph-mon[180297]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: osdmap e111: 8 total, 8 up, 8 in 2023-06-10T21:17:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: failed to encode map e111 with expected crc 2023-06-10T21:17:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 10329/78603 objects degraded (13.141%); 0 B/s, 18 objects/s recovering 2023-06-10T21:17:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:44 smithi152 ceph-mon[156819]: failed to encode map e111 with expected crc 2023-06-10T21:17:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10284/72606 objects degraded (14.164%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:46.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10284/72606 objects degraded (14.164%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:46.157 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10284/72606 objects degraded (14.164%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:46 smithi064 ceph-mon[180297]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10284/72606 objects degraded (14.164%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:46.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3207243672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:46.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:46 smithi064 ceph-mon[186706]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10284/72606 objects degraded (14.164%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:46.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3207243672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:46.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:46 smithi152 ceph-mon[156819]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10284/72606 objects degraded (14.164%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:46.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3207243672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[180297]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10284/72096 objects degraded (14.264%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1299014552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:48.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[186706]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10284/72096 objects degraded (14.264%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:48.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:48.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1299014552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:48 smithi152 ceph-mon[156819]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 10284/72096 objects degraded (14.264%); 0 B/s, 24 objects/s recovering 2023-06-10T21:17:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:17:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1299014552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:50.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:50 smithi064 ceph-mon[180297]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9974/68172 objects degraded (14.631%); 0 B/s, 48 objects/s recovering 2023-06-10T21:17:50.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9974/68172 objects degraded (14.631%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:50.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:50 smithi064 ceph-mon[186706]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9974/68172 objects degraded (14.631%); 0 B/s, 48 objects/s recovering 2023-06-10T21:17:50.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9974/68172 objects degraded (14.631%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:50 smithi152 ceph-mon[156819]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9974/68172 objects degraded (14.631%); 0 B/s, 48 objects/s recovering 2023-06-10T21:17:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9974/68172 objects degraded (14.631%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:51.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/96070322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:51.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/96070322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:51.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/96070322' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:52.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:52 smithi064 ceph-mon[180297]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9974/62514 objects degraded (15.955%); 0 B/s, 42 objects/s recovering 2023-06-10T21:17:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:52 smithi064 ceph-mon[186706]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9974/62514 objects degraded (15.955%); 0 B/s, 42 objects/s recovering 2023-06-10T21:17:52.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:52 smithi152 ceph-mon[156819]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9974/62514 objects degraded (15.955%); 0 B/s, 42 objects/s recovering 2023-06-10T21:17:53.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:17:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:17:52] "GET /metrics HTTP/1.1" 200 212948 "" "Prometheus/2.33.4" 2023-06-10T21:17:53.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2370251075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:53.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2370251075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:53.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2370251075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:54.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:54 smithi064 ceph-mon[180297]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9753/59865 objects degraded (16.292%); 0 B/s, 57 objects/s recovering 2023-06-10T21:17:54.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:54 smithi064 ceph-mon[186706]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9753/59865 objects degraded (16.292%); 0 B/s, 57 objects/s recovering 2023-06-10T21:17:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:54 smithi152 ceph-mon[156819]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 9753/59865 objects degraded (16.292%); 0 B/s, 57 objects/s recovering 2023-06-10T21:17:55.928 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:55 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:17:55.929 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9753/59865 objects degraded (16.292%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:55.929 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:55 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:17:55.929 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9753/59865 objects degraded (16.292%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:55 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:17:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9753/59865 objects degraded (16.292%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:17:56.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:56 smithi064 ceph-mon[186706]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 9753/52956 objects degraded (18.417%); 0 B/s, 47 objects/s recovering 2023-06-10T21:17:56.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1044128927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:56 smithi064 ceph-mon[180297]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 9753/52956 objects degraded (18.417%); 0 B/s, 47 objects/s recovering 2023-06-10T21:17:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1044128927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:56 smithi152 ceph-mon[156819]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 9753/52956 objects degraded (18.417%); 0 B/s, 47 objects/s recovering 2023-06-10T21:17:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1044128927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:58.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:58 smithi064 ceph-mon[180297]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 9753/52311 objects degraded (18.644%); 0 B/s, 44 objects/s recovering 2023-06-10T21:17:58.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:58 smithi064 ceph-mon[186706]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 9753/52311 objects degraded (18.644%); 0 B/s, 44 objects/s recovering 2023-06-10T21:17:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:58 smithi152 ceph-mon[156819]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 9753/52311 objects degraded (18.644%); 0 B/s, 44 objects/s recovering 2023-06-10T21:17:59.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:17:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3582713129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:17:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:17:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3582713129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:00.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:17:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3582713129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[186706]: pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1002 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9531/48129 objects degraded (19.803%); 0 B/s, 62 objects/s recovering 2023-06-10T21:18:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9531/48129 objects degraded (19.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:01.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[180297]: pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1002 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9531/48129 objects degraded (19.803%); 0 B/s, 62 objects/s recovering 2023-06-10T21:18:01.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9531/48129 objects degraded (19.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:00 smithi152 ceph-mon[156819]: pgmap v410: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1002 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9531/48129 objects degraded (19.803%); 0 B/s, 62 objects/s recovering 2023-06-10T21:18:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9531/48129 objects degraded (19.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:02.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/695380886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:02.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/695380886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:02.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/695380886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:03.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:02 smithi152 ceph-mon[156819]: pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 896 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 9531/43041 objects degraded (22.144%); 0 B/s, 36 objects/s recovering 2023-06-10T21:18:03.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:03.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:02 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:03.249 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:02] "GET /metrics HTTP/1.1" 200 212948 "" "Prometheus/2.33.4" 2023-06-10T21:18:03.250 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:03 smithi064 conmon[171866]: 2023-06-10T21:18:03.245+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:03.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[186706]: pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 896 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 9531/43041 objects degraded (22.144%); 0 B/s, 36 objects/s recovering 2023-06-10T21:18:03.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:03.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:03.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[180297]: pgmap v411: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 896 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 9531/43041 objects degraded (22.144%); 0 B/s, 36 objects/s recovering 2023-06-10T21:18:03.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:03.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:02 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:04.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:04 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:04 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:04 smithi152 ceph-mon[156819]: pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 848 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9305/40755 objects degraded (22.832%); 0 B/s, 55 objects/s recovering 2023-06-10T21:18:05.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2444018773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:05.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:05.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:05.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[180297]: pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 848 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9305/40755 objects degraded (22.832%); 0 B/s, 55 objects/s recovering 2023-06-10T21:18:05.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2444018773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:05.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:05.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:05.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[186706]: pgmap v412: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 848 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 9305/40755 objects degraded (22.832%); 0 B/s, 55 objects/s recovering 2023-06-10T21:18:05.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2444018773' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:06.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9305/40755 objects degraded (22.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9305/40755 objects degraded (22.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9305/40755 objects degraded (22.832%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:07.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:06 smithi064 ceph-mon[180297]: pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 710 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9305/34116 objects degraded (27.275%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:07.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2493883648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:07.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:06 smithi064 ceph-mon[186706]: pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 710 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9305/34116 objects degraded (27.275%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:07.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2493883648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:07.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:06 smithi152 ceph-mon[156819]: pgmap v413: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 710 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 9305/34116 objects degraded (27.275%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:07.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2493883648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:08 smithi064 ceph-mon[186706]: pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 698 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 9305/33531 objects degraded (27.750%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:09.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:08 smithi064 ceph-mon[180297]: pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 698 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 9305/33531 objects degraded (27.750%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:08 smithi152 ceph-mon[156819]: pgmap v414: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 698 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 9305/33531 objects degraded (27.750%); 0 B/s, 37 objects/s recovering 2023-06-10T21:18:10.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2371163627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:10.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2371163627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:10.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2371163627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:11.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:11 smithi064 ceph-mon[186706]: pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 608 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9052/29232 objects degraded (30.966%); 0 B/s, 58 objects/s recovering 2023-06-10T21:18:11.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9052/29232 objects degraded (30.966%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:11 smithi064 ceph-mon[180297]: pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 608 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9052/29232 objects degraded (30.966%); 0 B/s, 58 objects/s recovering 2023-06-10T21:18:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9052/29232 objects degraded (30.966%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:11 smithi152 ceph-mon[156819]: pgmap v415: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 608 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 9052/29232 objects degraded (30.966%); 0 B/s, 58 objects/s recovering 2023-06-10T21:18:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9052/29232 objects degraded (30.966%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:12.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2580939866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2580939866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:12.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2580939866' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:13.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:13 smithi064 ceph-mon[186706]: pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 485 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9052/23337 objects degraded (38.788%); 0 B/s, 39 objects/s recovering 2023-06-10T21:18:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:12] "GET /metrics HTTP/1.1" 200 212948 "" "Prometheus/2.33.4" 2023-06-10T21:18:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:13 smithi064 ceph-mon[180297]: pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 485 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9052/23337 objects degraded (38.788%); 0 B/s, 39 objects/s recovering 2023-06-10T21:18:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:13 smithi152 ceph-mon[156819]: pgmap v416: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 485 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9052/23337 objects degraded (38.788%); 0 B/s, 39 objects/s recovering 2023-06-10T21:18:15.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:15 smithi064 ceph-mon[186706]: pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 434 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8793/20877 objects degraded (42.118%); 0 B/s, 61 objects/s recovering 2023-06-10T21:18:15.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3528431663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:15.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:15 smithi064 ceph-mon[180297]: pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 434 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8793/20877 objects degraded (42.118%); 0 B/s, 61 objects/s recovering 2023-06-10T21:18:15.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3528431663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:15 smithi152 ceph-mon[156819]: pgmap v417: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 434 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8793/20877 objects degraded (42.118%); 0 B/s, 61 objects/s recovering 2023-06-10T21:18:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3528431663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:16.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8793/20877 objects degraded (42.118%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:16.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8793/20877 objects degraded (42.118%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8793/20877 objects degraded (42.118%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:17.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:17 smithi064 ceph-mon[180297]: pgmap v418: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 288 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8793/13869 objects degraded (63.400%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:17.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1497902717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:17.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:17 smithi064 ceph-mon[186706]: pgmap v418: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 288 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8793/13869 objects degraded (63.400%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:17.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1497902717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:17 smithi152 ceph-mon[156819]: pgmap v418: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 288 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8793/13869 objects degraded (63.400%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1497902717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:18.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:18 smithi064 conmon[171866]: 2023-06-10T21:18:18.267+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[186706]: pgmap v419: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 274 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 8793/13215 objects degraded (66.538%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[180297]: pgmap v419: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 274 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 8793/13215 objects degraded (66.538%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:19 smithi152 ceph-mon[156819]: pgmap v419: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 274 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 8793/13215 objects degraded (66.538%); 0 B/s, 42 objects/s recovering 2023-06-10T21:18:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:20.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:20.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:20.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2103645850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:20.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:20.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:20.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2103645850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:20 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:20 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-10T21:18:20.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2103645850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:20.649 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 115263 objects 2023-06-10T21:18:20.650 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :171.371 2023-06-10T21:18:21.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:21 smithi064 ceph-mon[186706]: pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 182 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8517/8805 objects degraded (96.729%); 0 B/s, 65 objects/s recovering 2023-06-10T21:18:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8517/8805 objects degraded (96.729%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:21 smithi064 ceph-mon[180297]: pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 182 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8517/8805 objects degraded (96.729%); 0 B/s, 65 objects/s recovering 2023-06-10T21:18:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8517/8805 objects degraded (96.729%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:21 smithi152 ceph-mon[156819]: pgmap v420: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 182 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 8517/8805 objects degraded (96.729%); 0 B/s, 65 objects/s recovering 2023-06-10T21:18:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8517/8805 objects degraded (96.729%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:21.502 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-06-10T21:18:21.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-06-10T21:18:22.135 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:18:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: pgmap v421: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 62 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8517/3033 objects degraded (280.811%); 0 B/s, 44 objects/s recovering 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: osdmap e112: 8 total, 8 up, 8 in 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2694142566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: pgmap v421: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 62 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8517/3033 objects degraded (280.811%); 0 B/s, 44 objects/s recovering 2023-06-10T21:18:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: osdmap e112: 8 total, 8 up, 8 in 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2694142566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: pgmap v421: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 62 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 8517/3033 objects degraded (280.811%); 0 B/s, 44 objects/s recovering 2023-06-10T21:18:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: osdmap e112: 8 total, 8 up, 8 in 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: failed to encode map e112 with expected crc 2023-06-10T21:18:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2694142566' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:22] "GET /metrics HTTP/1.1" 200 212934 "" "Prometheus/2.33.4" 2023-06-10T21:18:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[186706]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[186706]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[186706]: osdmap e113: 8 total, 8 up, 8 in 2023-06-10T21:18:23.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[186706]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[180297]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[180297]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[180297]: osdmap e113: 8 total, 8 up, 8 in 2023-06-10T21:18:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:23 smithi064 ceph-mon[180297]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:23 smithi152 ceph-mon[156819]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:23 smithi152 ceph-mon[156819]: failed to encode map e113 with expected crc 2023-06-10T21:18:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:23 smithi152 ceph-mon[156819]: osdmap e113: 8 total, 8 up, 8 in 2023-06-10T21:18:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:23 smithi152 ceph-mon[156819]: failed to encode map e113 with expected crc 2023-06-10T21:18:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:24 smithi064 ceph-mon[186706]: pgmap v424: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8517/1869 objects degraded (455.698%); 0 B/s, 34 objects/s recovering 2023-06-10T21:18:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/914604376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:24 smithi064 ceph-mon[180297]: pgmap v424: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8517/1869 objects degraded (455.698%); 0 B/s, 34 objects/s recovering 2023-06-10T21:18:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/914604376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:24 smithi152 ceph-mon[156819]: pgmap v424: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 38 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 8517/1869 objects degraded (455.698%); 0 B/s, 34 objects/s recovering 2023-06-10T21:18:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/914604376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:25.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8517/1869 objects degraded (455.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:25.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8517/1869 objects degraded (455.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8517/1869 objects degraded (455.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:26 smithi064 ceph-mon[186706]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 7468/405 objects degraded (1843.951%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:26.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:26 smithi064 ceph-mon[180297]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 7468/405 objects degraded (1843.951%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:26 smithi152 ceph-mon[156819]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 7.5 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 7468/405 objects degraded (1843.951%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:27.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1668952313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:27.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1668952313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:27.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1668952313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:28 smithi064 ceph-mon[186706]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:28 smithi064 ceph-mon[180297]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:28 smithi152 ceph-mon[156819]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 165 objects/s recovering 2023-06-10T21:18:29.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: osdmap e114: 8 total, 8 up, 8 in 2023-06-10T21:18:29.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/168319842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: osdmap e114: 8 total, 8 up, 8 in 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/168319842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: osdmap e114: 8 total, 8 up, 8 in 2023-06-10T21:18:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: failed to encode map e114 with expected crc 2023-06-10T21:18:29.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/168319842' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:30.476 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_3' removed 2023-06-10T21:18:30.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 167 objects/s recovering 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: osdmap e115: 8 total, 8 up, 8 in 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1772874362' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[180297]: osdmap e116: 8 total, 8 up, 8 in 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 167 objects/s recovering 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: osdmap e115: 8 total, 8 up, 8 in 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1772874362' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:18:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:30 smithi064 ceph-mon[186706]: osdmap e116: 8 total, 8 up, 8 in 2023-06-10T21:18:30.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 7192/72 objects degraded (9988.889%); 0 B/s, 167 objects/s recovering 2023-06-10T21:18:30.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: osdmap e115: 8 total, 8 up, 8 in 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1772874362' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:18:30.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-10T21:18:30.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:18:30.952 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:30 smithi152 ceph-mon[156819]: osdmap e116: 8 total, 8 up, 8 in 2023-06-10T21:18:31.395 INFO:teuthology.task.sequential:In sequential, running task print... 2023-06-10T21:18:31.395 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-06-10T21:18:31.395 DEBUG:teuthology.parallel:result is None 2023-06-10T21:18:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e115 with expected crc 2023-06-10T21:18:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3136842303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e115 with expected crc 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3136842303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e115 with expected crc 2023-06-10T21:18:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3136842303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[186706]: failed to encode map e116 with expected crc 2023-06-10T21:18:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[186706]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:18:32.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:18:32.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[180297]: failed to encode map e116 with expected crc 2023-06-10T21:18:32.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[180297]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:32.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:18:32.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:32 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:18:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:32 smithi152 ceph-mon[156819]: failed to encode map e116 with expected crc 2023-06-10T21:18:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:32 smithi152 ceph-mon[156819]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:32 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7192/72 objects degraded (9988.889%), 4 pgs degraded, 4 pgs undersized) 2023-06-10T21:18:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:32 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:18:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:32] "GET /metrics HTTP/1.1" 200 212934 "" "Prometheus/2.33.4" 2023-06-10T21:18:34.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:34.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:34.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:34.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3789744757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:34.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3789744757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3789744757' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: Upgrade: osd.2 is safe to restart 2023-06-10T21:18:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:18:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: Upgrade: osd.2 is safe to restart 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:18:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-10T21:18:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: Upgrade: osd.2 is safe to restart 2023-06-10T21:18:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-10T21:18:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:36.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[180297]: Upgrade: Updating osd.2 2023-06-10T21:18:36.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[180297]: Deploying daemon osd.2 on smithi064 2023-06-10T21:18:36.127 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/210382667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[186706]: Upgrade: Updating osd.2 2023-06-10T21:18:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[186706]: Deploying daemon osd.2 on smithi064 2023-06-10T21:18:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/210382667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:36 smithi152 ceph-mon[156819]: Upgrade: Updating osd.2 2023-06-10T21:18:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:36 smithi152 ceph-mon[156819]: Deploying daemon osd.2 on smithi064 2023-06-10T21:18:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/210382667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:37.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:37 smithi064 ceph-mon[186706]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:37.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:37 smithi064 ceph-mon[180297]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:37 smithi152 ceph-mon[156819]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:38.657 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:38 smithi064 systemd[1]: Stopping Ceph osd.2 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:18:39.131 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:38 smithi064 conmon[123976]: debug 2023-06-10T21:18:38.706+0000 7fe5630eb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:18:39.131 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:38 smithi064 conmon[123976]: debug 2023-06-10T21:18:38.706+0000 7fe5630eb700 -1 osd.2 116 *** Got signal Terminated *** 2023-06-10T21:18:39.131 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:38 smithi064 conmon[123976]: debug 2023-06-10T21:18:38.706+0000 7fe5630eb700 -1 osd.2 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:18:39.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[180297]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:39.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/141503035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:39.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:39.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[180297]: osd.2 marked itself down and dead 2023-06-10T21:18:39.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[186706]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/141503035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:39 smithi064 ceph-mon[186706]: osd.2 marked itself down and dead 2023-06-10T21:18:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:39 smithi152 ceph-mon[156819]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/141503035' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:39 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:39 smithi152 ceph-mon[156819]: osd.2 marked itself down and dead 2023-06-10T21:18:39.908 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:39 smithi064 bash[221421]: 8cca00e2828d57cb04628d8345f44ce8838c0cc251b1ae0e4410edfb67070e64 2023-06-10T21:18:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:18:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[180297]: failed to encode map e117 with expected crc 2023-06-10T21:18:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[180297]: osdmap e117: 8 total, 7 up, 8 in 2023-06-10T21:18:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:18:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[186706]: failed to encode map e117 with expected crc 2023-06-10T21:18:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:40 smithi064 ceph-mon[186706]: osdmap e117: 8 total, 7 up, 8 in 2023-06-10T21:18:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:40 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:18:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:40 smithi152 ceph-mon[156819]: failed to encode map e117 with expected crc 2023-06-10T21:18:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:40 smithi152 ceph-mon[156819]: osdmap e117: 8 total, 7 up, 8 in 2023-06-10T21:18:41.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: failed to encode map e117 with expected crc 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: pgmap v436: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: osdmap e118: 8 total, 7 up, 8 in 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3828839551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: failed to encode map e117 with expected crc 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: pgmap v436: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: osdmap e118: 8 total, 7 up, 8 in 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3828839551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: failed to encode map e117 with expected crc 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: pgmap v436: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: osdmap e118: 8 total, 7 up, 8 in 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3828839551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:42.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[186706]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.408 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:42 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.2.service: Succeeded. 2023-06-10T21:18:42.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:42 smithi064 systemd[1]: Stopped Ceph osd.2 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:18:42.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:42 smithi064 systemd[1]: Starting Ceph osd.2 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:18:42.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:42 smithi064 ceph-mon[180297]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:42 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:42 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:42 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:42.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:42 smithi152 ceph-mon[156819]: failed to encode map e118 with expected crc 2023-06-10T21:18:43.144 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[186706]: pgmap v438: 73 pgs: 7 peering, 4 active+undersized, 62 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:43.144 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:18:43.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:42] "GET /metrics HTTP/1.1" 200 210590 "" "Prometheus/2.33.4" 2023-06-10T21:18:43.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2792065567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:43.408 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-06-10T21:18:43.408 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-06-10T21:18:43.408 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-06-10T21:18:43.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-06-10T21:18:43.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2023-06-10T21:18:43.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-06-10T21:18:43.409 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:43 smithi064 bash[222093]: --> ceph-volume raw activate successful for osd ID: 2 2023-06-10T21:18:43.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[180297]: pgmap v438: 73 pgs: 7 peering, 4 active+undersized, 62 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:43.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:18:43.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2792065567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:43.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:43 smithi152 ceph-mon[156819]: pgmap v438: 73 pgs: 7 peering, 4 active+undersized, 62 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:18:43.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:43 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:18:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2792065567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:44.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:44 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:18:44.460 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:44 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:18:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:44 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:18:44.907 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:44 smithi064 bash[222562]: c76c1a3f425a8093aec1d9e92b5a22b4588c9706274e7349fc784248f1715a86 2023-06-10T21:18:44.908 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:44 smithi064 systemd[1]: Started Ceph osd.2 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:18:44.908 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:44 smithi064 ceph-osd[222627]: -- 172.21.15.64:0/3737836677 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564c269484e0 con 0x564c2691b400 2023-06-10T21:18:45.407 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:44 smithi064 conmon[222603]: 2023-06-10T21:18:44.984+0000 7f425ad95640 -1 Falling back to public interface 2023-06-10T21:18:45.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[186706]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 7 peering, 5 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:18:45.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[180297]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 7 peering, 5 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:18:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:45 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:45 smithi152 ceph-mon[156819]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 7 peering, 5 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:18:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:45 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:45 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:45 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3914862177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3914862177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3914862177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:47.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:47 smithi064 ceph-mon[180297]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:47.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:47 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:18:47.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:47 smithi064 ceph-mon[186706]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:47.469 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:47 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:18:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:47 smithi152 ceph-mon[156819]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:47 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:18:48.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/865197712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/865197712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:48.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:48.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:48.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/865197712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:48.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:50.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[180297]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:50.875 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3368143983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:50.876 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:18:50.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[186706]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:50.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3368143983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:50.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:50 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:18:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:50 smithi152 ceph-mon[156819]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3368143983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:50 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:18:51.742 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:51 smithi064 conmon[171866]: 2023-06-10T21:18:51.701+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-06-10T21:18:52.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-06-10T21:18:52.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/309624202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:52.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-06-10T21:18:52.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/309624202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:18:52.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:18:52.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-06-10T21:18:52.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/309624202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:18:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:18:52] "GET /metrics HTTP/1.1" 200 210579 "" "Prometheus/2.33.4" 2023-06-10T21:18:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:54 smithi064 ceph-mon[186706]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:54 smithi064 ceph-mon[180297]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:18:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:54 smithi152 ceph-mon[156819]: pgmap v444: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/688958719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/688958719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/688958719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:56.735 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:56 smithi064 ceph-mon[186706]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:56.735 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:56 smithi064 ceph-mon[180297]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:56 smithi152 ceph-mon[156819]: pgmap v445: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:57.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/623573503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/623573503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/623573503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[180297]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[180297]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[180297]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:58.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[186706]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:58.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[186706]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:58.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:58 smithi064 ceph-mon[186706]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:58.659 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:58 smithi064 conmon[222603]: 2023-06-10T21:18:58.285+0000 7f425ad95640 -1 osd.2 116 log_to_monitors true 2023-06-10T21:18:58.659 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:18:58 smithi064 conmon[222603]: 2023-06-10T21:18:58.501+0000 7f424a1fd700 -1 osd.2 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:18:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:58 smithi152 ceph-mon[156819]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:18:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:58 smithi152 ceph-mon[156819]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:58 smithi152 ceph-mon[156819]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-10T21:18:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: osdmap e119: 8 total, 7 up, 8 in 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3790467017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:59.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: osdmap e119: 8 total, 7 up, 8 in 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:18:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3790467017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:18:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: failed to encode map e119 with expected crc 2023-06-10T21:18:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-10T21:18:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: osdmap e119: 8 total, 7 up, 8 in 2023-06-10T21:18:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: from='osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:18:59.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:18:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3790467017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:00.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109] boot 2023-06-10T21:19:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: osdmap e120: 8 total, 8 up, 8 in 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109] boot 2023-06-10T21:19:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: osdmap e120: 8 total, 8 up, 8 in 2023-06-10T21:19:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:19:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: pgmap v448: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: failed to encode map e120 with expected crc 2023-06-10T21:19:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: osd.2 [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109] boot 2023-06-10T21:19:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: osdmap e120: 8 total, 8 up, 8 in 2023-06-10T21:19:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-10T21:19:01.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: osdmap e121: 8 total, 8 up, 8 in 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[180297]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: osdmap e121: 8 total, 8 up, 8 in 2023-06-10T21:19:01.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:01 smithi064 ceph-mon[186706]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: osdmap e121: 8 total, 8 up, 8 in 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:01 smithi152 ceph-mon[156819]: failed to encode map e121 with expected crc 2023-06-10T21:19:02.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:02 smithi064 ceph-mon[186706]: pgmap v451: 73 pgs: 2 peering, 2 active+undersized+degraded, 11 active+undersized, 58 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:02.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1339149742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:02 smithi064 ceph-mon[180297]: pgmap v451: 73 pgs: 2 peering, 2 active+undersized+degraded, 11 active+undersized, 58 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1339149742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:02 smithi152 ceph-mon[156819]: pgmap v451: 73 pgs: 2 peering, 2 active+undersized+degraded, 11 active+undersized, 58 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1339149742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:02] "GET /metrics HTTP/1.1" 200 210579 "" "Prometheus/2.33.4" 2023-06-10T21:19:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:04.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2866940711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2866940711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:04.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:04.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:04.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2866940711' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:05.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:05 smithi064 ceph-mon[180297]: pgmap v452: 73 pgs: 2 peering, 1 active+undersized+degraded, 10 active+undersized, 60 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:05.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:19:05.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:05 smithi064 ceph-mon[186706]: pgmap v452: 73 pgs: 2 peering, 1 active+undersized+degraded, 10 active+undersized, 60 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:05.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:19:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:05 smithi152 ceph-mon[156819]: pgmap v452: 73 pgs: 2 peering, 1 active+undersized+degraded, 10 active+undersized, 60 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:19:06.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[180297]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:06.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2300296622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[186706]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:19:06.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2300296622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:06 smithi152 ceph-mon[156819]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:06 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-10T21:19:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:06 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:19:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2300296622' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:07.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[186706]: Upgrade: osd.3 is safe to restart 2023-06-10T21:19:07.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:07 smithi064 ceph-mon[180297]: Upgrade: osd.3 is safe to restart 2023-06-10T21:19:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:07 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-10T21:19:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:07 smithi152 ceph-mon[156819]: Upgrade: osd.3 is safe to restart 2023-06-10T21:19:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[186706]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:19:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[180297]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:08.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:19:08.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:08 smithi152 ceph-mon[156819]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-10T21:19:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:09.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[186706]: Upgrade: Updating osd.3 2023-06-10T21:19:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[186706]: Deploying daemon osd.3 on smithi064 2023-06-10T21:19:09.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/999816098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[180297]: Upgrade: Updating osd.3 2023-06-10T21:19:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[180297]: Deploying daemon osd.3 on smithi064 2023-06-10T21:19:09.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/999816098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:09.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:09 smithi152 ceph-mon[156819]: Upgrade: Updating osd.3 2023-06-10T21:19:09.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:09 smithi152 ceph-mon[156819]: Deploying daemon osd.3 on smithi064 2023-06-10T21:19:09.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/999816098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:10 smithi064 ceph-mon[180297]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:10 smithi064 ceph-mon[186706]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:10.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:10 smithi152 ceph-mon[156819]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:11.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:11 smithi064 systemd[1]: Stopping Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:19:11.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2396401788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:11.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2396401788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:11.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:11 smithi064 conmon[129869]: debug 2023-06-10T21:19:11.580+0000 7f65737bf700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:19:11.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:11 smithi064 conmon[129869]: debug 2023-06-10T21:19:11.580+0000 7f65737bf700 -1 osd.3 121 *** Got signal Terminated *** 2023-06-10T21:19:11.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:11 smithi064 conmon[129869]: debug 2023-06-10T21:19:11.580+0000 7f65737bf700 -1 osd.3 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:19:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2396401788' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:12.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:12 smithi064 ceph-mon[186706]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:12.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:12 smithi064 ceph-mon[186706]: osd.3 marked itself down and dead 2023-06-10T21:19:12.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:12 smithi064 ceph-mon[180297]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:12.777 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:12 smithi064 ceph-mon[180297]: osd.3 marked itself down and dead 2023-06-10T21:19:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:12 smithi152 ceph-mon[156819]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:12 smithi152 ceph-mon[156819]: osd.3 marked itself down and dead 2023-06-10T21:19:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:12] "GET /metrics HTTP/1.1" 200 215381 "" "Prometheus/2.33.4" 2023-06-10T21:19:13.158 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:12 smithi064 bash[226392]: c9c0acd987a85f19a3f79ff08ac46481dcd9e7903c1722af0121e56926e92c4a 2023-06-10T21:19:13.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:19:13.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: osdmap e122: 8 total, 7 up, 8 in 2023-06-10T21:19:13.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.847 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.848 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1926323682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:13.849 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:19:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: osdmap e122: 8 total, 7 up, 8 in 2023-06-10T21:19:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.849 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.850 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1926323682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:13.851 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:19:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: osdmap e122: 8 total, 7 up, 8 in 2023-06-10T21:19:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: failed to encode map e122 with expected crc 2023-06-10T21:19:13.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1926323682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:13.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: pgmap v458: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: osdmap e123: 8 total, 7 up, 8 in 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: pgmap v458: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: osdmap e123: 8 total, 7 up, 8 in 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:14 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:19:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: pgmap v458: 73 pgs: 7 peering, 11 stale+active+clean, 55 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: osdmap e123: 8 total, 7 up, 8 in 2023-06-10T21:19:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: failed to encode map e123 with expected crc 2023-06-10T21:19:14.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:14 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:19:15.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:14 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3.service: Succeeded. 2023-06-10T21:19:15.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:14 smithi064 systemd[1]: Stopped Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:19:15.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:15 smithi064 systemd[1]: Starting Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:19:16.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-10T21:19:16.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-06-10T21:19:16.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-06-10T21:19:16.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-06-10T21:19:16.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2023-06-10T21:19:16.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-10T21:19:16.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:16 smithi064 bash[227063]: --> ceph-volume raw activate successful for osd ID: 3 2023-06-10T21:19:16.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[186706]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 10 peering, 4 stale+active+clean, 12 active+undersized, 45 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:16.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:16.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2772177201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:16.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[180297]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 10 peering, 4 stale+active+clean, 12 active+undersized, 45 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:16.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:16.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2772177201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:16 smithi152 ceph-mon[156819]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 10 peering, 4 stale+active+clean, 12 active+undersized, 45 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:16 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2772177201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:17.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:17 smithi064 ceph-osd[227587]: -- 172.21.15.64:0/4167113879 <== mon.2 v2:172.21.15.64:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ef86f7c4e0 con 0x55ef86f4ec00 2023-06-10T21:19:17.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:17 smithi064 bash[227520]: b4b26d4dbd297e03c4c2f3e53deb45e420a13c860b9abef5343cd9ced857edcd 2023-06-10T21:19:17.908 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:17 smithi064 systemd[1]: Started Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:19:18.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:17 smithi064 conmon[227562]: 2023-06-10T21:19:17.924+0000 7f8f18ff0640 -1 Falling back to public interface 2023-06-10T21:19:18.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 10 peering, 19 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:18.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2609828311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 10 peering, 19 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2609828311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 10 peering, 19 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-06-10T21:19:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2609828311' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:19.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:19 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-06-10T21:19:19.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:19 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-06-10T21:19:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:19 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-06-10T21:19:20.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[186706]: pgmap v462: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:20.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1737238439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[180297]: pgmap v462: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1737238439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:20 smithi152 ceph-mon[156819]: pgmap v462: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1737238439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.821 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:21.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:22.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:22] "GET /metrics HTTP/1.1" 200 215640 "" "Prometheus/2.33.4" 2023-06-10T21:19:23.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:23 smithi064 ceph-mon[186706]: pgmap v463: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:23.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1249776459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:23.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:23 smithi064 ceph-mon[180297]: pgmap v463: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:23.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1249776459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:23 smithi152 ceph-mon[156819]: pgmap v463: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1249776459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:24.657 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:24 smithi064 conmon[171866]: 2023-06-10T21:19:24.439+0000 7f6cfc057700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:19:25.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1101679112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:25.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1101679112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:25.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1101679112' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:25.907 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:25 smithi064 conmon[227562]: 2023-06-10T21:19:25.485+0000 7f8f18ff0640 -1 osd.3 121 log_to_monitors true 2023-06-10T21:19:26.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[186706]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[186706]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:19:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[186706]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[186706]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[180297]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:26.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[180297]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:19:26.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:26.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[180297]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:26 smithi064 ceph-mon[180297]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:26 smithi152 ceph-mon[156819]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-10T21:19:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:26 smithi152 ceph-mon[156819]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-06-10T21:19:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:26 smithi152 ceph-mon[156819]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:26 smithi152 ceph-mon[156819]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:19:26.907 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:19:26 smithi064 conmon[227562]: 2023-06-10T21:19:26.496+0000 7f8f08458700 -1 osd.3 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:19:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:27.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: failed to encode map e124 with expected crc 2023-06-10T21:19:27.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:19:27.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: osdmap e124: 8 total, 7 up, 8 in 2023-06-10T21:19:27.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:27.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[186706]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:27.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:27.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: failed to encode map e124 with expected crc 2023-06-10T21:19:27.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:19:27.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: osdmap e124: 8 total, 7 up, 8 in 2023-06-10T21:19:27.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:27.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:27 smithi064 ceph-mon[180297]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: failed to encode map e124 with expected crc 2023-06-10T21:19:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:19:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: osdmap e124: 8 total, 7 up, 8 in 2023-06-10T21:19:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: from='osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:27 smithi152 ceph-mon[156819]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:19:28.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: failed to encode map e124 with expected crc 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1064334544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: failed to encode map e125 with expected crc 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225] boot 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: osdmap e125: 8 total, 8 up, 8 in 2023-06-10T21:19:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: failed to encode map e124 with expected crc 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1064334544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: failed to encode map e125 with expected crc 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225] boot 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: osdmap e125: 8 total, 8 up, 8 in 2023-06-10T21:19:28.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:19:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: failed to encode map e124 with expected crc 2023-06-10T21:19:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1064334544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:28.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:19:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: failed to encode map e125 with expected crc 2023-06-10T21:19:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: osd.3 [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225] boot 2023-06-10T21:19:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: osdmap e125: 8 total, 8 up, 8 in 2023-06-10T21:19:28.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:19:29.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: failed to encode map e125 with expected crc 2023-06-10T21:19:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: osdmap e126: 8 total, 8 up, 8 in 2023-06-10T21:19:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: failed to encode map e125 with expected crc 2023-06-10T21:19:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: osdmap e126: 8 total, 8 up, 8 in 2023-06-10T21:19:29.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:29 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: failed to encode map e125 with expected crc 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: osdmap e126: 8 total, 8 up, 8 in 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:29 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:29.632 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 4 2023-06-10T21:19:30.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[186706]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2268887309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:30.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[180297]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2268887309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:30.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch upgrade pause' 2023-06-10T21:19:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:30 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:30 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:30 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:30 smithi152 ceph-mon[156819]: failed to encode map e126 with expected crc 2023-06-10T21:19:30.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2268887309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-10T21:19:31.264 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:31 smithi064 ceph-mon[186706]: pgmap v470: 73 pgs: 5 peering, 2 active+undersized+degraded, 17 active+undersized, 49 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:31.264 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:31.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:31 smithi064 ceph-mon[180297]: pgmap v470: 73 pgs: 5 peering, 2 active+undersized+degraded, 17 active+undersized, 49 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:31.265 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:31 smithi152 ceph-mon[156819]: pgmap v470: 73 pgs: 5 peering, 2 active+undersized+degraded, 17 active+undersized, 49 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:19:32.451 INFO:teuthology.orchestra.run.smithi064.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:19:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:32] "GET /metrics HTTP/1.1" 200 215640 "" "Prometheus/2.33.4" 2023-06-10T21:19:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:33 smithi064 ceph-mon[180297]: pgmap v471: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:33 smithi064 ceph-mon[186706]: pgmap v471: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:33.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:19:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:33 smithi152 ceph-mon[156819]: pgmap v471: 73 pgs: 5 peering, 2 active+undersized+degraded, 11 active+undersized, 55 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-10T21:19:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[180297]: from='client.35190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[180297]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:19:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[186706]: from='client.35190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[186706]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:19:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:34 smithi152 ceph-mon[156819]: from='client.35190 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:34 smithi152 ceph-mon[156819]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143 2023-06-10T21:19:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:35.274 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (16m) 15s ago 20m 30.8M - 0.23.0 ba2b418f427c 13fd7844ce61 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (15m) 14s ago 20m 61.7M - 8.3.5 dad864ee21e9 95a95736b9e9 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (19m) 15s ago 19m 24.5M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (19m) 14s ago 19m 17.9M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443,9283,8765 running (14m) 14s ago 24m 453M - 18.0.0-3603-g5937c68a 8fc92d915e8e 0e93fb4f51a4 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (15m) 15s ago 26m 530M - 18.0.0-3603-g5937c68a 8fc92d915e8e 2bda388cec1d 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (14m) 15s ago 26m 149M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 6f8d9991dea0 2023-06-10T21:19:35.275 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (10m) 14s ago 24m 88.0M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e a70d292c3272 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (12m) 15s ago 24m 96.6M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 925081a4753e 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (16m) 15s ago 20m 28.7M - 1.3.1 1dbe0e931976 c7179c864e62 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (16m) 14s ago 20m 29.5M - 1.3.1 1dbe0e931976 91ff512a05b1 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (9m) 15s ago 24m 4128M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e 0499b076219a 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (5m) 15s ago 23m 1372M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e b6d15648d91c 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (51s) 15s ago 23m 588M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e c76c1a3f425a 2023-06-10T21:19:35.276 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (18s) 15s ago 23m 12.5M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e b4b26d4dbd29 2023-06-10T21:19:35.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (22m) 14s ago 22m 5940M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:19:35.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (22m) 14s ago 22m 7817M 2500M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:19:35.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (22m) 14s ago 22m 5396M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:19:35.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (21m) 14s ago 21m 5624M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:19:35.277 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (14m) 14s ago 21m 90.4M - 2.33.4 514e6a882f6e 2013ef6ec5fb 2023-06-10T21:19:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[186706]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-06-10T21:19:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:19:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[180297]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-06-10T21:19:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:35 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:19:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:35 smithi152 ceph-mon[156819]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:35 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-06-10T21:19:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:35 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:19:36.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph orch ps' 2023-06-10T21:19:37.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:37 smithi064 ceph-mon[180297]: from='client.45191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:37.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:37 smithi064 ceph-mon[180297]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:37.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:37 smithi064 ceph-mon[186706]: from='client.45191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:37.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:37 smithi064 ceph-mon[186706]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:37 smithi152 ceph-mon[156819]: from='client.45191 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:37 smithi152 ceph-mon[156819]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:37.970 INFO:teuthology.orchestra.run.smithi064.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-10T21:19:37.970 INFO:teuthology.orchestra.run.smithi064.stdout:alertmanager.a smithi064 *:9093,9094 running (16m) 17s ago 20m 30.8M - 0.23.0 ba2b418f427c 13fd7844ce61 2023-06-10T21:19:37.970 INFO:teuthology.orchestra.run.smithi064.stdout:grafana.a smithi152 *:3000 running (15m) 17s ago 20m 61.7M - 8.3.5 dad864ee21e9 95a95736b9e9 2023-06-10T21:19:37.970 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi064.wuuunt smithi064 running (20m) 17s ago 20m 24.5M - 16.2.13-257-gd8c5d349 b644dd95899d 07055f0703bc 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mds.foo.smithi152.ciyxdy smithi152 running (19m) 17s ago 19m 17.9M - 16.2.13-257-gd8c5d349 b644dd95899d 2bc9035dd630 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.x smithi152 *:8443,9283,8765 running (14m) 17s ago 24m 453M - 18.0.0-3603-g5937c68a 8fc92d915e8e 0e93fb4f51a4 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mgr.y smithi064 *:8443,9283,8765 running (15m) 17s ago 26m 530M - 18.0.0-3603-g5937c68a 8fc92d915e8e 2bda388cec1d 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a smithi064 running (14m) 17s ago 26m 149M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 6f8d9991dea0 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b smithi152 running (10m) 17s ago 24m 88.0M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e a70d292c3272 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c smithi064 running (12m) 17s ago 24m 96.6M 2048M 18.0.0-3603-g5937c68a 8fc92d915e8e 925081a4753e 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.a smithi064 *:9100 running (16m) 17s ago 21m 28.7M - 1.3.1 1dbe0e931976 c7179c864e62 2023-06-10T21:19:37.971 INFO:teuthology.orchestra.run.smithi064.stdout:node-exporter.b smithi152 *:9100 running (16m) 17s ago 20m 29.5M - 1.3.1 1dbe0e931976 91ff512a05b1 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 smithi064 running (9m) 17s ago 24m 4128M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e 0499b076219a 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 smithi064 running (5m) 17s ago 23m 1372M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e b6d15648d91c 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 smithi064 running (53s) 17s ago 23m 588M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e c76c1a3f425a 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 smithi064 running (20s) 17s ago 23m 12.5M 2500M 18.0.0-3603-g5937c68a 8fc92d915e8e b4b26d4dbd29 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 smithi152 running (22m) 17s ago 22m 5940M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 8e3deb2c92b2 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 smithi152 running (22m) 17s ago 22m 7817M 2500M 16.2.13-257-gd8c5d349 b644dd95899d fa30a06be4b9 2023-06-10T21:19:37.972 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 smithi152 running (22m) 17s ago 22m 5396M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 6f9176fc7b52 2023-06-10T21:19:37.973 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 smithi152 running (21m) 17s ago 21m 5624M 2500M 16.2.13-257-gd8c5d349 b644dd95899d 746300ca8a91 2023-06-10T21:19:37.973 INFO:teuthology.orchestra.run.smithi064.stdout:prometheus.a smithi152 *:9095 running (14m) 17s ago 21m 90.4M - 2.33.4 514e6a882f6e 2013ef6ec5fb 2023-06-10T21:19:38.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -e sha1=5937c68aed42cde57f938ad8f8bb264f13176009 -- bash -c 'ceph versions' 2023-06-10T21:19:39.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:39 smithi064 ceph-mon[180297]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:39.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:39 smithi064 ceph-mon[180297]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:39 smithi064 ceph-mon[186706]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:39 smithi064 ceph-mon[186706]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:39 smithi152 ceph-mon[156819]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:39 smithi152 ceph-mon[156819]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:19:40.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.240 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.242 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:19:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:19:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:19:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:40.679 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:19:40.679 INFO:teuthology.orchestra.run.smithi064.stdout: "mon": { 2023-06-10T21:19:40.679 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 3 2023-06-10T21:19:40.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:19:40.679 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr": { 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 2 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: "osd": { 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 4, 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 4 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:19:40.680 INFO:teuthology.orchestra.run.smithi064.stdout: "mds": { 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 2 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: "overall": { 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 6, 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: "ceph version 18.0.0-3603-g5937c68a (5937c68aed42cde57f938ad8f8bb264f13176009) reef (rc)": 9 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:19:40.681 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:19:41.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:41 smithi064 ceph-mon[186706]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:41.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2165685681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:41.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:41 smithi064 ceph-mon[180297]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:41.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2165685681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:41.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:41 smithi152 ceph-mon[156819]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:41.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2165685681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-10T21:19:41.498 DEBUG:teuthology.parallel:result is None 2023-06-10T21:19:41.498 INFO:teuthology.run_tasks:Running task print... 2023-06-10T21:19:41.549 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-06-10T21:19:41.550 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-06-10T21:19:41.578 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-10T21:19:41.578 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-10T21:19:41.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph mgr dump --format=json 2023-06-10T21:19:41.946 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:43.139 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:42] "GET /metrics HTTP/1.1" 200 220652 "" "Prometheus/2.33.4" 2023-06-10T21:19:43.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:43 smithi064 ceph-mon[186706]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:43.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:43 smithi064 ceph-mon[180297]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:43.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:43 smithi152 ceph-mon[156819]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:43.795 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:19:44.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:44.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[186706]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:44.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/302959382' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T21:19:44.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:44.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[180297]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:44.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/302959382' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T21:19:44.629 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":39,"active_gid":34101,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6800","nonce":145433023},{"type":"v1","addr":"172.21.15.64:6801","nonce":145433023}]},"active_addr":"172.21.15.64:6801/145433023","active_change":"2023-06-10T21:05:32.903144+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25129,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":77,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":2758038732}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":615293838}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":2947718283}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.64:0","nonce":713388030}]}]} 2023-06-10T21:19:44.633 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-10T21:19:44.634 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-10T21:19:44.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T21:19:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:44 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:44 smithi152 ceph-mon[156819]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/302959382' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-10T21:19:44.995 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:46.579 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:19:46.579 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":126,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T21:19:28.465658+0000","last_up_change":"2023-06-10T21:19:27.463802+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":125,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6803","nonce":1597438149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6805","nonce":1597438149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6809","nonce":1597438149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6807","nonce":1597438149}]},"public_addr":"172.21.15.64:6803/1597438149","cluster_addr":"172.21.15.64:6805/1597438149","heartbeat_back_addr":"172.21.15.64:6809/1597438149","heartbeat_front_addr":"172.21.15.64:6807/1597438149","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":101,"up_from":105,"up_thru":125,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6811","nonce":215784888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6813","nonce":215784888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6817","nonce":215784888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6815","nonce":215784888}]},"public_addr":"172.21.15.64:6811/215784888","cluster_addr":"172.21.15.64:6813/215784888","heartbeat_back_addr":"172.21.15.64:6817/215784888","heartbeat_front_addr":"172.21.15.64:6815/215784888","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":125,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6819","nonce":746809109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6821","nonce":746809109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6825","nonce":746809109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6823","nonce":746809109}]},"public_addr":"172.21.15.64:6819/746809109","cluster_addr":"172.21.15.64:6821/746809109","heartbeat_back_addr":"172.21.15.64:6825/746809109","heartbeat_front_addr":"172.21.15.64:6823/746809109","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":121,"up_from":125,"up_thru":125,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6827","nonce":4028577225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6829","nonce":4028577225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6833","nonce":4028577225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6831","nonce":4028577225}]},"public_addr":"172.21.15.64:6827/4028577225","cluster_addr":"172.21.15.64:6829/4028577225","heartbeat_back_addr":"172.21.15.64:6833/4028577225","heartbeat_front_addr":"172.21.15.64:6831/4028577225","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":122,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":101},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":121},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6835/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1656599775":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/1955159746":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1184928448":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.152:6834/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.152:0/1248818928":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1338325282":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3731648028":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/703163662":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/3424306201":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3850833042":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.152:0/1618902179":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/1551224238":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/790526867":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/3618131890":"2023-06-11T21:02:16.546685+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T21:19:46.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:46 smithi064 ceph-mon[186706]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:46.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:46 smithi064 ceph-mon[180297]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:46 smithi152 ceph-mon[156819]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:47.234 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-10T21:19:47.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T21:19:47.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2774526384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:47.475 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2774526384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:47.598 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2774526384' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:49.341 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:19:49.342 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":126,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T21:19:28.465658+0000","last_up_change":"2023-06-10T21:19:27.463802+0000","last_in_change":"2023-06-10T20:57:38.831861+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":125,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6803","nonce":1597438149}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6805","nonce":1597438149}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6809","nonce":1597438149}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1597438149},{"type":"v1","addr":"172.21.15.64:6807","nonce":1597438149}]},"public_addr":"172.21.15.64:6803/1597438149","cluster_addr":"172.21.15.64:6805/1597438149","heartbeat_back_addr":"172.21.15.64:6809/1597438149","heartbeat_front_addr":"172.21.15.64:6807/1597438149","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":101,"up_from":105,"up_thru":125,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6811","nonce":215784888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6813","nonce":215784888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6817","nonce":215784888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6815","nonce":215784888}]},"public_addr":"172.21.15.64:6811/215784888","cluster_addr":"172.21.15.64:6813/215784888","heartbeat_back_addr":"172.21.15.64:6817/215784888","heartbeat_front_addr":"172.21.15.64:6815/215784888","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":125,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6819","nonce":746809109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6821","nonce":746809109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6825","nonce":746809109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6823","nonce":746809109}]},"public_addr":"172.21.15.64:6819/746809109","cluster_addr":"172.21.15.64:6821/746809109","heartbeat_back_addr":"172.21.15.64:6825/746809109","heartbeat_front_addr":"172.21.15.64:6823/746809109","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":121,"up_from":125,"up_thru":125,"down_at":122,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6827","nonce":4028577225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6829","nonce":4028577225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6833","nonce":4028577225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":4028577225},{"type":"v1","addr":"172.21.15.64:6831","nonce":4028577225}]},"public_addr":"172.21.15.64:6827/4028577225","cluster_addr":"172.21.15.64:6829/4028577225","heartbeat_back_addr":"172.21.15.64:6833/4028577225","heartbeat_front_addr":"172.21.15.64:6831/4028577225","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6801","nonce":3009869563}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6803","nonce":3009869563}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6807","nonce":3009869563}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3009869563},{"type":"v1","addr":"172.21.15.152:6805","nonce":3009869563}]},"public_addr":"172.21.15.152:6801/3009869563","cluster_addr":"172.21.15.152:6803/3009869563","heartbeat_back_addr":"172.21.15.152:6807/3009869563","heartbeat_front_addr":"172.21.15.152:6805/3009869563","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6809","nonce":1122141943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6811","nonce":1122141943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6815","nonce":1122141943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":1122141943},{"type":"v1","addr":"172.21.15.152:6813","nonce":1122141943}]},"public_addr":"172.21.15.152:6809/1122141943","cluster_addr":"172.21.15.152:6811/1122141943","heartbeat_back_addr":"172.21.15.152:6815/1122141943","heartbeat_front_addr":"172.21.15.152:6813/1122141943","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":122,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":125,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":101},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":121},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:6835/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.64:0/1656599775":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/1955159746":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1184928448":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.152:6834/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.152:0/1248818928":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1338325282":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3731648028":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/703163662":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/3424306201":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3850833042":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.152:0/1618902179":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/1551224238":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/790526867":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/3618131890":"2023-06-11T21:02:16.546685+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T21:19:49.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[180297]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:49.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:49.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:49.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[186706]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:49.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:49.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:49 smithi152 ceph-mon[156819]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:19:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:19:50.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-10T21:19:50.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-10T21:19:50.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-10T21:19:50.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-10T21:19:50.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-10T21:19:50.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-10T21:19:50.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-10T21:19:50.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-10T21:19:50.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/818038834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:50.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/818038834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/818038834' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:19:50.752 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:50.806 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:50.841 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:50.943 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:51.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:51 smithi064 ceph-mon[186706]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:51.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:51 smithi064 ceph-mon[180297]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:51 smithi152 ceph-mon[156819]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:52.221 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:52.823 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:53.138 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:19:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:19:52] "GET /metrics HTTP/1.1" 200 220630 "" "Prometheus/2.33.4" 2023-06-10T21:19:53.204 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:53 smithi064 ceph-mon[180297]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:53 smithi064 ceph-mon[186706]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:53 smithi152 ceph-mon[156819]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:54.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:19:55.279 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:55 smithi064 ceph-mon[186706]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:55.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:55 smithi064 ceph-mon[180297]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:55 smithi152 ceph-mon[156819]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:57 smithi064 ceph-mon[180297]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:57.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:57 smithi064 ceph-mon[186706]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:57 smithi152 ceph-mon[156819]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:58.281 INFO:teuthology.orchestra.run.smithi064.stdout:206158430474 2023-06-10T21:19:58.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-10T21:19:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:19:59 smithi064 ceph-mon[186706]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:19:59 smithi064 ceph-mon[180297]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:19:59.414 INFO:teuthology.orchestra.run.smithi064.stdout:515396075533 2023-06-10T21:19:59.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-10T21:19:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:19:59 smithi152 ceph-mon[156819]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:00.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:00 smithi064 ceph-mon[180297]: overall HEALTH_OK 2023-06-10T21:20:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:00 smithi064 ceph-mon[186706]: overall HEALTH_OK 2023-06-10T21:20:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:00 smithi152 ceph-mon[156819]: overall HEALTH_OK 2023-06-10T21:20:01.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:01 smithi064 ceph-mon[186706]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:01 smithi064 ceph-mon[180297]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:01 smithi152 ceph-mon[156819]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:01.588 INFO:teuthology.orchestra.run.smithi064.stdout:128849019158 2023-06-10T21:20:01.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-10T21:20:03.085 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:02] "GET /metrics HTTP/1.1" 200 220630 "" "Prometheus/2.33.4" 2023-06-10T21:20:03.114 INFO:teuthology.orchestra.run.smithi064.stdout:154618822930 2023-06-10T21:20:03.114 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-10T21:20:03.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:03 smithi064 ceph-mon[186706]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:03.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:03.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:03 smithi064 ceph-mon[180297]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:03.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:03.385 INFO:teuthology.orchestra.run.smithi064.stdout:369367187568 2023-06-10T21:20:03.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-10T21:20:03.402 INFO:teuthology.orchestra.run.smithi064.stdout:536870912008 2023-06-10T21:20:03.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-10T21:20:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:03 smithi152 ceph-mon[156819]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:03.485 INFO:teuthology.orchestra.run.smithi064.stdout:180388626703 2023-06-10T21:20:03.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-10T21:20:03.603 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:03.640 INFO:teuthology.orchestra.run.smithi064.stdout:450971566146 2023-06-10T21:20:03.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-10T21:20:03.650 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:03.913 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:04.357 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:05.105 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:05.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:05 smithi064 ceph-mon[186706]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:05.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:05 smithi064 ceph-mon[180297]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:05 smithi152 ceph-mon[156819]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:07.435 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:07 smithi064 ceph-mon[186706]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:07.436 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:07 smithi064 ceph-mon[180297]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:07.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:07 smithi152 ceph-mon[156819]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:07.503 INFO:teuthology.orchestra.run.smithi064.stdout:206158430476 2023-06-10T21:20:08.427 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4112102557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:20:08.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4112102557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:20:08.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4112102557' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:20:08.472 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:08.814 INFO:teuthology.orchestra.run.smithi064.stdout:128849019160 2023-06-10T21:20:09.421 INFO:teuthology.orchestra.run.smithi064.stdout:369367187569 2023-06-10T21:20:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:09 smithi152 ceph-mon[156819]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1611879043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:20:09.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:09 smithi064 ceph-mon[186706]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:09.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1611879043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:20:09.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:09 smithi064 ceph-mon[180297]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:09.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1611879043' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:20:09.936 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430474 got 206158430476 for osd.7 2023-06-10T21:20:09.937 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:10.172 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1609749331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:20:10.179 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:10.278 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019158 got 128849019160 for osd.4 2023-06-10T21:20:10.278 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1609749331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:20:10.521 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:10.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1609749331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:20:10.889 INFO:teuthology.orchestra.run.smithi064.stdout:515396075536 2023-06-10T21:20:11.210 INFO:teuthology.orchestra.run.smithi064.stdout:154618822932 2023-06-10T21:20:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:11 smithi152 ceph-mon[156819]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1942237503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:20:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2722498772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:20:11.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[186706]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:11.574 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1942237503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:20:11.574 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2722498772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:20:11.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[180297]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:11.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1942237503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:20:11.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2722498772' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:20:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:12] "GET /metrics HTTP/1.1" 200 220620 "" "Prometheus/2.33.4" 2023-06-10T21:20:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:13 smithi152 ceph-mon[156819]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:13.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:13 smithi064 ceph-mon[180297]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:13.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:13 smithi064 ceph-mon[186706]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:13.526 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822930 got 154618822932 for osd.5 2023-06-10T21:20:13.527 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:13.657 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367187568 got 369367187569 for osd.0 2023-06-10T21:20:13.658 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:14.707 INFO:tasks.cephadm.ceph_manager.ceph:need seq 515396075533 got 515396075536 for osd.2 2023-06-10T21:20:14.708 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:14.973 INFO:teuthology.orchestra.run.smithi064.stdout:180388626705 2023-06-10T21:20:15.030 INFO:teuthology.orchestra.run.smithi064.stdout:450971566148 2023-06-10T21:20:15.291 INFO:teuthology.orchestra.run.smithi064.stdout:536870912011 2023-06-10T21:20:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:15 smithi152 ceph-mon[156819]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/909682535' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:20:15.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1490869406' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:20:15.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[180297]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/909682535' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:20:15.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1490869406' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:20:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[186706]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/909682535' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:20:15.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1490869406' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:20:16.105 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626703 got 180388626705 for osd.6 2023-06-10T21:20:16.106 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:16.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/217793121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:20:16.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/217793121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:20:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/217793121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:20:16.624 INFO:tasks.cephadm.ceph_manager.ceph:need seq 450971566146 got 450971566148 for osd.1 2023-06-10T21:20:16.625 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:16.945 INFO:tasks.cephadm.ceph_manager.ceph:need seq 536870912008 got 536870912011 for osd.3 2023-06-10T21:20:16.945 DEBUG:teuthology.parallel:result is None 2023-06-10T21:20:16.945 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-10T21:20:16.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:20:17.311 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:17 smithi152 ceph-mon[156819]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:17.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:17 smithi064 ceph-mon[186706]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:17.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:17 smithi064 ceph-mon[180297]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:18.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:18.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:19.304 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:20:19.305 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:20:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:19 smithi152 ceph-mon[156819]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:19.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:19 smithi064 ceph-mon[186706]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:19.524 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:19 smithi064 ceph-mon[180297]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:20.006 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":494,"stamp":"2023-06-10T21:20:17.405433+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2112156,"kb_used_data":11612,"kb_used_omap":20,"kb_used_meta":2100459,"kb_avail":747849060,"statfs":{"total":767960285184,"available":765797437440,"internally_reserved":0,"allocated":11890688,"data_stored":7934249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2150870670},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003034"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471845+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:19:27.471845+0000","last_peered":"2023-06-10T21:19:27.471845+0000","last_clean":"2023-06-10T21:19:27.471845+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:19:27.471845+0000","last_undegraded":"2023-06-10T21:19:27.471845+0000","last_fullsized":"2023-06-10T21:19:27.471845+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.478832+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:19:28.478832+0000","last_peered":"2023-06-10T21:19:28.478832+0000","last_clean":"2023-06-10T21:19:28.478832+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:19:28.478832+0000","last_undegraded":"2023-06-10T21:19:28.478832+0000","last_fullsized":"2023-06-10T21:19:28.478832+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487021+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:19:28.487021+0000","last_peered":"2023-06-10T21:19:28.487021+0000","last_clean":"2023-06-10T21:19:28.487021+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:19:28.487021+0000","last_undegraded":"2023-06-10T21:19:28.487021+0000","last_fullsized":"2023-06-10T21:19:28.487021+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483727+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:19:28.483727+0000","last_peered":"2023-06-10T21:19:28.483727+0000","last_clean":"2023-06-10T21:19:28.483727+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:19:28.483727+0000","last_undegraded":"2023-06-10T21:19:28.483727+0000","last_fullsized":"2023-06-10T21:19:28.483727+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480757+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:19:28.480757+0000","last_peered":"2023-06-10T21:19:28.480757+0000","last_clean":"2023-06-10T21:19:28.480757+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:19:28.480757+0000","last_undegraded":"2023-06-10T21:19:28.480757+0000","last_fullsized":"2023-06-10T21:19:28.480757+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470669+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:19:27.470669+0000","last_peered":"2023-06-10T21:19:27.470669+0000","last_clean":"2023-06-10T21:19:27.470669+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:19:27.470669+0000","last_undegraded":"2023-06-10T21:19:27.470669+0000","last_fullsized":"2023-06-10T21:19:27.470669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486071+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:19:28.486071+0000","last_peered":"2023-06-10T21:19:28.486071+0000","last_clean":"2023-06-10T21:19:28.486071+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:19:28.486071+0000","last_undegraded":"2023-06-10T21:19:28.486071+0000","last_fullsized":"2023-06-10T21:19:28.486071+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480260+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:19:28.480260+0000","last_peered":"2023-06-10T21:19:28.480260+0000","last_clean":"2023-06-10T21:19:28.480260+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:19:28.480260+0000","last_undegraded":"2023-06-10T21:19:28.480260+0000","last_fullsized":"2023-06-10T21:19:28.480260+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476985+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:19:27.476985+0000","last_peered":"2023-06-10T21:19:27.476985+0000","last_clean":"2023-06-10T21:19:27.476985+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:19:27.476985+0000","last_undegraded":"2023-06-10T21:19:27.476985+0000","last_fullsized":"2023-06-10T21:19:27.476985+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483795+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.483795+0000","last_peered":"2023-06-10T21:19:28.483795+0000","last_clean":"2023-06-10T21:19:28.483795+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:19:28.483795+0000","last_undegraded":"2023-06-10T21:19:28.483795+0000","last_fullsized":"2023-06-10T21:19:28.483795+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483994+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:19:28.483994+0000","last_peered":"2023-06-10T21:19:28.483994+0000","last_clean":"2023-06-10T21:19:28.483994+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:19:28.483994+0000","last_undegraded":"2023-06-10T21:19:28.483994+0000","last_fullsized":"2023-06-10T21:19:28.483994+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474354+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:19:27.474354+0000","last_peered":"2023-06-10T21:19:27.474354+0000","last_clean":"2023-06-10T21:19:27.474354+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:19:27.474354+0000","last_undegraded":"2023-06-10T21:19:27.474354+0000","last_fullsized":"2023-06-10T21:19:27.474354+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472826+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:19:27.472826+0000","last_peered":"2023-06-10T21:19:27.472826+0000","last_clean":"2023-06-10T21:19:27.472826+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:19:27.472826+0000","last_undegraded":"2023-06-10T21:19:27.472826+0000","last_fullsized":"2023-06-10T21:19:27.472826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474064+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:19:27.474064+0000","last_peered":"2023-06-10T21:19:27.474064+0000","last_clean":"2023-06-10T21:19:27.474064+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:19:27.474064+0000","last_undegraded":"2023-06-10T21:19:27.474064+0000","last_fullsized":"2023-06-10T21:19:27.474064+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475311+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:19:27.475311+0000","last_peered":"2023-06-10T21:19:27.475311+0000","last_clean":"2023-06-10T21:19:27.475311+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:19:27.475311+0000","last_undegraded":"2023-06-10T21:19:27.475311+0000","last_fullsized":"2023-06-10T21:19:27.475311+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483638+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:19:28.483638+0000","last_peered":"2023-06-10T21:19:28.483638+0000","last_clean":"2023-06-10T21:19:28.483638+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:19:28.483638+0000","last_undegraded":"2023-06-10T21:19:28.483638+0000","last_fullsized":"2023-06-10T21:19:28.483638+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486854+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.486854+0000","last_peered":"2023-06-10T21:19:28.486854+0000","last_clean":"2023-06-10T21:19:28.486854+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:19:28.486854+0000","last_undegraded":"2023-06-10T21:19:28.486854+0000","last_fullsized":"2023-06-10T21:19:28.486854+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481386+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:19:28.481386+0000","last_peered":"2023-06-10T21:19:28.481386+0000","last_clean":"2023-06-10T21:19:28.481386+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:19:28.481386+0000","last_undegraded":"2023-06-10T21:19:28.481386+0000","last_fullsized":"2023-06-10T21:19:28.481386+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475349+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:19:27.475349+0000","last_peered":"2023-06-10T21:19:27.475349+0000","last_clean":"2023-06-10T21:19:27.475349+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:19:27.475349+0000","last_undegraded":"2023-06-10T21:19:27.475349+0000","last_fullsized":"2023-06-10T21:19:27.475349+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483808+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:19:28.483808+0000","last_peered":"2023-06-10T21:19:28.483808+0000","last_clean":"2023-06-10T21:19:28.483808+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:19:28.483808+0000","last_undegraded":"2023-06-10T21:19:28.483808+0000","last_fullsized":"2023-06-10T21:19:28.483808+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470529+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:19:27.470529+0000","last_peered":"2023-06-10T21:19:27.470529+0000","last_clean":"2023-06-10T21:19:27.470529+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:19:27.470529+0000","last_undegraded":"2023-06-10T21:19:27.470529+0000","last_fullsized":"2023-06-10T21:19:27.470529+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484436+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:19:28.484436+0000","last_peered":"2023-06-10T21:19:28.484436+0000","last_clean":"2023-06-10T21:19:28.484436+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:19:28.484436+0000","last_undegraded":"2023-06-10T21:19:28.484436+0000","last_fullsized":"2023-06-10T21:19:28.484436+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476812+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:19:27.476812+0000","last_peered":"2023-06-10T21:19:27.476812+0000","last_clean":"2023-06-10T21:19:27.476812+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:19:27.476812+0000","last_undegraded":"2023-06-10T21:19:27.476812+0000","last_fullsized":"2023-06-10T21:19:27.476812+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474154+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:19:27.474154+0000","last_peered":"2023-06-10T21:19:27.474154+0000","last_clean":"2023-06-10T21:19:27.474154+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:19:27.474154+0000","last_undegraded":"2023-06-10T21:19:27.474154+0000","last_fullsized":"2023-06-10T21:19:27.474154+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476603+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:19:27.476603+0000","last_peered":"2023-06-10T21:19:27.476603+0000","last_clean":"2023-06-10T21:19:27.476603+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:19:27.476603+0000","last_undegraded":"2023-06-10T21:19:27.476603+0000","last_fullsized":"2023-06-10T21:19:27.476603+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487906+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:19:28.487906+0000","last_peered":"2023-06-10T21:19:28.487906+0000","last_clean":"2023-06-10T21:19:28.487906+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:19:28.487906+0000","last_undegraded":"2023-06-10T21:19:28.487906+0000","last_fullsized":"2023-06-10T21:19:28.487906+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477579+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:19:27.477579+0000","last_peered":"2023-06-10T21:19:27.477579+0000","last_clean":"2023-06-10T21:19:27.477579+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:19:27.477579+0000","last_undegraded":"2023-06-10T21:19:27.477579+0000","last_fullsized":"2023-06-10T21:19:27.477579+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470358+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:19:27.470358+0000","last_peered":"2023-06-10T21:19:27.470358+0000","last_clean":"2023-06-10T21:19:27.470358+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:19:27.470358+0000","last_undegraded":"2023-06-10T21:19:27.470358+0000","last_fullsized":"2023-06-10T21:19:27.470358+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480945+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:19:28.480945+0000","last_peered":"2023-06-10T21:19:28.480945+0000","last_clean":"2023-06-10T21:19:28.480945+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:19:28.480945+0000","last_undegraded":"2023-06-10T21:19:28.480945+0000","last_fullsized":"2023-06-10T21:19:28.480945+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.477277+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:19:28.477277+0000","last_peered":"2023-06-10T21:19:28.477277+0000","last_clean":"2023-06-10T21:19:28.477277+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:19:28.477277+0000","last_undegraded":"2023-06-10T21:19:28.477277+0000","last_fullsized":"2023-06-10T21:19:28.477277+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481924+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:19:28.481924+0000","last_peered":"2023-06-10T21:19:28.481924+0000","last_clean":"2023-06-10T21:19:28.481924+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:19:28.481924+0000","last_undegraded":"2023-06-10T21:19:28.481924+0000","last_fullsized":"2023-06-10T21:19:28.481924+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.488064+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:19:28.488064+0000","last_peered":"2023-06-10T21:19:28.488064+0000","last_clean":"2023-06-10T21:19:28.488064+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:19:28.488064+0000","last_undegraded":"2023-06-10T21:19:28.488064+0000","last_fullsized":"2023-06-10T21:19:28.488064+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470197+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:19:27.470197+0000","last_peered":"2023-06-10T21:19:27.470197+0000","last_clean":"2023-06-10T21:19:27.470197+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:19:27.470197+0000","last_undegraded":"2023-06-10T21:19:27.470197+0000","last_fullsized":"2023-06-10T21:19:27.470197+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472148+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:19:27.472148+0000","last_peered":"2023-06-10T21:19:27.472148+0000","last_clean":"2023-06-10T21:19:27.472148+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:19:27.472148+0000","last_undegraded":"2023-06-10T21:19:27.472148+0000","last_fullsized":"2023-06-10T21:19:27.472148+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472225+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:19:27.472225+0000","last_peered":"2023-06-10T21:19:27.472225+0000","last_clean":"2023-06-10T21:19:27.472225+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:19:27.472225+0000","last_undegraded":"2023-06-10T21:19:27.472225+0000","last_fullsized":"2023-06-10T21:19:27.472225+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471728+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:19:27.471728+0000","last_peered":"2023-06-10T21:19:27.471728+0000","last_clean":"2023-06-10T21:19:27.471728+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:19:27.471728+0000","last_undegraded":"2023-06-10T21:19:27.471728+0000","last_fullsized":"2023-06-10T21:19:27.471728+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483541+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:19:28.483541+0000","last_peered":"2023-06-10T21:19:28.483541+0000","last_clean":"2023-06-10T21:19:28.483541+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:19:28.483541+0000","last_undegraded":"2023-06-10T21:19:28.483541+0000","last_fullsized":"2023-06-10T21:19:28.483541+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484999+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:19:28.484999+0000","last_peered":"2023-06-10T21:19:28.484999+0000","last_clean":"2023-06-10T21:19:28.484999+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:19:28.484999+0000","last_undegraded":"2023-06-10T21:19:28.484999+0000","last_fullsized":"2023-06-10T21:19:28.484999+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474695+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:19:27.474695+0000","last_peered":"2023-06-10T21:19:27.474695+0000","last_clean":"2023-06-10T21:19:27.474695+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:19:27.474695+0000","last_undegraded":"2023-06-10T21:19:27.474695+0000","last_fullsized":"2023-06-10T21:19:27.474695+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482192+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:19:28.482192+0000","last_peered":"2023-06-10T21:19:28.482192+0000","last_clean":"2023-06-10T21:19:28.482192+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:19:28.482192+0000","last_undegraded":"2023-06-10T21:19:28.482192+0000","last_fullsized":"2023-06-10T21:19:28.482192+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.480734+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:19:27.480734+0000","last_peered":"2023-06-10T21:19:27.480734+0000","last_clean":"2023-06-10T21:19:27.480734+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:19:27.480734+0000","last_undegraded":"2023-06-10T21:19:27.480734+0000","last_fullsized":"2023-06-10T21:19:27.480734+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475368+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:19:27.475368+0000","last_peered":"2023-06-10T21:19:27.475368+0000","last_clean":"2023-06-10T21:19:27.475368+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:19:27.475368+0000","last_undegraded":"2023-06-10T21:19:27.475368+0000","last_fullsized":"2023-06-10T21:19:27.475368+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474220+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:19:27.474220+0000","last_peered":"2023-06-10T21:19:27.474220+0000","last_clean":"2023-06-10T21:19:27.474220+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:19:27.474220+0000","last_undegraded":"2023-06-10T21:19:27.474220+0000","last_fullsized":"2023-06-10T21:19:27.474220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486084+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:19:28.486084+0000","last_peered":"2023-06-10T21:19:28.486084+0000","last_clean":"2023-06-10T21:19:28.486084+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:19:28.486084+0000","last_undegraded":"2023-06-10T21:19:28.486084+0000","last_fullsized":"2023-06-10T21:19:28.486084+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482444+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:19:28.482444+0000","last_peered":"2023-06-10T21:19:28.482444+0000","last_clean":"2023-06-10T21:19:28.482444+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:19:28.482444+0000","last_undegraded":"2023-06-10T21:19:28.482444+0000","last_fullsized":"2023-06-10T21:19:28.482444+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474223+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:19:27.474223+0000","last_peered":"2023-06-10T21:19:27.474223+0000","last_clean":"2023-06-10T21:19:27.474223+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:19:27.474223+0000","last_undegraded":"2023-06-10T21:19:27.474223+0000","last_fullsized":"2023-06-10T21:19:27.474223+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478558+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:19:27.478558+0000","last_peered":"2023-06-10T21:19:27.478558+0000","last_clean":"2023-06-10T21:19:27.478558+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:19:27.478558+0000","last_undegraded":"2023-06-10T21:19:27.478558+0000","last_fullsized":"2023-06-10T21:19:27.478558+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475296+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:19:27.475296+0000","last_peered":"2023-06-10T21:19:27.475296+0000","last_clean":"2023-06-10T21:19:27.475296+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:19:27.475296+0000","last_undegraded":"2023-06-10T21:19:27.475296+0000","last_fullsized":"2023-06-10T21:19:27.475296+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":329,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475365+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:19:27.475365+0000","last_peered":"2023-06-10T21:19:27.475365+0000","last_clean":"2023-06-10T21:19:27.475365+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:19:27.475365+0000","last_undegraded":"2023-06-10T21:19:27.475365+0000","last_fullsized":"2023-06-10T21:19:27.475365+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481502+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:19:28.481502+0000","last_peered":"2023-06-10T21:19:28.481502+0000","last_clean":"2023-06-10T21:19:28.481502+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:19:28.481502+0000","last_undegraded":"2023-06-10T21:19:28.481502+0000","last_fullsized":"2023-06-10T21:19:28.481502+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470465+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:19:27.470465+0000","last_peered":"2023-06-10T21:19:27.470465+0000","last_clean":"2023-06-10T21:19:27.470465+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:19:27.470465+0000","last_undegraded":"2023-06-10T21:19:27.470465+0000","last_fullsized":"2023-06-10T21:19:27.470465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475272+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:19:27.475272+0000","last_peered":"2023-06-10T21:19:27.475272+0000","last_clean":"2023-06-10T21:19:27.475272+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:19:27.475272+0000","last_undegraded":"2023-06-10T21:19:27.475272+0000","last_fullsized":"2023-06-10T21:19:27.475272+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487756+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:19:28.487756+0000","last_peered":"2023-06-10T21:19:28.487756+0000","last_clean":"2023-06-10T21:19:28.487756+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:19:28.487756+0000","last_undegraded":"2023-06-10T21:19:28.487756+0000","last_fullsized":"2023-06-10T21:19:28.487756+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686568+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:19:27.686568+0000","last_peered":"2023-06-10T21:19:27.686568+0000","last_clean":"2023-06-10T21:19:27.686568+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:19:27.686568+0000","last_undegraded":"2023-06-10T21:19:27.686568+0000","last_fullsized":"2023-06-10T21:19:27.686568+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477662+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:19:27.477662+0000","last_peered":"2023-06-10T21:19:27.477662+0000","last_clean":"2023-06-10T21:19:27.477662+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:19:27.477662+0000","last_undegraded":"2023-06-10T21:19:27.477662+0000","last_fullsized":"2023-06-10T21:19:27.477662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475077+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:19:27.475077+0000","last_peered":"2023-06-10T21:19:27.475077+0000","last_clean":"2023-06-10T21:19:27.475077+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:19:27.475077+0000","last_undegraded":"2023-06-10T21:19:27.475077+0000","last_fullsized":"2023-06-10T21:19:27.475077+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482778+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:19:27.482778+0000","last_peered":"2023-06-10T21:19:27.482778+0000","last_clean":"2023-06-10T21:19:27.482778+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:19:27.482778+0000","last_undegraded":"2023-06-10T21:19:27.482778+0000","last_fullsized":"2023-06-10T21:19:27.482778+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484610+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:19:28.484610+0000","last_peered":"2023-06-10T21:19:28.484610+0000","last_clean":"2023-06-10T21:19:28.484610+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:19:28.484610+0000","last_undegraded":"2023-06-10T21:19:28.484610+0000","last_fullsized":"2023-06-10T21:19:28.484610+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":86,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475914+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:19:27.475914+0000","last_peered":"2023-06-10T21:19:27.475914+0000","last_clean":"2023-06-10T21:19:27.475914+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:19:27.475914+0000","last_undegraded":"2023-06-10T21:19:27.475914+0000","last_fullsized":"2023-06-10T21:19:27.475914+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487200+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:19:28.487200+0000","last_peered":"2023-06-10T21:19:28.487200+0000","last_clean":"2023-06-10T21:19:28.487200+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:19:28.487200+0000","last_undegraded":"2023-06-10T21:19:28.487200+0000","last_fullsized":"2023-06-10T21:19:28.487200+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470586+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:19:27.470586+0000","last_peered":"2023-06-10T21:19:27.470586+0000","last_clean":"2023-06-10T21:19:27.470586+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:19:27.470586+0000","last_undegraded":"2023-06-10T21:19:27.470586+0000","last_fullsized":"2023-06-10T21:19:27.470586+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686543+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:19:27.686543+0000","last_peered":"2023-06-10T21:19:27.686543+0000","last_clean":"2023-06-10T21:19:27.686543+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:19:27.686543+0000","last_undegraded":"2023-06-10T21:19:27.686543+0000","last_fullsized":"2023-06-10T21:19:27.686543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472930+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:19:27.472930+0000","last_peered":"2023-06-10T21:19:27.472930+0000","last_clean":"2023-06-10T21:19:27.472930+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:19:27.472930+0000","last_undegraded":"2023-06-10T21:19:27.472930+0000","last_fullsized":"2023-06-10T21:19:27.472930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":68,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482973+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:19:27.482973+0000","last_peered":"2023-06-10T21:19:27.482973+0000","last_clean":"2023-06-10T21:19:27.482973+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:19:27.482973+0000","last_undegraded":"2023-06-10T21:19:27.482973+0000","last_fullsized":"2023-06-10T21:19:27.482973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482329+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:19:28.482329+0000","last_peered":"2023-06-10T21:19:28.482329+0000","last_clean":"2023-06-10T21:19:28.482329+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:19:28.482329+0000","last_undegraded":"2023-06-10T21:19:28.482329+0000","last_fullsized":"2023-06-10T21:19:28.482329+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478090+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:19:27.478090+0000","last_peered":"2023-06-10T21:19:27.478090+0000","last_clean":"2023-06-10T21:19:27.478090+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:19:27.478090+0000","last_undegraded":"2023-06-10T21:19:27.478090+0000","last_fullsized":"2023-06-10T21:19:27.478090+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487998+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:19:28.487998+0000","last_peered":"2023-06-10T21:19:28.487998+0000","last_clean":"2023-06-10T21:19:28.487998+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:19:28.487998+0000","last_undegraded":"2023-06-10T21:19:28.487998+0000","last_fullsized":"2023-06-10T21:19:28.487998+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477850+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:19:27.477850+0000","last_peered":"2023-06-10T21:19:27.477850+0000","last_clean":"2023-06-10T21:19:27.477850+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:19:27.477850+0000","last_undegraded":"2023-06-10T21:19:27.477850+0000","last_fullsized":"2023-06-10T21:19:27.477850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482280+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:19:28.482280+0000","last_peered":"2023-06-10T21:19:28.482280+0000","last_clean":"2023-06-10T21:19:28.482280+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:19:28.482280+0000","last_undegraded":"2023-06-10T21:19:28.482280+0000","last_fullsized":"2023-06-10T21:19:28.482280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482461+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:19:28.482461+0000","last_peered":"2023-06-10T21:19:28.482461+0000","last_clean":"2023-06-10T21:19:28.482461+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:19:28.482461+0000","last_undegraded":"2023-06-10T21:19:28.482461+0000","last_fullsized":"2023-06-10T21:19:28.482461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481801+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:19:28.481801+0000","last_peered":"2023-06-10T21:19:28.481801+0000","last_clean":"2023-06-10T21:19:28.481801+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:19:28.481801+0000","last_undegraded":"2023-06-10T21:19:28.481801+0000","last_fullsized":"2023-06-10T21:19:28.481801+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482857+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:19:27.482857+0000","last_peered":"2023-06-10T21:19:27.482857+0000","last_clean":"2023-06-10T21:19:27.482857+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:19:27.482857+0000","last_undegraded":"2023-06-10T21:19:27.482857+0000","last_fullsized":"2023-06-10T21:19:27.482857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471651+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:19:27.471651+0000","last_peered":"2023-06-10T21:19:27.471651+0000","last_clean":"2023-06-10T21:19:27.471651+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:19:27.471651+0000","last_undegraded":"2023-06-10T21:19:27.471651+0000","last_fullsized":"2023-06-10T21:19:27.471651+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430478,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305940,"kb_used_data":1804,"kb_used_omap":0,"kb_used_meta":304128,"kb_avail":93439212,"statfs":{"total":95995035648,"available":95681753088,"internally_reserved":0,"allocated":1847296,"data_stored":1359411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":311427072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.258,"15min":1.3480000000000001},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.68899999999999995,"5min":10.914999999999999,"15min":10.914999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.96,"15min":1.95},"min":{"1min":0.41999999999999998,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":0.75600000000000001,"5min":15.925000000000001,"15min":15.925000000000001},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:19:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.855,"15min":1.5369999999999999},"min":{"1min":0.40300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77900000000000003,"5min":10.083,"15min":10.083},"last":0.629},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.5600000000000001,"15min":1.52},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.751,"5min":7,"15min":7},"last":0.58699999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":2.7200000000000002,"15min":2.8069999999999999},"min":{"1min":0.51200000000000001,"5min":0.35099999999999998,"15min":0.29699999999999999},"max":{"1min":15.127000000000001,"5min":15.127000000000001,"15min":15.127000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.246,"5min":2.556,"15min":2.7149999999999999},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":15.275,"5min":15.275,"15min":15.275},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.1579999999999999,"15min":2.0960000000000001},"min":{"1min":0.309,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.67000000000000004,"5min":8.0830000000000002,"15min":219.17699999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4219999999999999,"15min":2.2469999999999999},"min":{"1min":0.34300000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.77200000000000002,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.522,"15min":2.7719999999999998},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.254},"max":{"1min":0.77100000000000002,"5min":37.759,"15min":311.202},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4990000000000001,"15min":2.6040000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.29699999999999999},"max":{"1min":0.72899999999999998,"5min":30.074999999999999,"15min":311.875},"last":0.67500000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.2130000000000001,"15min":1.2769999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78500000000000003,"5min":6.8570000000000002,"15min":15.58},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.01,"15min":1.1259999999999999},"min":{"1min":0.46200000000000002,"5min":0.33400000000000002,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":7.6900000000000004,"15min":8.8829999999999991},"last":0.64200000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626706,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281028,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":279232,"kb_avail":93464124,"statfs":{"total":95995035648,"available":95707262976,"internally_reserved":0,"allocated":1830912,"data_stored":1358950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285933568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.4059999999999999,"15min":2.1970000000000001},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":0.80200000000000005,"5min":9.7929999999999993,"15min":119.276},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.704,"15min":2.8410000000000002},"min":{"1min":0.378,"5min":0.252,"15min":0.252},"max":{"1min":0.73099999999999998,"5min":16.324999999999999,"15min":119.22},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.788,"15min":1.823},"min":{"1min":0.37,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70399999999999996,"5min":14.897,"15min":14.897},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.9910000000000001,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.84899999999999998,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.55200000000000005}]},{"osd":3,"last update":"Sat Jun 10 21:18:17 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.891,"15min":2.7789999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29099999999999998},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":74.248000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.6659999999999999,"15min":2.9849999999999999},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":7.3029999999999999,"5min":15.488,"15min":74.314999999999998},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:19:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.016,"15min":1.8500000000000001},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.98799999999999999,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.3140000000000001,"15min":2.1040000000000001},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.745,"5min":11.263999999999999,"15min":221.905},"last":0.52900000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:19:21 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.0780000000000001,"15min":2.3639999999999999},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.879,"5min":6.6680000000000001,"15min":313.661},"last":0.5},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.2609999999999999,"15min":2.488},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.745,"5min":9.75,"15min":314.38900000000001},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.2689999999999999,"15min":1.1579999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.81100000000000005,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.1859999999999999,"15min":1.2130000000000001},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":0.78500000000000003,"5min":8.0980000000000008,"15min":12.583},"last":0.48999999999999999}]}]},{"osd":1,"up_from":105,"seq":450971566149,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213956,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":212667,"kb_avail":93531196,"statfs":{"total":95995035648,"available":95775944704,"internally_reserved":0,"allocated":1306624,"data_stored":776163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":217771918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":1.9319999999999999,"15min":1.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.49099999999999999,"5min":10.898,"15min":10.898},"last":0.317},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.4420000000000002,"15min":2.383},"min":{"1min":0.315,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.843,"5min":16.488,"15min":16.488},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.895,"5min":1.895,"15min":1.895},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.28100000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.927,"15min":3.1629999999999998},"min":{"1min":0.32400000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.19,"5min":13.349,"15min":13.349},"last":0.38},{"interface":"front","average":{"1min":2.0030000000000001,"5min":2.9740000000000002,"15min":2.4500000000000002},"min":{"1min":0.35499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.7530000000000001,"5min":22.033999999999999,"15min":22.033999999999999},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.8400000000000001,"15min":1.9299999999999999},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.9550000000000001,"5min":12.94,"15min":12.94},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.008,"15min":2.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.83299999999999996,"5min":12.603,"15min":12.603},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.2050000000000001,"15min":2.1720000000000002},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.9529999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.2730000000000001,"15min":2.5},"min":{"1min":0.45600000000000002,"5min":0.309,"15min":0.309},"max":{"1min":1.9870000000000001,"5min":13.602,"15min":13.602},"last":0.65700000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.2050000000000001,"15min":2.2130000000000001},"min":{"1min":0.504,"5min":0.379,"15min":0.379},"max":{"1min":3.0169999999999999,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.622},{"interface":"front","average":{"1min":0.752,"5min":2.2029999999999998,"15min":2.524},"min":{"1min":0.47599999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.004,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4180000000000001,"15min":3.2530000000000001},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.024,"5min":13.025,"15min":13.025},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.1110000000000002,"15min":2.161},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75900000000000001,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.60399999999999998}]}]},{"osd":0,"up_from":86,"seq":369367187571,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313180,"kb_used_data":1876,"kb_used_omap":5,"kb_used_meta":311290,"kb_avail":93431972,"statfs":{"total":95995035648,"available":95674339328,"internally_reserved":0,"allocated":1921024,"data_stored":1368213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":318761594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.994,"15min":2.0670000000000002},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.502,"5min":16.780999999999999,"15min":16.780999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.33400000000000002,"5min":3.2679999999999998,"15min":3.032},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48599999999999999,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.249}]},{"osd":2,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.441,"5min":0.441,"15min":0.441},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":2.4489999999999998,"15min":2.7400000000000002},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.247},"max":{"1min":5.6269999999999998,"5min":16.827000000000002,"15min":16.827000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.405,"5min":3.8439999999999999,"15min":4.8090000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.3780000000000001,"5min":22.053000000000001,"15min":22.379999999999999},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.093,"15min":2.2639999999999998},"min":{"1min":0.437,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1609999999999996,"5min":10.948,"15min":10.948},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.5780000000000001,"15min":2.0179999999999998},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":0.81799999999999995,"5min":8.8160000000000007,"15min":10.667},"last":0.68300000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.754,"15min":2.375},"min":{"1min":0.42699999999999999,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.86099999999999999,"5min":9.7119999999999997,"15min":10.693},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.5649999999999999,"15min":2.5840000000000001},"min":{"1min":0.442,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":10.964,"15min":17.460000000000001},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.873,"15min":2.4460000000000002},"min":{"1min":0.505,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":0.95799999999999996,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.6520000000000001,"15min":2.7029999999999998},"min":{"1min":0.503,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":0.94499999999999995,"5min":11.022,"15min":17.411000000000001},"last":0.748}]},{"osd":7,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.5390000000000001,"15min":2.7029999999999998},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.36399999999999999},"max":{"1min":0.81299999999999994,"5min":11.224,"15min":17.486999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.3100000000000001,"15min":2.194},"min":{"1min":0.53000000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82699999999999996,"5min":9.3960000000000008,"15min":10.867000000000001},"last":0.75900000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075537,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161944,"kb_used_data":1232,"kb_used_omap":7,"kb_used_meta":160696,"kb_avail":93583208,"statfs":{"total":95995035648,"available":95829204992,"internally_reserved":0,"allocated":1261568,"data_stored":769786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164552785},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.378}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28100000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.73099999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.77400000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.746}]}]},{"osd":3,"up_from":125,"seq":536870912012,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115404,"kb_used_data":1220,"kb_used_omap":2,"kb_used_meta":114173,"kb_avail":93629748,"statfs":{"total":95995035648,"available":95876861952,"internally_reserved":0,"allocated":1249280,"data_stored":768648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":116913205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019162,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333384,"kb_used_data":1216,"kb_used_omap":0,"kb_used_meta":332160,"kb_avail":93411768,"statfs":{"total":95995035648,"available":95653650432,"internally_reserved":0,"allocated":1245184,"data_stored":768658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340131840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.2869999999999999,"15min":2.0550000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":8.9269999999999996,"15min":10.01},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.48599999999999999,"5min":1.8520000000000001,"15min":2.298},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":0.71499999999999997,"5min":9.593,"15min":15.476000000000001},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.6890000000000001,"15min":1.819},"min":{"1min":0.36599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72199999999999998,"5min":15.757,"15min":15.757},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.8700000000000001,"15min":2.351},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.087,"5min":15.125,"15min":15.125},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.3690000000000002,"15min":3.2959999999999998},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":10.095000000000001,"5min":11.964,"15min":140.40799999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.4449999999999998,"5min":2.2970000000000002,"15min":3.2930000000000001},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.312},"max":{"1min":7.1849999999999996,"5min":9.5619999999999994,"15min":140.38200000000001},"last":0.502}]},{"osd":5,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.81899999999999995,"15min":1.2430000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":0.70399999999999996,"5min":6.5010000000000003,"15min":92.653000000000006},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.0649999999999999,"15min":1.532},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":0.83199999999999996,"5min":6.9539999999999997,"15min":93.281999999999996},"last":0.54100000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:19:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.97899999999999998,"15min":1.099},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.254},"max":{"1min":0.80200000000000005,"5min":5.867,"15min":8.3179999999999996},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.034,"15min":1.0720000000000001},"min":{"1min":0.41999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73799999999999999,"5min":8.8420000000000005,"15min":13.045999999999999},"last":0.64500000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:19:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.284,"15min":1.3440000000000001},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":0.74299999999999999,"5min":8.4459999999999997,"15min":19.989000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0880000000000001,"15min":1.1739999999999999},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.90400000000000003,"5min":5.4379999999999997,"15min":19.783000000000001},"last":0.62}]}]},{"osd":5,"up_from":36,"seq":154618822934,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387320,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":386112,"kb_avail":93357832,"statfs":{"total":95995035648,"available":95598419968,"internally_reserved":0,"allocated":1228800,"data_stored":764420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395378688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.3220000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.0139999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.85599999999999998,"5min":11.807,"15min":11.807},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.0550000000000002,"15min":2.0049999999999999},"min":{"1min":0.41999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":2.3690000000000002,"15min":2.6360000000000001},"min":{"1min":0.45100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":15.962999999999999,"15min":15.962999999999999},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59499999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.508,"15min":3.6869999999999998},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.444,"5min":13.186999999999999,"15min":237.95599999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.427,"5min":2.556,"15min":3.6200000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.317},"max":{"1min":7.508,"5min":14.259,"15min":237.96600000000001},"last":0.44500000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.1779999999999999,"15min":1.0229999999999999},"min":{"1min":0.32400000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":0.63,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.4279999999999999,"15min":1.2270000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.76500000000000001,"5min":12.901,"15min":20.356999999999999},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:20:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.107,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.72399999999999998,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0700000000000001,"15min":1.2430000000000001},"min":{"1min":0.44,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":0.84399999999999997,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:19:42 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.4279999999999999,"15min":1.365},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.78300000000000003,"5min":8.5109999999999992,"15min":9.5039999999999996},"last":0.501},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3280000000000001,"15min":1.1759999999999999},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.75600000000000001,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.58399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:20:20.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:20:20.379 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:21 smithi152 ceph-mon[156819]: from='client.35328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:21 smithi152 ceph-mon[156819]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:21 smithi064 ceph-mon[186706]: from='client.35328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:21 smithi064 ceph-mon[186706]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:21.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:21 smithi064 ceph-mon[180297]: from='client.35328 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:21.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:21 smithi064 ceph-mon[180297]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:22.086 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:20:22.087 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:20:22.788 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":496,"stamp":"2023-06-10T21:20:21.406549+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2112156,"kb_used_data":11612,"kb_used_omap":20,"kb_used_meta":2100459,"kb_avail":747849060,"statfs":{"total":767960285184,"available":765797437440,"internally_reserved":0,"allocated":11890688,"data_stored":7934249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2150870670},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003247"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471845+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:19:27.471845+0000","last_peered":"2023-06-10T21:19:27.471845+0000","last_clean":"2023-06-10T21:19:27.471845+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:19:27.471845+0000","last_undegraded":"2023-06-10T21:19:27.471845+0000","last_fullsized":"2023-06-10T21:19:27.471845+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.478832+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:19:28.478832+0000","last_peered":"2023-06-10T21:19:28.478832+0000","last_clean":"2023-06-10T21:19:28.478832+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:19:28.478832+0000","last_undegraded":"2023-06-10T21:19:28.478832+0000","last_fullsized":"2023-06-10T21:19:28.478832+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487021+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:19:28.487021+0000","last_peered":"2023-06-10T21:19:28.487021+0000","last_clean":"2023-06-10T21:19:28.487021+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:19:28.487021+0000","last_undegraded":"2023-06-10T21:19:28.487021+0000","last_fullsized":"2023-06-10T21:19:28.487021+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483727+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:19:28.483727+0000","last_peered":"2023-06-10T21:19:28.483727+0000","last_clean":"2023-06-10T21:19:28.483727+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:19:28.483727+0000","last_undegraded":"2023-06-10T21:19:28.483727+0000","last_fullsized":"2023-06-10T21:19:28.483727+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480757+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:19:28.480757+0000","last_peered":"2023-06-10T21:19:28.480757+0000","last_clean":"2023-06-10T21:19:28.480757+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:19:28.480757+0000","last_undegraded":"2023-06-10T21:19:28.480757+0000","last_fullsized":"2023-06-10T21:19:28.480757+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470669+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:19:27.470669+0000","last_peered":"2023-06-10T21:19:27.470669+0000","last_clean":"2023-06-10T21:19:27.470669+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:19:27.470669+0000","last_undegraded":"2023-06-10T21:19:27.470669+0000","last_fullsized":"2023-06-10T21:19:27.470669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486071+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:19:28.486071+0000","last_peered":"2023-06-10T21:19:28.486071+0000","last_clean":"2023-06-10T21:19:28.486071+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:19:28.486071+0000","last_undegraded":"2023-06-10T21:19:28.486071+0000","last_fullsized":"2023-06-10T21:19:28.486071+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480260+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:19:28.480260+0000","last_peered":"2023-06-10T21:19:28.480260+0000","last_clean":"2023-06-10T21:19:28.480260+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:19:28.480260+0000","last_undegraded":"2023-06-10T21:19:28.480260+0000","last_fullsized":"2023-06-10T21:19:28.480260+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476985+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:19:27.476985+0000","last_peered":"2023-06-10T21:19:27.476985+0000","last_clean":"2023-06-10T21:19:27.476985+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:19:27.476985+0000","last_undegraded":"2023-06-10T21:19:27.476985+0000","last_fullsized":"2023-06-10T21:19:27.476985+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483795+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.483795+0000","last_peered":"2023-06-10T21:19:28.483795+0000","last_clean":"2023-06-10T21:19:28.483795+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:19:28.483795+0000","last_undegraded":"2023-06-10T21:19:28.483795+0000","last_fullsized":"2023-06-10T21:19:28.483795+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483994+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:19:28.483994+0000","last_peered":"2023-06-10T21:19:28.483994+0000","last_clean":"2023-06-10T21:19:28.483994+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:19:28.483994+0000","last_undegraded":"2023-06-10T21:19:28.483994+0000","last_fullsized":"2023-06-10T21:19:28.483994+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474354+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:19:27.474354+0000","last_peered":"2023-06-10T21:19:27.474354+0000","last_clean":"2023-06-10T21:19:27.474354+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:19:27.474354+0000","last_undegraded":"2023-06-10T21:19:27.474354+0000","last_fullsized":"2023-06-10T21:19:27.474354+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472826+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:19:27.472826+0000","last_peered":"2023-06-10T21:19:27.472826+0000","last_clean":"2023-06-10T21:19:27.472826+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:19:27.472826+0000","last_undegraded":"2023-06-10T21:19:27.472826+0000","last_fullsized":"2023-06-10T21:19:27.472826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474064+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:19:27.474064+0000","last_peered":"2023-06-10T21:19:27.474064+0000","last_clean":"2023-06-10T21:19:27.474064+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:19:27.474064+0000","last_undegraded":"2023-06-10T21:19:27.474064+0000","last_fullsized":"2023-06-10T21:19:27.474064+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475311+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:19:27.475311+0000","last_peered":"2023-06-10T21:19:27.475311+0000","last_clean":"2023-06-10T21:19:27.475311+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:19:27.475311+0000","last_undegraded":"2023-06-10T21:19:27.475311+0000","last_fullsized":"2023-06-10T21:19:27.475311+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483638+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:19:28.483638+0000","last_peered":"2023-06-10T21:19:28.483638+0000","last_clean":"2023-06-10T21:19:28.483638+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:19:28.483638+0000","last_undegraded":"2023-06-10T21:19:28.483638+0000","last_fullsized":"2023-06-10T21:19:28.483638+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486854+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.486854+0000","last_peered":"2023-06-10T21:19:28.486854+0000","last_clean":"2023-06-10T21:19:28.486854+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:19:28.486854+0000","last_undegraded":"2023-06-10T21:19:28.486854+0000","last_fullsized":"2023-06-10T21:19:28.486854+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481386+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:19:28.481386+0000","last_peered":"2023-06-10T21:19:28.481386+0000","last_clean":"2023-06-10T21:19:28.481386+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:19:28.481386+0000","last_undegraded":"2023-06-10T21:19:28.481386+0000","last_fullsized":"2023-06-10T21:19:28.481386+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475349+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:19:27.475349+0000","last_peered":"2023-06-10T21:19:27.475349+0000","last_clean":"2023-06-10T21:19:27.475349+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:19:27.475349+0000","last_undegraded":"2023-06-10T21:19:27.475349+0000","last_fullsized":"2023-06-10T21:19:27.475349+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483808+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:19:28.483808+0000","last_peered":"2023-06-10T21:19:28.483808+0000","last_clean":"2023-06-10T21:19:28.483808+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:19:28.483808+0000","last_undegraded":"2023-06-10T21:19:28.483808+0000","last_fullsized":"2023-06-10T21:19:28.483808+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470529+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:19:27.470529+0000","last_peered":"2023-06-10T21:19:27.470529+0000","last_clean":"2023-06-10T21:19:27.470529+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:19:27.470529+0000","last_undegraded":"2023-06-10T21:19:27.470529+0000","last_fullsized":"2023-06-10T21:19:27.470529+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484436+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:19:28.484436+0000","last_peered":"2023-06-10T21:19:28.484436+0000","last_clean":"2023-06-10T21:19:28.484436+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:19:28.484436+0000","last_undegraded":"2023-06-10T21:19:28.484436+0000","last_fullsized":"2023-06-10T21:19:28.484436+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476812+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:19:27.476812+0000","last_peered":"2023-06-10T21:19:27.476812+0000","last_clean":"2023-06-10T21:19:27.476812+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:19:27.476812+0000","last_undegraded":"2023-06-10T21:19:27.476812+0000","last_fullsized":"2023-06-10T21:19:27.476812+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474154+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:19:27.474154+0000","last_peered":"2023-06-10T21:19:27.474154+0000","last_clean":"2023-06-10T21:19:27.474154+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:19:27.474154+0000","last_undegraded":"2023-06-10T21:19:27.474154+0000","last_fullsized":"2023-06-10T21:19:27.474154+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476603+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:19:27.476603+0000","last_peered":"2023-06-10T21:19:27.476603+0000","last_clean":"2023-06-10T21:19:27.476603+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:19:27.476603+0000","last_undegraded":"2023-06-10T21:19:27.476603+0000","last_fullsized":"2023-06-10T21:19:27.476603+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487906+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:19:28.487906+0000","last_peered":"2023-06-10T21:19:28.487906+0000","last_clean":"2023-06-10T21:19:28.487906+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:19:28.487906+0000","last_undegraded":"2023-06-10T21:19:28.487906+0000","last_fullsized":"2023-06-10T21:19:28.487906+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477579+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:19:27.477579+0000","last_peered":"2023-06-10T21:19:27.477579+0000","last_clean":"2023-06-10T21:19:27.477579+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:19:27.477579+0000","last_undegraded":"2023-06-10T21:19:27.477579+0000","last_fullsized":"2023-06-10T21:19:27.477579+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470358+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:19:27.470358+0000","last_peered":"2023-06-10T21:19:27.470358+0000","last_clean":"2023-06-10T21:19:27.470358+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:19:27.470358+0000","last_undegraded":"2023-06-10T21:19:27.470358+0000","last_fullsized":"2023-06-10T21:19:27.470358+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480945+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:19:28.480945+0000","last_peered":"2023-06-10T21:19:28.480945+0000","last_clean":"2023-06-10T21:19:28.480945+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:19:28.480945+0000","last_undegraded":"2023-06-10T21:19:28.480945+0000","last_fullsized":"2023-06-10T21:19:28.480945+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.477277+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:19:28.477277+0000","last_peered":"2023-06-10T21:19:28.477277+0000","last_clean":"2023-06-10T21:19:28.477277+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:19:28.477277+0000","last_undegraded":"2023-06-10T21:19:28.477277+0000","last_fullsized":"2023-06-10T21:19:28.477277+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481924+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:19:28.481924+0000","last_peered":"2023-06-10T21:19:28.481924+0000","last_clean":"2023-06-10T21:19:28.481924+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:19:28.481924+0000","last_undegraded":"2023-06-10T21:19:28.481924+0000","last_fullsized":"2023-06-10T21:19:28.481924+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.488064+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:19:28.488064+0000","last_peered":"2023-06-10T21:19:28.488064+0000","last_clean":"2023-06-10T21:19:28.488064+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:19:28.488064+0000","last_undegraded":"2023-06-10T21:19:28.488064+0000","last_fullsized":"2023-06-10T21:19:28.488064+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470197+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:19:27.470197+0000","last_peered":"2023-06-10T21:19:27.470197+0000","last_clean":"2023-06-10T21:19:27.470197+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:19:27.470197+0000","last_undegraded":"2023-06-10T21:19:27.470197+0000","last_fullsized":"2023-06-10T21:19:27.470197+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472148+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:19:27.472148+0000","last_peered":"2023-06-10T21:19:27.472148+0000","last_clean":"2023-06-10T21:19:27.472148+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:19:27.472148+0000","last_undegraded":"2023-06-10T21:19:27.472148+0000","last_fullsized":"2023-06-10T21:19:27.472148+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472225+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:19:27.472225+0000","last_peered":"2023-06-10T21:19:27.472225+0000","last_clean":"2023-06-10T21:19:27.472225+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:19:27.472225+0000","last_undegraded":"2023-06-10T21:19:27.472225+0000","last_fullsized":"2023-06-10T21:19:27.472225+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471728+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:19:27.471728+0000","last_peered":"2023-06-10T21:19:27.471728+0000","last_clean":"2023-06-10T21:19:27.471728+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:19:27.471728+0000","last_undegraded":"2023-06-10T21:19:27.471728+0000","last_fullsized":"2023-06-10T21:19:27.471728+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483541+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:19:28.483541+0000","last_peered":"2023-06-10T21:19:28.483541+0000","last_clean":"2023-06-10T21:19:28.483541+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:19:28.483541+0000","last_undegraded":"2023-06-10T21:19:28.483541+0000","last_fullsized":"2023-06-10T21:19:28.483541+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484999+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:19:28.484999+0000","last_peered":"2023-06-10T21:19:28.484999+0000","last_clean":"2023-06-10T21:19:28.484999+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:19:28.484999+0000","last_undegraded":"2023-06-10T21:19:28.484999+0000","last_fullsized":"2023-06-10T21:19:28.484999+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474695+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:19:27.474695+0000","last_peered":"2023-06-10T21:19:27.474695+0000","last_clean":"2023-06-10T21:19:27.474695+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:19:27.474695+0000","last_undegraded":"2023-06-10T21:19:27.474695+0000","last_fullsized":"2023-06-10T21:19:27.474695+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482192+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:19:28.482192+0000","last_peered":"2023-06-10T21:19:28.482192+0000","last_clean":"2023-06-10T21:19:28.482192+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:19:28.482192+0000","last_undegraded":"2023-06-10T21:19:28.482192+0000","last_fullsized":"2023-06-10T21:19:28.482192+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.480734+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:19:27.480734+0000","last_peered":"2023-06-10T21:19:27.480734+0000","last_clean":"2023-06-10T21:19:27.480734+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:19:27.480734+0000","last_undegraded":"2023-06-10T21:19:27.480734+0000","last_fullsized":"2023-06-10T21:19:27.480734+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475368+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:19:27.475368+0000","last_peered":"2023-06-10T21:19:27.475368+0000","last_clean":"2023-06-10T21:19:27.475368+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:19:27.475368+0000","last_undegraded":"2023-06-10T21:19:27.475368+0000","last_fullsized":"2023-06-10T21:19:27.475368+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474220+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:19:27.474220+0000","last_peered":"2023-06-10T21:19:27.474220+0000","last_clean":"2023-06-10T21:19:27.474220+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:19:27.474220+0000","last_undegraded":"2023-06-10T21:19:27.474220+0000","last_fullsized":"2023-06-10T21:19:27.474220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486084+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:19:28.486084+0000","last_peered":"2023-06-10T21:19:28.486084+0000","last_clean":"2023-06-10T21:19:28.486084+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:19:28.486084+0000","last_undegraded":"2023-06-10T21:19:28.486084+0000","last_fullsized":"2023-06-10T21:19:28.486084+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482444+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:19:28.482444+0000","last_peered":"2023-06-10T21:19:28.482444+0000","last_clean":"2023-06-10T21:19:28.482444+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:19:28.482444+0000","last_undegraded":"2023-06-10T21:19:28.482444+0000","last_fullsized":"2023-06-10T21:19:28.482444+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474223+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:19:27.474223+0000","last_peered":"2023-06-10T21:19:27.474223+0000","last_clean":"2023-06-10T21:19:27.474223+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:19:27.474223+0000","last_undegraded":"2023-06-10T21:19:27.474223+0000","last_fullsized":"2023-06-10T21:19:27.474223+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478558+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:19:27.478558+0000","last_peered":"2023-06-10T21:19:27.478558+0000","last_clean":"2023-06-10T21:19:27.478558+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:19:27.478558+0000","last_undegraded":"2023-06-10T21:19:27.478558+0000","last_fullsized":"2023-06-10T21:19:27.478558+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475296+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:19:27.475296+0000","last_peered":"2023-06-10T21:19:27.475296+0000","last_clean":"2023-06-10T21:19:27.475296+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:19:27.475296+0000","last_undegraded":"2023-06-10T21:19:27.475296+0000","last_fullsized":"2023-06-10T21:19:27.475296+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":329,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475365+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:19:27.475365+0000","last_peered":"2023-06-10T21:19:27.475365+0000","last_clean":"2023-06-10T21:19:27.475365+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:19:27.475365+0000","last_undegraded":"2023-06-10T21:19:27.475365+0000","last_fullsized":"2023-06-10T21:19:27.475365+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481502+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:19:28.481502+0000","last_peered":"2023-06-10T21:19:28.481502+0000","last_clean":"2023-06-10T21:19:28.481502+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:19:28.481502+0000","last_undegraded":"2023-06-10T21:19:28.481502+0000","last_fullsized":"2023-06-10T21:19:28.481502+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470465+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:19:27.470465+0000","last_peered":"2023-06-10T21:19:27.470465+0000","last_clean":"2023-06-10T21:19:27.470465+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:19:27.470465+0000","last_undegraded":"2023-06-10T21:19:27.470465+0000","last_fullsized":"2023-06-10T21:19:27.470465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475272+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:19:27.475272+0000","last_peered":"2023-06-10T21:19:27.475272+0000","last_clean":"2023-06-10T21:19:27.475272+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:19:27.475272+0000","last_undegraded":"2023-06-10T21:19:27.475272+0000","last_fullsized":"2023-06-10T21:19:27.475272+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487756+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:19:28.487756+0000","last_peered":"2023-06-10T21:19:28.487756+0000","last_clean":"2023-06-10T21:19:28.487756+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:19:28.487756+0000","last_undegraded":"2023-06-10T21:19:28.487756+0000","last_fullsized":"2023-06-10T21:19:28.487756+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686568+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:19:27.686568+0000","last_peered":"2023-06-10T21:19:27.686568+0000","last_clean":"2023-06-10T21:19:27.686568+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:19:27.686568+0000","last_undegraded":"2023-06-10T21:19:27.686568+0000","last_fullsized":"2023-06-10T21:19:27.686568+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477662+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:19:27.477662+0000","last_peered":"2023-06-10T21:19:27.477662+0000","last_clean":"2023-06-10T21:19:27.477662+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:19:27.477662+0000","last_undegraded":"2023-06-10T21:19:27.477662+0000","last_fullsized":"2023-06-10T21:19:27.477662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475077+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:19:27.475077+0000","last_peered":"2023-06-10T21:19:27.475077+0000","last_clean":"2023-06-10T21:19:27.475077+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:19:27.475077+0000","last_undegraded":"2023-06-10T21:19:27.475077+0000","last_fullsized":"2023-06-10T21:19:27.475077+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482778+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:19:27.482778+0000","last_peered":"2023-06-10T21:19:27.482778+0000","last_clean":"2023-06-10T21:19:27.482778+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:19:27.482778+0000","last_undegraded":"2023-06-10T21:19:27.482778+0000","last_fullsized":"2023-06-10T21:19:27.482778+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484610+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:19:28.484610+0000","last_peered":"2023-06-10T21:19:28.484610+0000","last_clean":"2023-06-10T21:19:28.484610+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:19:28.484610+0000","last_undegraded":"2023-06-10T21:19:28.484610+0000","last_fullsized":"2023-06-10T21:19:28.484610+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":86,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475914+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:19:27.475914+0000","last_peered":"2023-06-10T21:19:27.475914+0000","last_clean":"2023-06-10T21:19:27.475914+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:19:27.475914+0000","last_undegraded":"2023-06-10T21:19:27.475914+0000","last_fullsized":"2023-06-10T21:19:27.475914+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487200+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:19:28.487200+0000","last_peered":"2023-06-10T21:19:28.487200+0000","last_clean":"2023-06-10T21:19:28.487200+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:19:28.487200+0000","last_undegraded":"2023-06-10T21:19:28.487200+0000","last_fullsized":"2023-06-10T21:19:28.487200+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470586+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:19:27.470586+0000","last_peered":"2023-06-10T21:19:27.470586+0000","last_clean":"2023-06-10T21:19:27.470586+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:19:27.470586+0000","last_undegraded":"2023-06-10T21:19:27.470586+0000","last_fullsized":"2023-06-10T21:19:27.470586+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686543+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:19:27.686543+0000","last_peered":"2023-06-10T21:19:27.686543+0000","last_clean":"2023-06-10T21:19:27.686543+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:19:27.686543+0000","last_undegraded":"2023-06-10T21:19:27.686543+0000","last_fullsized":"2023-06-10T21:19:27.686543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472930+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:19:27.472930+0000","last_peered":"2023-06-10T21:19:27.472930+0000","last_clean":"2023-06-10T21:19:27.472930+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:19:27.472930+0000","last_undegraded":"2023-06-10T21:19:27.472930+0000","last_fullsized":"2023-06-10T21:19:27.472930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":68,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482973+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:19:27.482973+0000","last_peered":"2023-06-10T21:19:27.482973+0000","last_clean":"2023-06-10T21:19:27.482973+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:19:27.482973+0000","last_undegraded":"2023-06-10T21:19:27.482973+0000","last_fullsized":"2023-06-10T21:19:27.482973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482329+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:19:28.482329+0000","last_peered":"2023-06-10T21:19:28.482329+0000","last_clean":"2023-06-10T21:19:28.482329+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:19:28.482329+0000","last_undegraded":"2023-06-10T21:19:28.482329+0000","last_fullsized":"2023-06-10T21:19:28.482329+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478090+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:19:27.478090+0000","last_peered":"2023-06-10T21:19:27.478090+0000","last_clean":"2023-06-10T21:19:27.478090+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:19:27.478090+0000","last_undegraded":"2023-06-10T21:19:27.478090+0000","last_fullsized":"2023-06-10T21:19:27.478090+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487998+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:19:28.487998+0000","last_peered":"2023-06-10T21:19:28.487998+0000","last_clean":"2023-06-10T21:19:28.487998+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:19:28.487998+0000","last_undegraded":"2023-06-10T21:19:28.487998+0000","last_fullsized":"2023-06-10T21:19:28.487998+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477850+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:19:27.477850+0000","last_peered":"2023-06-10T21:19:27.477850+0000","last_clean":"2023-06-10T21:19:27.477850+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:19:27.477850+0000","last_undegraded":"2023-06-10T21:19:27.477850+0000","last_fullsized":"2023-06-10T21:19:27.477850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482280+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:19:28.482280+0000","last_peered":"2023-06-10T21:19:28.482280+0000","last_clean":"2023-06-10T21:19:28.482280+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:19:28.482280+0000","last_undegraded":"2023-06-10T21:19:28.482280+0000","last_fullsized":"2023-06-10T21:19:28.482280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482461+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:19:28.482461+0000","last_peered":"2023-06-10T21:19:28.482461+0000","last_clean":"2023-06-10T21:19:28.482461+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:19:28.482461+0000","last_undegraded":"2023-06-10T21:19:28.482461+0000","last_fullsized":"2023-06-10T21:19:28.482461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481801+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:19:28.481801+0000","last_peered":"2023-06-10T21:19:28.481801+0000","last_clean":"2023-06-10T21:19:28.481801+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:19:28.481801+0000","last_undegraded":"2023-06-10T21:19:28.481801+0000","last_fullsized":"2023-06-10T21:19:28.481801+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482857+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:19:27.482857+0000","last_peered":"2023-06-10T21:19:27.482857+0000","last_clean":"2023-06-10T21:19:27.482857+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:19:27.482857+0000","last_undegraded":"2023-06-10T21:19:27.482857+0000","last_fullsized":"2023-06-10T21:19:27.482857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471651+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:19:27.471651+0000","last_peered":"2023-06-10T21:19:27.471651+0000","last_clean":"2023-06-10T21:19:27.471651+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:19:27.471651+0000","last_undegraded":"2023-06-10T21:19:27.471651+0000","last_fullsized":"2023-06-10T21:19:27.471651+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430479,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305940,"kb_used_data":1804,"kb_used_omap":0,"kb_used_meta":304128,"kb_avail":93439212,"statfs":{"total":95995035648,"available":95681753088,"internally_reserved":0,"allocated":1847296,"data_stored":1359411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":311427072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.258,"15min":1.3480000000000001},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.68899999999999995,"5min":10.914999999999999,"15min":10.914999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.96,"15min":1.95},"min":{"1min":0.41999999999999998,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":0.75600000000000001,"5min":15.925000000000001,"15min":15.925000000000001},"last":0.499}]},{"osd":1,"last update":"Sat Jun 10 21:19:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.855,"15min":1.5369999999999999},"min":{"1min":0.40300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77900000000000003,"5min":10.083,"15min":10.083},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.5600000000000001,"15min":1.52},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.751,"5min":7,"15min":7},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Jun 10 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":2.7200000000000002,"15min":2.8069999999999999},"min":{"1min":0.51200000000000001,"5min":0.35099999999999998,"15min":0.29699999999999999},"max":{"1min":15.127000000000001,"5min":15.127000000000001,"15min":15.127000000000001},"last":0.628},{"interface":"front","average":{"1min":3.246,"5min":2.556,"15min":2.7149999999999999},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":15.275,"5min":15.275,"15min":15.275},"last":0.442}]},{"osd":4,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.1579999999999999,"15min":2.0960000000000001},"min":{"1min":0.309,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.67000000000000004,"5min":8.0830000000000002,"15min":219.17699999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4219999999999999,"15min":2.2469999999999999},"min":{"1min":0.34300000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.77200000000000002,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.522,"15min":2.7719999999999998},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.254},"max":{"1min":0.77100000000000002,"5min":37.759,"15min":311.202},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4990000000000001,"15min":2.6040000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.29699999999999999},"max":{"1min":0.72899999999999998,"5min":30.074999999999999,"15min":311.875},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.2130000000000001,"15min":1.2769999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78500000000000003,"5min":6.8570000000000002,"15min":15.58},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.01,"15min":1.1259999999999999},"min":{"1min":0.46200000000000002,"5min":0.33400000000000002,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":7.6900000000000004,"15min":8.8829999999999991},"last":0.59699999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626707,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281028,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":279232,"kb_avail":93464124,"statfs":{"total":95995035648,"available":95707262976,"internally_reserved":0,"allocated":1830912,"data_stored":1358950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285933568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.4059999999999999,"15min":2.1970000000000001},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":0.80200000000000005,"5min":9.7929999999999993,"15min":119.276},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.704,"15min":2.8410000000000002},"min":{"1min":0.378,"5min":0.252,"15min":0.252},"max":{"1min":0.73099999999999998,"5min":16.324999999999999,"15min":119.22},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.788,"15min":1.823},"min":{"1min":0.37,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70399999999999996,"5min":14.897,"15min":14.897},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.9910000000000001,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.84899999999999998,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.68300000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:18:17 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.891,"15min":2.7789999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29099999999999998},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":74.248000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.6659999999999999,"15min":2.9849999999999999},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":7.3029999999999999,"5min":15.488,"15min":74.314999999999998},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:19:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.016,"15min":1.8500000000000001},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.98799999999999999,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.3140000000000001,"15min":2.1040000000000001},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.745,"5min":11.263999999999999,"15min":221.905},"last":0.67300000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:19:21 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.0780000000000001,"15min":2.3639999999999999},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.879,"5min":6.6680000000000001,"15min":313.661},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.56499999999999995,"5min":1.2609999999999999,"15min":2.488},"min":{"1min":0.35399999999999998,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.745,"5min":9.75,"15min":314.38900000000001},"last":0.629}]},{"osd":7,"last update":"Sat Jun 10 21:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.2689999999999999,"15min":1.1579999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.81100000000000005,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.1859999999999999,"15min":1.2130000000000001},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":0.78500000000000003,"5min":8.0980000000000008,"15min":12.583},"last":0.65400000000000003}]}]},{"osd":1,"up_from":105,"seq":450971566150,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213956,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":212667,"kb_avail":93531196,"statfs":{"total":95995035648,"available":95775944704,"internally_reserved":0,"allocated":1306624,"data_stored":776163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":217771918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":1.9319999999999999,"15min":1.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.49099999999999999,"5min":10.898,"15min":10.898},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.4420000000000002,"15min":2.383},"min":{"1min":0.315,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.843,"5min":16.488,"15min":16.488},"last":0.379}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.895,"5min":1.895,"15min":1.895},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.927,"15min":3.1629999999999998},"min":{"1min":0.32400000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.19,"5min":13.349,"15min":13.349},"last":0.308},{"interface":"front","average":{"1min":2.0030000000000001,"5min":2.9740000000000002,"15min":2.4500000000000002},"min":{"1min":0.35499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.7530000000000001,"5min":22.033999999999999,"15min":22.033999999999999},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.8400000000000001,"15min":1.9299999999999999},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.9550000000000001,"5min":12.94,"15min":12.94},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.008,"15min":2.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.83299999999999996,"5min":12.603,"15min":12.603},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.2050000000000001,"15min":2.1720000000000002},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.9529999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.2730000000000001,"15min":2.5},"min":{"1min":0.45600000000000002,"5min":0.309,"15min":0.309},"max":{"1min":1.9870000000000001,"5min":13.602,"15min":13.602},"last":0.72299999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.2050000000000001,"15min":2.2130000000000001},"min":{"1min":0.504,"5min":0.379,"15min":0.379},"max":{"1min":3.0169999999999999,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.752,"5min":2.2029999999999998,"15min":2.524},"min":{"1min":0.47599999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.004,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4180000000000001,"15min":3.2530000000000001},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.024,"5min":13.025,"15min":13.025},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.1110000000000002,"15min":2.161},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75900000000000001,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.63500000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187572,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313180,"kb_used_data":1876,"kb_used_omap":5,"kb_used_meta":311290,"kb_avail":93431972,"statfs":{"total":95995035648,"available":95674339328,"internally_reserved":0,"allocated":1921024,"data_stored":1368213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":318761594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.994,"15min":2.0670000000000002},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.502,"5min":16.780999999999999,"15min":16.780999999999999},"last":0.311},{"interface":"front","average":{"1min":0.33400000000000002,"5min":3.2679999999999998,"15min":3.032},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48599999999999999,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.441,"5min":0.441,"15min":0.441},"last":0.251},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":2.4489999999999998,"15min":2.7400000000000002},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.247},"max":{"1min":5.6269999999999998,"5min":16.827000000000002,"15min":16.827000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.405,"5min":3.8439999999999999,"15min":4.8090000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.3780000000000001,"5min":22.053000000000001,"15min":22.379999999999999},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.093,"15min":2.2639999999999998},"min":{"1min":0.437,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1609999999999996,"5min":10.948,"15min":10.948},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.5780000000000001,"15min":2.0179999999999998},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":0.81799999999999995,"5min":8.8160000000000007,"15min":10.667},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.754,"15min":2.375},"min":{"1min":0.42699999999999999,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.86099999999999999,"5min":9.7119999999999997,"15min":10.693},"last":0.622},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.5649999999999999,"15min":2.5840000000000001},"min":{"1min":0.442,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":10.964,"15min":17.460000000000001},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.873,"15min":2.4460000000000002},"min":{"1min":0.505,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":0.95799999999999996,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.6520000000000001,"15min":2.7029999999999998},"min":{"1min":0.503,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":0.94499999999999995,"5min":11.022,"15min":17.411000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.5390000000000001,"15min":2.7029999999999998},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.36399999999999999},"max":{"1min":0.81299999999999994,"5min":11.224,"15min":17.486999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.3100000000000001,"15min":2.194},"min":{"1min":0.53000000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82699999999999996,"5min":9.3960000000000008,"15min":10.867000000000001},"last":0.59499999999999997}]}]},{"osd":2,"up_from":120,"seq":515396075538,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161944,"kb_used_data":1232,"kb_used_omap":7,"kb_used_meta":160696,"kb_avail":93583208,"statfs":{"total":95995035648,"available":95829204992,"internally_reserved":0,"allocated":1261568,"data_stored":769786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164552785},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.26200000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.307},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.42899999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.61699999999999999}]}]},{"osd":3,"up_from":125,"seq":536870912012,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115404,"kb_used_data":1220,"kb_used_omap":2,"kb_used_meta":114173,"kb_avail":93629748,"statfs":{"total":95995035648,"available":95876861952,"internally_reserved":0,"allocated":1249280,"data_stored":768648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":116913205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41299999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64400000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65700000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019163,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333384,"kb_used_data":1216,"kb_used_omap":0,"kb_used_meta":332160,"kb_avail":93411768,"statfs":{"total":95995035648,"available":95653650432,"internally_reserved":0,"allocated":1245184,"data_stored":768658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340131840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.2869999999999999,"15min":2.0550000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":8.9269999999999996,"15min":10.01},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":1.8520000000000001,"15min":2.298},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":0.71499999999999997,"5min":9.593,"15min":15.476000000000001},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.6890000000000001,"15min":1.819},"min":{"1min":0.36599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72199999999999998,"5min":15.757,"15min":15.757},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.8700000000000001,"15min":2.351},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.087,"5min":15.125,"15min":15.125},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.61099999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.3690000000000002,"15min":3.2959999999999998},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":10.095000000000001,"5min":11.964,"15min":140.40799999999999},"last":0.498},{"interface":"front","average":{"1min":2.4449999999999998,"5min":2.2970000000000002,"15min":3.2930000000000001},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.312},"max":{"1min":7.1849999999999996,"5min":9.5619999999999994,"15min":140.38200000000001},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.81899999999999995,"15min":1.2430000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":0.70399999999999996,"5min":6.5010000000000003,"15min":92.653000000000006},"last":0.755},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.0649999999999999,"15min":1.532},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":0.83199999999999996,"5min":6.9539999999999997,"15min":93.281999999999996},"last":0.63}]},{"osd":6,"last update":"Sat Jun 10 21:19:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.97899999999999998,"15min":1.099},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.254},"max":{"1min":0.80200000000000005,"5min":5.867,"15min":8.3179999999999996},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.034,"15min":1.0720000000000001},"min":{"1min":0.41999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73799999999999999,"5min":8.8420000000000005,"15min":13.045999999999999},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 10 21:19:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.284,"15min":1.3440000000000001},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":0.74299999999999999,"5min":8.4459999999999997,"15min":19.989000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0880000000000001,"15min":1.1739999999999999},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.90400000000000003,"5min":5.4379999999999997,"15min":19.783000000000001},"last":0.59599999999999997}]}]},{"osd":5,"up_from":36,"seq":154618822935,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387320,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":386112,"kb_avail":93357832,"statfs":{"total":95995035648,"available":95598419968,"internally_reserved":0,"allocated":1228800,"data_stored":764420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395378688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.3220000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.0139999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.85599999999999998,"5min":11.807,"15min":11.807},"last":0.72199999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.0550000000000002,"15min":2.0049999999999999},"min":{"1min":0.41999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":2.3690000000000002,"15min":2.6360000000000001},"min":{"1min":0.45100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":15.962999999999999,"15min":15.962999999999999},"last":0.629}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.67300000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.508,"15min":3.6869999999999998},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.444,"5min":13.186999999999999,"15min":237.95599999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.427,"5min":2.556,"15min":3.6200000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.317},"max":{"1min":7.508,"5min":14.259,"15min":237.96600000000001},"last":0.64200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.1779999999999999,"15min":1.0229999999999999},"min":{"1min":0.32400000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":0.63,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.499},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.4279999999999999,"15min":1.2270000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.76500000000000001,"5min":12.901,"15min":20.356999999999999},"last":0.51300000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:20:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.107,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.72399999999999998,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0700000000000001,"15min":1.2430000000000001},"min":{"1min":0.44,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":0.84399999999999997,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:19:42 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.4279999999999999,"15min":1.365},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.78300000000000003,"5min":8.5109999999999992,"15min":9.5039999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3280000000000001,"15min":1.1759999999999999},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.75600000000000001,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.69699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:20:22.795 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-10T21:20:22.796 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-10T21:20:22.796 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-10T21:20:22.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph health --format=json 2023-06-10T21:20:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:22] "GET /metrics HTTP/1.1" 200 220626 "" "Prometheus/2.33.4" 2023-06-10T21:20:23.162 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:23 smithi152 ceph-mon[156819]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:23 smithi152 ceph-mon[156819]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:23.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:23 smithi064 ceph-mon[186706]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:23.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:23 smithi064 ceph-mon[186706]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:23.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:23 smithi064 ceph-mon[180297]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:23.451 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:23 smithi064 ceph-mon[180297]: from='client.35334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:24.887 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:20:24.887 INFO:teuthology.orchestra.run.smithi064.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-10T21:20:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:25 smithi064 ceph-mon[180297]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1300839347' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:20:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:25 smithi064 ceph-mon[186706]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1300839347' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:20:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:25 smithi152 ceph-mon[156819]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1300839347' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-10T21:20:25.729 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-10T21:20:25.730 INFO:teuthology.run_tasks:Running task print... 2023-06-10T21:20:25.741 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-06-10T21:20:25.741 INFO:teuthology.run_tasks:Running task parallel... 2023-06-10T21:20:25.758 INFO:teuthology.task.parallel:starting parallel... 2023-06-10T21:20:25.758 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-10T21:20:25.759 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-06-10T21:20:25.761 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-06-10T21:20:25.761 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': '5937c68aed42cde57f938ad8f8bb264f13176009'}} 2023-06-10T21:20:25.761 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-06-10T21:20:25.762 INFO:tasks.thrashosds:Beginning thrashosds... 2023-06-10T21:20:25.762 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-10T21:20:25.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:20:26.125 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:27.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:27 smithi064 ceph-mon[180297]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:27.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:27 smithi064 ceph-mon[186706]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:27 smithi152 ceph-mon[156819]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:27.924 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:20:27.925 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:20:28.773 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":499,"stamp":"2023-06-10T21:20:27.408008+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2112156,"kb_used_data":11612,"kb_used_omap":20,"kb_used_meta":2100459,"kb_avail":747849060,"statfs":{"total":767960285184,"available":765797437440,"internally_reserved":0,"allocated":11890688,"data_stored":7934249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2150870670},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003078"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471845+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:19:27.471845+0000","last_peered":"2023-06-10T21:19:27.471845+0000","last_clean":"2023-06-10T21:19:27.471845+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:19:27.471845+0000","last_undegraded":"2023-06-10T21:19:27.471845+0000","last_fullsized":"2023-06-10T21:19:27.471845+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.478832+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:19:28.478832+0000","last_peered":"2023-06-10T21:19:28.478832+0000","last_clean":"2023-06-10T21:19:28.478832+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:19:28.478832+0000","last_undegraded":"2023-06-10T21:19:28.478832+0000","last_fullsized":"2023-06-10T21:19:28.478832+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487021+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:19:28.487021+0000","last_peered":"2023-06-10T21:19:28.487021+0000","last_clean":"2023-06-10T21:19:28.487021+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:19:28.487021+0000","last_undegraded":"2023-06-10T21:19:28.487021+0000","last_fullsized":"2023-06-10T21:19:28.487021+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483727+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:19:28.483727+0000","last_peered":"2023-06-10T21:19:28.483727+0000","last_clean":"2023-06-10T21:19:28.483727+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:19:28.483727+0000","last_undegraded":"2023-06-10T21:19:28.483727+0000","last_fullsized":"2023-06-10T21:19:28.483727+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480757+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:19:28.480757+0000","last_peered":"2023-06-10T21:19:28.480757+0000","last_clean":"2023-06-10T21:19:28.480757+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:19:28.480757+0000","last_undegraded":"2023-06-10T21:19:28.480757+0000","last_fullsized":"2023-06-10T21:19:28.480757+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470669+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:19:27.470669+0000","last_peered":"2023-06-10T21:19:27.470669+0000","last_clean":"2023-06-10T21:19:27.470669+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:19:27.470669+0000","last_undegraded":"2023-06-10T21:19:27.470669+0000","last_fullsized":"2023-06-10T21:19:27.470669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486071+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:19:28.486071+0000","last_peered":"2023-06-10T21:19:28.486071+0000","last_clean":"2023-06-10T21:19:28.486071+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:19:28.486071+0000","last_undegraded":"2023-06-10T21:19:28.486071+0000","last_fullsized":"2023-06-10T21:19:28.486071+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480260+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:19:28.480260+0000","last_peered":"2023-06-10T21:19:28.480260+0000","last_clean":"2023-06-10T21:19:28.480260+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:19:28.480260+0000","last_undegraded":"2023-06-10T21:19:28.480260+0000","last_fullsized":"2023-06-10T21:19:28.480260+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476985+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:19:27.476985+0000","last_peered":"2023-06-10T21:19:27.476985+0000","last_clean":"2023-06-10T21:19:27.476985+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:19:27.476985+0000","last_undegraded":"2023-06-10T21:19:27.476985+0000","last_fullsized":"2023-06-10T21:19:27.476985+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483795+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.483795+0000","last_peered":"2023-06-10T21:19:28.483795+0000","last_clean":"2023-06-10T21:19:28.483795+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:19:28.483795+0000","last_undegraded":"2023-06-10T21:19:28.483795+0000","last_fullsized":"2023-06-10T21:19:28.483795+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483994+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:19:28.483994+0000","last_peered":"2023-06-10T21:19:28.483994+0000","last_clean":"2023-06-10T21:19:28.483994+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:19:28.483994+0000","last_undegraded":"2023-06-10T21:19:28.483994+0000","last_fullsized":"2023-06-10T21:19:28.483994+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474354+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:19:27.474354+0000","last_peered":"2023-06-10T21:19:27.474354+0000","last_clean":"2023-06-10T21:19:27.474354+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:19:27.474354+0000","last_undegraded":"2023-06-10T21:19:27.474354+0000","last_fullsized":"2023-06-10T21:19:27.474354+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472826+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:19:27.472826+0000","last_peered":"2023-06-10T21:19:27.472826+0000","last_clean":"2023-06-10T21:19:27.472826+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:19:27.472826+0000","last_undegraded":"2023-06-10T21:19:27.472826+0000","last_fullsized":"2023-06-10T21:19:27.472826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474064+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:19:27.474064+0000","last_peered":"2023-06-10T21:19:27.474064+0000","last_clean":"2023-06-10T21:19:27.474064+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:19:27.474064+0000","last_undegraded":"2023-06-10T21:19:27.474064+0000","last_fullsized":"2023-06-10T21:19:27.474064+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475311+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:19:27.475311+0000","last_peered":"2023-06-10T21:19:27.475311+0000","last_clean":"2023-06-10T21:19:27.475311+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:19:27.475311+0000","last_undegraded":"2023-06-10T21:19:27.475311+0000","last_fullsized":"2023-06-10T21:19:27.475311+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483638+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:19:28.483638+0000","last_peered":"2023-06-10T21:19:28.483638+0000","last_clean":"2023-06-10T21:19:28.483638+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:19:28.483638+0000","last_undegraded":"2023-06-10T21:19:28.483638+0000","last_fullsized":"2023-06-10T21:19:28.483638+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486854+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.486854+0000","last_peered":"2023-06-10T21:19:28.486854+0000","last_clean":"2023-06-10T21:19:28.486854+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:19:28.486854+0000","last_undegraded":"2023-06-10T21:19:28.486854+0000","last_fullsized":"2023-06-10T21:19:28.486854+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481386+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:19:28.481386+0000","last_peered":"2023-06-10T21:19:28.481386+0000","last_clean":"2023-06-10T21:19:28.481386+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:19:28.481386+0000","last_undegraded":"2023-06-10T21:19:28.481386+0000","last_fullsized":"2023-06-10T21:19:28.481386+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475349+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:19:27.475349+0000","last_peered":"2023-06-10T21:19:27.475349+0000","last_clean":"2023-06-10T21:19:27.475349+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:19:27.475349+0000","last_undegraded":"2023-06-10T21:19:27.475349+0000","last_fullsized":"2023-06-10T21:19:27.475349+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483808+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:19:28.483808+0000","last_peered":"2023-06-10T21:19:28.483808+0000","last_clean":"2023-06-10T21:19:28.483808+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:19:28.483808+0000","last_undegraded":"2023-06-10T21:19:28.483808+0000","last_fullsized":"2023-06-10T21:19:28.483808+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470529+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:19:27.470529+0000","last_peered":"2023-06-10T21:19:27.470529+0000","last_clean":"2023-06-10T21:19:27.470529+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:19:27.470529+0000","last_undegraded":"2023-06-10T21:19:27.470529+0000","last_fullsized":"2023-06-10T21:19:27.470529+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484436+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:19:28.484436+0000","last_peered":"2023-06-10T21:19:28.484436+0000","last_clean":"2023-06-10T21:19:28.484436+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:19:28.484436+0000","last_undegraded":"2023-06-10T21:19:28.484436+0000","last_fullsized":"2023-06-10T21:19:28.484436+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476812+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:19:27.476812+0000","last_peered":"2023-06-10T21:19:27.476812+0000","last_clean":"2023-06-10T21:19:27.476812+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:19:27.476812+0000","last_undegraded":"2023-06-10T21:19:27.476812+0000","last_fullsized":"2023-06-10T21:19:27.476812+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474154+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:19:27.474154+0000","last_peered":"2023-06-10T21:19:27.474154+0000","last_clean":"2023-06-10T21:19:27.474154+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:19:27.474154+0000","last_undegraded":"2023-06-10T21:19:27.474154+0000","last_fullsized":"2023-06-10T21:19:27.474154+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476603+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:19:27.476603+0000","last_peered":"2023-06-10T21:19:27.476603+0000","last_clean":"2023-06-10T21:19:27.476603+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:19:27.476603+0000","last_undegraded":"2023-06-10T21:19:27.476603+0000","last_fullsized":"2023-06-10T21:19:27.476603+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487906+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:19:28.487906+0000","last_peered":"2023-06-10T21:19:28.487906+0000","last_clean":"2023-06-10T21:19:28.487906+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:19:28.487906+0000","last_undegraded":"2023-06-10T21:19:28.487906+0000","last_fullsized":"2023-06-10T21:19:28.487906+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477579+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:19:27.477579+0000","last_peered":"2023-06-10T21:19:27.477579+0000","last_clean":"2023-06-10T21:19:27.477579+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:19:27.477579+0000","last_undegraded":"2023-06-10T21:19:27.477579+0000","last_fullsized":"2023-06-10T21:19:27.477579+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470358+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:19:27.470358+0000","last_peered":"2023-06-10T21:19:27.470358+0000","last_clean":"2023-06-10T21:19:27.470358+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:19:27.470358+0000","last_undegraded":"2023-06-10T21:19:27.470358+0000","last_fullsized":"2023-06-10T21:19:27.470358+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480945+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:19:28.480945+0000","last_peered":"2023-06-10T21:19:28.480945+0000","last_clean":"2023-06-10T21:19:28.480945+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:19:28.480945+0000","last_undegraded":"2023-06-10T21:19:28.480945+0000","last_fullsized":"2023-06-10T21:19:28.480945+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.477277+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:19:28.477277+0000","last_peered":"2023-06-10T21:19:28.477277+0000","last_clean":"2023-06-10T21:19:28.477277+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:19:28.477277+0000","last_undegraded":"2023-06-10T21:19:28.477277+0000","last_fullsized":"2023-06-10T21:19:28.477277+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481924+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:19:28.481924+0000","last_peered":"2023-06-10T21:19:28.481924+0000","last_clean":"2023-06-10T21:19:28.481924+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:19:28.481924+0000","last_undegraded":"2023-06-10T21:19:28.481924+0000","last_fullsized":"2023-06-10T21:19:28.481924+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.488064+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:19:28.488064+0000","last_peered":"2023-06-10T21:19:28.488064+0000","last_clean":"2023-06-10T21:19:28.488064+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:19:28.488064+0000","last_undegraded":"2023-06-10T21:19:28.488064+0000","last_fullsized":"2023-06-10T21:19:28.488064+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470197+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:19:27.470197+0000","last_peered":"2023-06-10T21:19:27.470197+0000","last_clean":"2023-06-10T21:19:27.470197+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:19:27.470197+0000","last_undegraded":"2023-06-10T21:19:27.470197+0000","last_fullsized":"2023-06-10T21:19:27.470197+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472148+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:19:27.472148+0000","last_peered":"2023-06-10T21:19:27.472148+0000","last_clean":"2023-06-10T21:19:27.472148+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:19:27.472148+0000","last_undegraded":"2023-06-10T21:19:27.472148+0000","last_fullsized":"2023-06-10T21:19:27.472148+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472225+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:19:27.472225+0000","last_peered":"2023-06-10T21:19:27.472225+0000","last_clean":"2023-06-10T21:19:27.472225+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:19:27.472225+0000","last_undegraded":"2023-06-10T21:19:27.472225+0000","last_fullsized":"2023-06-10T21:19:27.472225+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471728+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:19:27.471728+0000","last_peered":"2023-06-10T21:19:27.471728+0000","last_clean":"2023-06-10T21:19:27.471728+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:19:27.471728+0000","last_undegraded":"2023-06-10T21:19:27.471728+0000","last_fullsized":"2023-06-10T21:19:27.471728+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483541+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:19:28.483541+0000","last_peered":"2023-06-10T21:19:28.483541+0000","last_clean":"2023-06-10T21:19:28.483541+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:19:28.483541+0000","last_undegraded":"2023-06-10T21:19:28.483541+0000","last_fullsized":"2023-06-10T21:19:28.483541+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484999+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:19:28.484999+0000","last_peered":"2023-06-10T21:19:28.484999+0000","last_clean":"2023-06-10T21:19:28.484999+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:19:28.484999+0000","last_undegraded":"2023-06-10T21:19:28.484999+0000","last_fullsized":"2023-06-10T21:19:28.484999+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474695+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:19:27.474695+0000","last_peered":"2023-06-10T21:19:27.474695+0000","last_clean":"2023-06-10T21:19:27.474695+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:19:27.474695+0000","last_undegraded":"2023-06-10T21:19:27.474695+0000","last_fullsized":"2023-06-10T21:19:27.474695+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482192+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:19:28.482192+0000","last_peered":"2023-06-10T21:19:28.482192+0000","last_clean":"2023-06-10T21:19:28.482192+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:19:28.482192+0000","last_undegraded":"2023-06-10T21:19:28.482192+0000","last_fullsized":"2023-06-10T21:19:28.482192+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.480734+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:19:27.480734+0000","last_peered":"2023-06-10T21:19:27.480734+0000","last_clean":"2023-06-10T21:19:27.480734+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:19:27.480734+0000","last_undegraded":"2023-06-10T21:19:27.480734+0000","last_fullsized":"2023-06-10T21:19:27.480734+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475368+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:19:27.475368+0000","last_peered":"2023-06-10T21:19:27.475368+0000","last_clean":"2023-06-10T21:19:27.475368+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:19:27.475368+0000","last_undegraded":"2023-06-10T21:19:27.475368+0000","last_fullsized":"2023-06-10T21:19:27.475368+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474220+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:19:27.474220+0000","last_peered":"2023-06-10T21:19:27.474220+0000","last_clean":"2023-06-10T21:19:27.474220+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:19:27.474220+0000","last_undegraded":"2023-06-10T21:19:27.474220+0000","last_fullsized":"2023-06-10T21:19:27.474220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486084+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:19:28.486084+0000","last_peered":"2023-06-10T21:19:28.486084+0000","last_clean":"2023-06-10T21:19:28.486084+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:19:28.486084+0000","last_undegraded":"2023-06-10T21:19:28.486084+0000","last_fullsized":"2023-06-10T21:19:28.486084+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482444+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:19:28.482444+0000","last_peered":"2023-06-10T21:19:28.482444+0000","last_clean":"2023-06-10T21:19:28.482444+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:19:28.482444+0000","last_undegraded":"2023-06-10T21:19:28.482444+0000","last_fullsized":"2023-06-10T21:19:28.482444+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474223+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:19:27.474223+0000","last_peered":"2023-06-10T21:19:27.474223+0000","last_clean":"2023-06-10T21:19:27.474223+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:19:27.474223+0000","last_undegraded":"2023-06-10T21:19:27.474223+0000","last_fullsized":"2023-06-10T21:19:27.474223+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478558+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:19:27.478558+0000","last_peered":"2023-06-10T21:19:27.478558+0000","last_clean":"2023-06-10T21:19:27.478558+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:19:27.478558+0000","last_undegraded":"2023-06-10T21:19:27.478558+0000","last_fullsized":"2023-06-10T21:19:27.478558+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475296+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:19:27.475296+0000","last_peered":"2023-06-10T21:19:27.475296+0000","last_clean":"2023-06-10T21:19:27.475296+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:19:27.475296+0000","last_undegraded":"2023-06-10T21:19:27.475296+0000","last_fullsized":"2023-06-10T21:19:27.475296+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":329,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475365+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:19:27.475365+0000","last_peered":"2023-06-10T21:19:27.475365+0000","last_clean":"2023-06-10T21:19:27.475365+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:19:27.475365+0000","last_undegraded":"2023-06-10T21:19:27.475365+0000","last_fullsized":"2023-06-10T21:19:27.475365+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481502+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:19:28.481502+0000","last_peered":"2023-06-10T21:19:28.481502+0000","last_clean":"2023-06-10T21:19:28.481502+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:19:28.481502+0000","last_undegraded":"2023-06-10T21:19:28.481502+0000","last_fullsized":"2023-06-10T21:19:28.481502+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470465+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:19:27.470465+0000","last_peered":"2023-06-10T21:19:27.470465+0000","last_clean":"2023-06-10T21:19:27.470465+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:19:27.470465+0000","last_undegraded":"2023-06-10T21:19:27.470465+0000","last_fullsized":"2023-06-10T21:19:27.470465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475272+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:19:27.475272+0000","last_peered":"2023-06-10T21:19:27.475272+0000","last_clean":"2023-06-10T21:19:27.475272+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:19:27.475272+0000","last_undegraded":"2023-06-10T21:19:27.475272+0000","last_fullsized":"2023-06-10T21:19:27.475272+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487756+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:19:28.487756+0000","last_peered":"2023-06-10T21:19:28.487756+0000","last_clean":"2023-06-10T21:19:28.487756+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:19:28.487756+0000","last_undegraded":"2023-06-10T21:19:28.487756+0000","last_fullsized":"2023-06-10T21:19:28.487756+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686568+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:19:27.686568+0000","last_peered":"2023-06-10T21:19:27.686568+0000","last_clean":"2023-06-10T21:19:27.686568+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:19:27.686568+0000","last_undegraded":"2023-06-10T21:19:27.686568+0000","last_fullsized":"2023-06-10T21:19:27.686568+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477662+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:19:27.477662+0000","last_peered":"2023-06-10T21:19:27.477662+0000","last_clean":"2023-06-10T21:19:27.477662+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:19:27.477662+0000","last_undegraded":"2023-06-10T21:19:27.477662+0000","last_fullsized":"2023-06-10T21:19:27.477662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475077+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:19:27.475077+0000","last_peered":"2023-06-10T21:19:27.475077+0000","last_clean":"2023-06-10T21:19:27.475077+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:19:27.475077+0000","last_undegraded":"2023-06-10T21:19:27.475077+0000","last_fullsized":"2023-06-10T21:19:27.475077+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482778+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:19:27.482778+0000","last_peered":"2023-06-10T21:19:27.482778+0000","last_clean":"2023-06-10T21:19:27.482778+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:19:27.482778+0000","last_undegraded":"2023-06-10T21:19:27.482778+0000","last_fullsized":"2023-06-10T21:19:27.482778+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484610+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:19:28.484610+0000","last_peered":"2023-06-10T21:19:28.484610+0000","last_clean":"2023-06-10T21:19:28.484610+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:19:28.484610+0000","last_undegraded":"2023-06-10T21:19:28.484610+0000","last_fullsized":"2023-06-10T21:19:28.484610+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":86,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475914+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:19:27.475914+0000","last_peered":"2023-06-10T21:19:27.475914+0000","last_clean":"2023-06-10T21:19:27.475914+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:19:27.475914+0000","last_undegraded":"2023-06-10T21:19:27.475914+0000","last_fullsized":"2023-06-10T21:19:27.475914+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487200+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:19:28.487200+0000","last_peered":"2023-06-10T21:19:28.487200+0000","last_clean":"2023-06-10T21:19:28.487200+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:19:28.487200+0000","last_undegraded":"2023-06-10T21:19:28.487200+0000","last_fullsized":"2023-06-10T21:19:28.487200+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470586+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:19:27.470586+0000","last_peered":"2023-06-10T21:19:27.470586+0000","last_clean":"2023-06-10T21:19:27.470586+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:19:27.470586+0000","last_undegraded":"2023-06-10T21:19:27.470586+0000","last_fullsized":"2023-06-10T21:19:27.470586+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686543+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:19:27.686543+0000","last_peered":"2023-06-10T21:19:27.686543+0000","last_clean":"2023-06-10T21:19:27.686543+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:19:27.686543+0000","last_undegraded":"2023-06-10T21:19:27.686543+0000","last_fullsized":"2023-06-10T21:19:27.686543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472930+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:19:27.472930+0000","last_peered":"2023-06-10T21:19:27.472930+0000","last_clean":"2023-06-10T21:19:27.472930+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:19:27.472930+0000","last_undegraded":"2023-06-10T21:19:27.472930+0000","last_fullsized":"2023-06-10T21:19:27.472930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":68,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482973+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:19:27.482973+0000","last_peered":"2023-06-10T21:19:27.482973+0000","last_clean":"2023-06-10T21:19:27.482973+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:19:27.482973+0000","last_undegraded":"2023-06-10T21:19:27.482973+0000","last_fullsized":"2023-06-10T21:19:27.482973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482329+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:19:28.482329+0000","last_peered":"2023-06-10T21:19:28.482329+0000","last_clean":"2023-06-10T21:19:28.482329+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:19:28.482329+0000","last_undegraded":"2023-06-10T21:19:28.482329+0000","last_fullsized":"2023-06-10T21:19:28.482329+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478090+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:19:27.478090+0000","last_peered":"2023-06-10T21:19:27.478090+0000","last_clean":"2023-06-10T21:19:27.478090+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:19:27.478090+0000","last_undegraded":"2023-06-10T21:19:27.478090+0000","last_fullsized":"2023-06-10T21:19:27.478090+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487998+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:19:28.487998+0000","last_peered":"2023-06-10T21:19:28.487998+0000","last_clean":"2023-06-10T21:19:28.487998+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:19:28.487998+0000","last_undegraded":"2023-06-10T21:19:28.487998+0000","last_fullsized":"2023-06-10T21:19:28.487998+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477850+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:19:27.477850+0000","last_peered":"2023-06-10T21:19:27.477850+0000","last_clean":"2023-06-10T21:19:27.477850+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:19:27.477850+0000","last_undegraded":"2023-06-10T21:19:27.477850+0000","last_fullsized":"2023-06-10T21:19:27.477850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482280+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:19:28.482280+0000","last_peered":"2023-06-10T21:19:28.482280+0000","last_clean":"2023-06-10T21:19:28.482280+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:19:28.482280+0000","last_undegraded":"2023-06-10T21:19:28.482280+0000","last_fullsized":"2023-06-10T21:19:28.482280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482461+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:19:28.482461+0000","last_peered":"2023-06-10T21:19:28.482461+0000","last_clean":"2023-06-10T21:19:28.482461+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:19:28.482461+0000","last_undegraded":"2023-06-10T21:19:28.482461+0000","last_fullsized":"2023-06-10T21:19:28.482461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481801+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:19:28.481801+0000","last_peered":"2023-06-10T21:19:28.481801+0000","last_clean":"2023-06-10T21:19:28.481801+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:19:28.481801+0000","last_undegraded":"2023-06-10T21:19:28.481801+0000","last_fullsized":"2023-06-10T21:19:28.481801+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482857+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:19:27.482857+0000","last_peered":"2023-06-10T21:19:27.482857+0000","last_clean":"2023-06-10T21:19:27.482857+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:19:27.482857+0000","last_undegraded":"2023-06-10T21:19:27.482857+0000","last_fullsized":"2023-06-10T21:19:27.482857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471651+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:19:27.471651+0000","last_peered":"2023-06-10T21:19:27.471651+0000","last_clean":"2023-06-10T21:19:27.471651+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:19:27.471651+0000","last_undegraded":"2023-06-10T21:19:27.471651+0000","last_fullsized":"2023-06-10T21:19:27.471651+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430480,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305940,"kb_used_data":1804,"kb_used_omap":0,"kb_used_meta":304128,"kb_avail":93439212,"statfs":{"total":95995035648,"available":95681753088,"internally_reserved":0,"allocated":1847296,"data_stored":1359411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":311427072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.258,"15min":1.3480000000000001},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.68899999999999995,"5min":10.914999999999999,"15min":10.914999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.96,"15min":1.95},"min":{"1min":0.41999999999999998,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":0.75600000000000001,"5min":15.925000000000001,"15min":15.925000000000001},"last":0.48599999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:19:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.855,"15min":1.5369999999999999},"min":{"1min":0.40300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77900000000000003,"5min":10.083,"15min":10.083},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.5600000000000001,"15min":1.52},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.751,"5min":7,"15min":7},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.624}]},{"osd":3,"last update":"Sat Jun 10 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":2.7200000000000002,"15min":2.8069999999999999},"min":{"1min":0.51200000000000001,"5min":0.35099999999999998,"15min":0.29699999999999999},"max":{"1min":15.127000000000001,"5min":15.127000000000001,"15min":15.127000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.246,"5min":2.556,"15min":2.7149999999999999},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":15.275,"5min":15.275,"15min":15.275},"last":0.73999999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.1579999999999999,"15min":2.0960000000000001},"min":{"1min":0.309,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.67000000000000004,"5min":8.0830000000000002,"15min":219.17699999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4219999999999999,"15min":2.2469999999999999},"min":{"1min":0.34300000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.77200000000000002,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.522,"15min":2.7719999999999998},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.254},"max":{"1min":0.77100000000000002,"5min":37.759,"15min":311.202},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4990000000000001,"15min":2.6040000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.29699999999999999},"max":{"1min":0.72899999999999998,"5min":30.074999999999999,"15min":311.875},"last":0.79000000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.2130000000000001,"15min":1.2769999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78500000000000003,"5min":6.8570000000000002,"15min":15.58},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.01,"15min":1.1259999999999999},"min":{"1min":0.46200000000000002,"5min":0.33400000000000002,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":7.6900000000000004,"15min":8.8829999999999991},"last":0.75900000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626708,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281028,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":279232,"kb_avail":93464124,"statfs":{"total":95995035648,"available":95707262976,"internally_reserved":0,"allocated":1830912,"data_stored":1358950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285933568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.4059999999999999,"15min":2.1970000000000001},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":0.80200000000000005,"5min":9.7929999999999993,"15min":119.276},"last":0.623},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.704,"15min":2.8410000000000002},"min":{"1min":0.378,"5min":0.252,"15min":0.252},"max":{"1min":0.73099999999999998,"5min":16.324999999999999,"15min":119.22},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.788,"15min":1.823},"min":{"1min":0.37,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70399999999999996,"5min":14.897,"15min":14.897},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.9910000000000001,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.84899999999999998,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:18:17 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.891,"15min":2.7789999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29099999999999998},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":74.248000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.6659999999999999,"15min":2.9849999999999999},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":7.3029999999999999,"5min":15.488,"15min":74.314999999999998},"last":0.69699999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.95599999999999996,"15min":1.7909999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.66600000000000004,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.1890000000000001,"15min":1.994},"min":{"1min":0.379,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":0.71099999999999997,"5min":11.263999999999999,"15min":221.905},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.0149999999999999,"15min":2.2799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.72099999999999997,"5min":6.6680000000000001,"15min":313.661},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1080000000000001,"15min":2.3490000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.74399999999999999,"5min":9.75,"15min":314.38900000000001},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.2689999999999999,"15min":1.1579999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.81100000000000005,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.1859999999999999,"15min":1.2130000000000001},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":0.78500000000000003,"5min":8.0980000000000008,"15min":12.583},"last":0.64100000000000001}]}]},{"osd":1,"up_from":105,"seq":450971566151,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213956,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":212667,"kb_avail":93531196,"statfs":{"total":95995035648,"available":95775944704,"internally_reserved":0,"allocated":1306624,"data_stored":776163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":217771918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":1.9319999999999999,"15min":1.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.49099999999999999,"5min":10.898,"15min":10.898},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.4420000000000002,"15min":2.383},"min":{"1min":0.315,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.843,"5min":16.488,"15min":16.488},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.895,"5min":1.895,"15min":1.895},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.927,"15min":3.1629999999999998},"min":{"1min":0.32400000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.19,"5min":13.349,"15min":13.349},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.0030000000000001,"5min":2.9740000000000002,"15min":2.4500000000000002},"min":{"1min":0.35499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.7530000000000001,"5min":22.033999999999999,"15min":22.033999999999999},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.8400000000000001,"15min":1.9299999999999999},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.9550000000000001,"5min":12.94,"15min":12.94},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.008,"15min":2.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.83299999999999996,"5min":12.603,"15min":12.603},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.2050000000000001,"15min":2.1720000000000002},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.9529999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.2730000000000001,"15min":2.5},"min":{"1min":0.45600000000000002,"5min":0.309,"15min":0.309},"max":{"1min":1.9870000000000001,"5min":13.602,"15min":13.602},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.2050000000000001,"15min":2.2130000000000001},"min":{"1min":0.504,"5min":0.379,"15min":0.379},"max":{"1min":3.0169999999999999,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.752,"5min":2.2029999999999998,"15min":2.524},"min":{"1min":0.47599999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.004,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4180000000000001,"15min":3.2530000000000001},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.024,"5min":13.025,"15min":13.025},"last":0.439},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.1110000000000002,"15min":2.161},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75900000000000001,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.54800000000000004}]}]},{"osd":0,"up_from":86,"seq":369367187573,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313180,"kb_used_data":1876,"kb_used_omap":5,"kb_used_meta":311290,"kb_avail":93431972,"statfs":{"total":95995035648,"available":95674339328,"internally_reserved":0,"allocated":1921024,"data_stored":1368213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":318761594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.994,"15min":2.0670000000000002},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.502,"5min":16.780999999999999,"15min":16.780999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.33400000000000002,"5min":3.2679999999999998,"15min":3.032},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48599999999999999,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.441,"5min":0.441,"15min":0.441},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":2.4489999999999998,"15min":2.7400000000000002},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.247},"max":{"1min":5.6269999999999998,"5min":16.827000000000002,"15min":16.827000000000002},"last":0.435},{"interface":"front","average":{"1min":1.405,"5min":3.8439999999999999,"15min":4.8090000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.3780000000000001,"5min":22.053000000000001,"15min":22.379999999999999},"last":0.51400000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.093,"15min":2.2639999999999998},"min":{"1min":0.437,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1609999999999996,"5min":10.948,"15min":10.948},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.5780000000000001,"15min":2.0179999999999998},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":0.81799999999999995,"5min":8.8160000000000007,"15min":10.667},"last":0.79800000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.754,"15min":2.375},"min":{"1min":0.42699999999999999,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.86099999999999999,"5min":9.7119999999999997,"15min":10.693},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.5649999999999999,"15min":2.5840000000000001},"min":{"1min":0.442,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":10.964,"15min":17.460000000000001},"last":0.83099999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.873,"15min":2.4460000000000002},"min":{"1min":0.505,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":0.95799999999999996,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.6520000000000001,"15min":2.7029999999999998},"min":{"1min":0.503,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":0.94499999999999995,"5min":11.022,"15min":17.411000000000001},"last":0.76400000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.5390000000000001,"15min":2.7029999999999998},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.36399999999999999},"max":{"1min":0.81299999999999994,"5min":11.224,"15min":17.486999999999998},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.3100000000000001,"15min":2.194},"min":{"1min":0.53000000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82699999999999996,"5min":9.3960000000000008,"15min":10.867000000000001},"last":0.78100000000000003}]}]},{"osd":2,"up_from":120,"seq":515396075539,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161944,"kb_used_data":1232,"kb_used_omap":7,"kb_used_meta":160696,"kb_avail":93583208,"statfs":{"total":95995035648,"available":95829204992,"internally_reserved":0,"allocated":1261568,"data_stored":769786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164552785},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.45400000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.48599999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.78600000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.91100000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.92300000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.95799999999999996}]}]},{"osd":3,"up_from":125,"seq":536870912014,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115404,"kb_used_data":1220,"kb_used_omap":2,"kb_used_meta":114173,"kb_avail":93629748,"statfs":{"total":95995035648,"available":95876861952,"internally_reserved":0,"allocated":1249280,"data_stored":768648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":116913205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73699999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019164,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333384,"kb_used_data":1216,"kb_used_omap":0,"kb_used_meta":332160,"kb_avail":93411768,"statfs":{"total":95995035648,"available":95653650432,"internally_reserved":0,"allocated":1245184,"data_stored":768658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340131840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.2869999999999999,"15min":2.0550000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":8.9269999999999996,"15min":10.01},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":1.8520000000000001,"15min":2.298},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":0.71499999999999997,"5min":9.593,"15min":15.476000000000001},"last":0.42599999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.6890000000000001,"15min":1.819},"min":{"1min":0.36599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72199999999999998,"5min":15.757,"15min":15.757},"last":0.371},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.8700000000000001,"15min":2.351},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.087,"5min":15.125,"15min":15.125},"last":0.50800000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.628}]},{"osd":3,"last update":"Sat Jun 10 21:18:35 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":2.3690000000000002,"15min":3.2959999999999998},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.30499999999999999},"max":{"1min":10.095000000000001,"5min":11.964,"15min":140.40799999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.4449999999999998,"5min":2.2970000000000002,"15min":3.2930000000000001},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.312},"max":{"1min":7.1849999999999996,"5min":9.5619999999999994,"15min":140.38200000000001},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.81899999999999995,"15min":1.2430000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":0.70399999999999996,"5min":6.5010000000000003,"15min":92.653000000000006},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.0649999999999999,"15min":1.532},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":0.83199999999999996,"5min":6.9539999999999997,"15min":93.281999999999996},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:19:28 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.97899999999999998,"15min":1.099},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.254},"max":{"1min":0.80200000000000005,"5min":5.867,"15min":8.3179999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.034,"15min":1.0720000000000001},"min":{"1min":0.41999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73799999999999999,"5min":8.8420000000000005,"15min":13.045999999999999},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:19:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.284,"15min":1.3440000000000001},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":0.74299999999999999,"5min":8.4459999999999997,"15min":19.989000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0880000000000001,"15min":1.1739999999999999},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.90400000000000003,"5min":5.4379999999999997,"15min":19.783000000000001},"last":0.621}]}]},{"osd":5,"up_from":36,"seq":154618822936,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387320,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":386112,"kb_avail":93357832,"statfs":{"total":95995035648,"available":95598419968,"internally_reserved":0,"allocated":1228800,"data_stored":764420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395378688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.3220000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.0139999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.85599999999999998,"5min":11.807,"15min":11.807},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.0550000000000002,"15min":2.0049999999999999},"min":{"1min":0.41999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.437},{"interface":"front","average":{"1min":0.55800000000000005,"5min":2.3690000000000002,"15min":2.6360000000000001},"min":{"1min":0.45100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":15.962999999999999,"15min":15.962999999999999},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Jun 10 21:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.508,"15min":3.6869999999999998},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.444,"5min":13.186999999999999,"15min":237.95599999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.427,"5min":2.556,"15min":3.6200000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.317},"max":{"1min":7.508,"5min":14.259,"15min":237.96600000000001},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.1779999999999999,"15min":1.0229999999999999},"min":{"1min":0.32400000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":0.63,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.4279999999999999,"15min":1.2270000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.76500000000000001,"5min":12.901,"15min":20.356999999999999},"last":0.627}]},{"osd":6,"last update":"Sat Jun 10 21:20:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.107,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.72399999999999998,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0700000000000001,"15min":1.2430000000000001},"min":{"1min":0.44,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":0.84399999999999997,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:19:42 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.4279999999999999,"15min":1.365},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.78300000000000003,"5min":8.5109999999999992,"15min":9.5039999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3280000000000001,"15min":1.1759999999999999},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.75600000000000001,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.64100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:20:28.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:20:29.144 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:29.494 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:29 smithi064 ceph-mon[186706]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:29.495 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:29 smithi064 ceph-mon[186706]: from='client.35346 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:29.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:29 smithi064 ceph-mon[180297]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:29.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:29 smithi064 ceph-mon[180297]: from='client.35346 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:29 smithi152 ceph-mon[156819]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:29 smithi152 ceph-mon[156819]: from='client.35346 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:20:30.817 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:20:31.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:31 smithi064 ceph-mon[180297]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:31.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:31 smithi064 ceph-mon[186706]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:31.539 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":500,"stamp":"2023-06-10T21:20:29.408525+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2112156,"kb_used_data":11612,"kb_used_omap":20,"kb_used_meta":2100459,"kb_avail":747849060,"statfs":{"total":767960285184,"available":765797437440,"internally_reserved":0,"allocated":11890688,"data_stored":7934249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2150870670},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003091"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471845+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:19:27.471845+0000","last_peered":"2023-06-10T21:19:27.471845+0000","last_clean":"2023-06-10T21:19:27.471845+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:19:27.471845+0000","last_undegraded":"2023-06-10T21:19:27.471845+0000","last_fullsized":"2023-06-10T21:19:27.471845+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.478832+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:19:28.478832+0000","last_peered":"2023-06-10T21:19:28.478832+0000","last_clean":"2023-06-10T21:19:28.478832+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:19:28.478832+0000","last_undegraded":"2023-06-10T21:19:28.478832+0000","last_fullsized":"2023-06-10T21:19:28.478832+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487021+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:19:28.487021+0000","last_peered":"2023-06-10T21:19:28.487021+0000","last_clean":"2023-06-10T21:19:28.487021+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:19:28.487021+0000","last_undegraded":"2023-06-10T21:19:28.487021+0000","last_fullsized":"2023-06-10T21:19:28.487021+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483727+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:19:28.483727+0000","last_peered":"2023-06-10T21:19:28.483727+0000","last_clean":"2023-06-10T21:19:28.483727+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:19:28.483727+0000","last_undegraded":"2023-06-10T21:19:28.483727+0000","last_fullsized":"2023-06-10T21:19:28.483727+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480757+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:19:28.480757+0000","last_peered":"2023-06-10T21:19:28.480757+0000","last_clean":"2023-06-10T21:19:28.480757+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:19:28.480757+0000","last_undegraded":"2023-06-10T21:19:28.480757+0000","last_fullsized":"2023-06-10T21:19:28.480757+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470669+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:19:27.470669+0000","last_peered":"2023-06-10T21:19:27.470669+0000","last_clean":"2023-06-10T21:19:27.470669+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:19:27.470669+0000","last_undegraded":"2023-06-10T21:19:27.470669+0000","last_fullsized":"2023-06-10T21:19:27.470669+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486932+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:19:28.486932+0000","last_peered":"2023-06-10T21:19:28.486932+0000","last_clean":"2023-06-10T21:19:28.486932+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:19:28.486932+0000","last_undegraded":"2023-06-10T21:19:28.486932+0000","last_fullsized":"2023-06-10T21:19:28.486932+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480260+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:19:28.480260+0000","last_peered":"2023-06-10T21:19:28.480260+0000","last_clean":"2023-06-10T21:19:28.480260+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:19:28.480260+0000","last_undegraded":"2023-06-10T21:19:28.480260+0000","last_fullsized":"2023-06-10T21:19:28.480260+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476985+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:19:27.476985+0000","last_peered":"2023-06-10T21:19:27.476985+0000","last_clean":"2023-06-10T21:19:27.476985+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:19:27.476985+0000","last_undegraded":"2023-06-10T21:19:27.476985+0000","last_fullsized":"2023-06-10T21:19:27.476985+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483795+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.483795+0000","last_peered":"2023-06-10T21:19:28.483795+0000","last_clean":"2023-06-10T21:19:28.483795+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:19:28.483795+0000","last_undegraded":"2023-06-10T21:19:28.483795+0000","last_fullsized":"2023-06-10T21:19:28.483795+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":96,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483994+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:19:28.483994+0000","last_peered":"2023-06-10T21:19:28.483994+0000","last_clean":"2023-06-10T21:19:28.483994+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:19:28.483994+0000","last_undegraded":"2023-06-10T21:19:28.483994+0000","last_fullsized":"2023-06-10T21:19:28.483994+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474354+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:19:27.474354+0000","last_peered":"2023-06-10T21:19:27.474354+0000","last_clean":"2023-06-10T21:19:27.474354+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:19:27.474354+0000","last_undegraded":"2023-06-10T21:19:27.474354+0000","last_fullsized":"2023-06-10T21:19:27.474354+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472826+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:19:27.472826+0000","last_peered":"2023-06-10T21:19:27.472826+0000","last_clean":"2023-06-10T21:19:27.472826+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:19:27.472826+0000","last_undegraded":"2023-06-10T21:19:27.472826+0000","last_fullsized":"2023-06-10T21:19:27.472826+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474064+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:19:27.474064+0000","last_peered":"2023-06-10T21:19:27.474064+0000","last_clean":"2023-06-10T21:19:27.474064+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:19:27.474064+0000","last_undegraded":"2023-06-10T21:19:27.474064+0000","last_fullsized":"2023-06-10T21:19:27.474064+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475311+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:19:27.475311+0000","last_peered":"2023-06-10T21:19:27.475311+0000","last_clean":"2023-06-10T21:19:27.475311+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:19:27.475311+0000","last_undegraded":"2023-06-10T21:19:27.475311+0000","last_fullsized":"2023-06-10T21:19:27.475311+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483638+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:19:28.483638+0000","last_peered":"2023-06-10T21:19:28.483638+0000","last_clean":"2023-06-10T21:19:28.483638+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:19:28.483638+0000","last_undegraded":"2023-06-10T21:19:28.483638+0000","last_fullsized":"2023-06-10T21:19:28.483638+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486854+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:19:28.486854+0000","last_peered":"2023-06-10T21:19:28.486854+0000","last_clean":"2023-06-10T21:19:28.486854+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:19:28.486854+0000","last_undegraded":"2023-06-10T21:19:28.486854+0000","last_fullsized":"2023-06-10T21:19:28.486854+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481386+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:19:28.481386+0000","last_peered":"2023-06-10T21:19:28.481386+0000","last_clean":"2023-06-10T21:19:28.481386+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:19:28.481386+0000","last_undegraded":"2023-06-10T21:19:28.481386+0000","last_fullsized":"2023-06-10T21:19:28.481386+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475349+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:19:27.475349+0000","last_peered":"2023-06-10T21:19:27.475349+0000","last_clean":"2023-06-10T21:19:27.475349+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:19:27.475349+0000","last_undegraded":"2023-06-10T21:19:27.475349+0000","last_fullsized":"2023-06-10T21:19:27.475349+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486623+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:19:28.486623+0000","last_peered":"2023-06-10T21:19:28.486623+0000","last_clean":"2023-06-10T21:19:28.486623+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:19:28.486623+0000","last_undegraded":"2023-06-10T21:19:28.486623+0000","last_fullsized":"2023-06-10T21:19:28.486623+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470529+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:19:27.470529+0000","last_peered":"2023-06-10T21:19:27.470529+0000","last_clean":"2023-06-10T21:19:27.470529+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:19:27.470529+0000","last_undegraded":"2023-06-10T21:19:27.470529+0000","last_fullsized":"2023-06-10T21:19:27.470529+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484436+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:19:28.484436+0000","last_peered":"2023-06-10T21:19:28.484436+0000","last_clean":"2023-06-10T21:19:28.484436+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:19:28.484436+0000","last_undegraded":"2023-06-10T21:19:28.484436+0000","last_fullsized":"2023-06-10T21:19:28.484436+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476812+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:19:27.476812+0000","last_peered":"2023-06-10T21:19:27.476812+0000","last_clean":"2023-06-10T21:19:27.476812+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:19:27.476812+0000","last_undegraded":"2023-06-10T21:19:27.476812+0000","last_fullsized":"2023-06-10T21:19:27.476812+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474154+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:19:27.474154+0000","last_peered":"2023-06-10T21:19:27.474154+0000","last_clean":"2023-06-10T21:19:27.474154+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:19:27.474154+0000","last_undegraded":"2023-06-10T21:19:27.474154+0000","last_fullsized":"2023-06-10T21:19:27.474154+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.476603+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:19:27.476603+0000","last_peered":"2023-06-10T21:19:27.476603+0000","last_clean":"2023-06-10T21:19:27.476603+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:19:27.476603+0000","last_undegraded":"2023-06-10T21:19:27.476603+0000","last_fullsized":"2023-06-10T21:19:27.476603+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487906+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:19:28.487906+0000","last_peered":"2023-06-10T21:19:28.487906+0000","last_clean":"2023-06-10T21:19:28.487906+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:19:28.487906+0000","last_undegraded":"2023-06-10T21:19:28.487906+0000","last_fullsized":"2023-06-10T21:19:28.487906+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477579+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:19:27.477579+0000","last_peered":"2023-06-10T21:19:27.477579+0000","last_clean":"2023-06-10T21:19:27.477579+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:19:27.477579+0000","last_undegraded":"2023-06-10T21:19:27.477579+0000","last_fullsized":"2023-06-10T21:19:27.477579+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470358+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:19:27.470358+0000","last_peered":"2023-06-10T21:19:27.470358+0000","last_clean":"2023-06-10T21:19:27.470358+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:19:27.470358+0000","last_undegraded":"2023-06-10T21:19:27.470358+0000","last_fullsized":"2023-06-10T21:19:27.470358+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.480945+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:19:28.480945+0000","last_peered":"2023-06-10T21:19:28.480945+0000","last_clean":"2023-06-10T21:19:28.480945+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:19:28.480945+0000","last_undegraded":"2023-06-10T21:19:28.480945+0000","last_fullsized":"2023-06-10T21:19:28.480945+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.477277+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:19:28.477277+0000","last_peered":"2023-06-10T21:19:28.477277+0000","last_clean":"2023-06-10T21:19:28.477277+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:19:28.477277+0000","last_undegraded":"2023-06-10T21:19:28.477277+0000","last_fullsized":"2023-06-10T21:19:28.477277+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481924+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:19:28.481924+0000","last_peered":"2023-06-10T21:19:28.481924+0000","last_clean":"2023-06-10T21:19:28.481924+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:19:28.481924+0000","last_undegraded":"2023-06-10T21:19:28.481924+0000","last_fullsized":"2023-06-10T21:19:28.481924+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.488064+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:19:28.488064+0000","last_peered":"2023-06-10T21:19:28.488064+0000","last_clean":"2023-06-10T21:19:28.488064+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:19:28.488064+0000","last_undegraded":"2023-06-10T21:19:28.488064+0000","last_fullsized":"2023-06-10T21:19:28.488064+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470197+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:19:27.470197+0000","last_peered":"2023-06-10T21:19:27.470197+0000","last_clean":"2023-06-10T21:19:27.470197+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:19:27.470197+0000","last_undegraded":"2023-06-10T21:19:27.470197+0000","last_fullsized":"2023-06-10T21:19:27.470197+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472148+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:19:27.472148+0000","last_peered":"2023-06-10T21:19:27.472148+0000","last_clean":"2023-06-10T21:19:27.472148+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:19:27.472148+0000","last_undegraded":"2023-06-10T21:19:27.472148+0000","last_fullsized":"2023-06-10T21:19:27.472148+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472225+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:19:27.472225+0000","last_peered":"2023-06-10T21:19:27.472225+0000","last_clean":"2023-06-10T21:19:27.472225+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:19:27.472225+0000","last_undegraded":"2023-06-10T21:19:27.472225+0000","last_fullsized":"2023-06-10T21:19:27.472225+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471728+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:19:27.471728+0000","last_peered":"2023-06-10T21:19:27.471728+0000","last_clean":"2023-06-10T21:19:27.471728+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:19:27.471728+0000","last_undegraded":"2023-06-10T21:19:27.471728+0000","last_fullsized":"2023-06-10T21:19:27.471728+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.483541+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:19:28.483541+0000","last_peered":"2023-06-10T21:19:28.483541+0000","last_clean":"2023-06-10T21:19:28.483541+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:19:28.483541+0000","last_undegraded":"2023-06-10T21:19:28.483541+0000","last_fullsized":"2023-06-10T21:19:28.483541+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484999+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:19:28.484999+0000","last_peered":"2023-06-10T21:19:28.484999+0000","last_clean":"2023-06-10T21:19:28.484999+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:19:28.484999+0000","last_undegraded":"2023-06-10T21:19:28.484999+0000","last_fullsized":"2023-06-10T21:19:28.484999+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474695+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:19:27.474695+0000","last_peered":"2023-06-10T21:19:27.474695+0000","last_clean":"2023-06-10T21:19:27.474695+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:19:27.474695+0000","last_undegraded":"2023-06-10T21:19:27.474695+0000","last_fullsized":"2023-06-10T21:19:27.474695+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482192+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:19:28.482192+0000","last_peered":"2023-06-10T21:19:28.482192+0000","last_clean":"2023-06-10T21:19:28.482192+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:19:28.482192+0000","last_undegraded":"2023-06-10T21:19:28.482192+0000","last_fullsized":"2023-06-10T21:19:28.482192+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.480734+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:19:27.480734+0000","last_peered":"2023-06-10T21:19:27.480734+0000","last_clean":"2023-06-10T21:19:27.480734+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:19:27.480734+0000","last_undegraded":"2023-06-10T21:19:27.480734+0000","last_fullsized":"2023-06-10T21:19:27.480734+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475368+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:19:27.475368+0000","last_peered":"2023-06-10T21:19:27.475368+0000","last_clean":"2023-06-10T21:19:27.475368+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:19:27.475368+0000","last_undegraded":"2023-06-10T21:19:27.475368+0000","last_fullsized":"2023-06-10T21:19:27.475368+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474220+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:19:27.474220+0000","last_peered":"2023-06-10T21:19:27.474220+0000","last_clean":"2023-06-10T21:19:27.474220+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:19:27.474220+0000","last_undegraded":"2023-06-10T21:19:27.474220+0000","last_fullsized":"2023-06-10T21:19:27.474220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.486084+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:19:28.486084+0000","last_peered":"2023-06-10T21:19:28.486084+0000","last_clean":"2023-06-10T21:19:28.486084+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:19:28.486084+0000","last_undegraded":"2023-06-10T21:19:28.486084+0000","last_fullsized":"2023-06-10T21:19:28.486084+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482444+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:19:28.482444+0000","last_peered":"2023-06-10T21:19:28.482444+0000","last_clean":"2023-06-10T21:19:28.482444+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:19:28.482444+0000","last_undegraded":"2023-06-10T21:19:28.482444+0000","last_fullsized":"2023-06-10T21:19:28.482444+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.474223+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:19:27.474223+0000","last_peered":"2023-06-10T21:19:27.474223+0000","last_clean":"2023-06-10T21:19:27.474223+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:19:27.474223+0000","last_undegraded":"2023-06-10T21:19:27.474223+0000","last_fullsized":"2023-06-10T21:19:27.474223+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478558+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:19:27.478558+0000","last_peered":"2023-06-10T21:19:27.478558+0000","last_clean":"2023-06-10T21:19:27.478558+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:19:27.478558+0000","last_undegraded":"2023-06-10T21:19:27.478558+0000","last_fullsized":"2023-06-10T21:19:27.478558+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475296+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:19:27.475296+0000","last_peered":"2023-06-10T21:19:27.475296+0000","last_clean":"2023-06-10T21:19:27.475296+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:19:27.475296+0000","last_undegraded":"2023-06-10T21:19:27.475296+0000","last_fullsized":"2023-06-10T21:19:27.475296+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":329,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475365+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:19:27.475365+0000","last_peered":"2023-06-10T21:19:27.475365+0000","last_clean":"2023-06-10T21:19:27.475365+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:19:27.475365+0000","last_undegraded":"2023-06-10T21:19:27.475365+0000","last_fullsized":"2023-06-10T21:19:27.475365+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481502+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:19:28.481502+0000","last_peered":"2023-06-10T21:19:28.481502+0000","last_clean":"2023-06-10T21:19:28.481502+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:19:28.481502+0000","last_undegraded":"2023-06-10T21:19:28.481502+0000","last_fullsized":"2023-06-10T21:19:28.481502+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470465+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:19:27.470465+0000","last_peered":"2023-06-10T21:19:27.470465+0000","last_clean":"2023-06-10T21:19:27.470465+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:19:27.470465+0000","last_undegraded":"2023-06-10T21:19:27.470465+0000","last_fullsized":"2023-06-10T21:19:27.470465+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475272+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:19:27.475272+0000","last_peered":"2023-06-10T21:19:27.475272+0000","last_clean":"2023-06-10T21:19:27.475272+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:19:27.475272+0000","last_undegraded":"2023-06-10T21:19:27.475272+0000","last_fullsized":"2023-06-10T21:19:27.475272+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487756+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:19:28.487756+0000","last_peered":"2023-06-10T21:19:28.487756+0000","last_clean":"2023-06-10T21:19:28.487756+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:19:28.487756+0000","last_undegraded":"2023-06-10T21:19:28.487756+0000","last_fullsized":"2023-06-10T21:19:28.487756+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686568+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:19:27.686568+0000","last_peered":"2023-06-10T21:19:27.686568+0000","last_clean":"2023-06-10T21:19:27.686568+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:19:27.686568+0000","last_undegraded":"2023-06-10T21:19:27.686568+0000","last_fullsized":"2023-06-10T21:19:27.686568+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477662+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:19:27.477662+0000","last_peered":"2023-06-10T21:19:27.477662+0000","last_clean":"2023-06-10T21:19:27.477662+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:19:27.477662+0000","last_undegraded":"2023-06-10T21:19:27.477662+0000","last_fullsized":"2023-06-10T21:19:27.477662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475077+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:19:27.475077+0000","last_peered":"2023-06-10T21:19:27.475077+0000","last_clean":"2023-06-10T21:19:27.475077+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:19:27.475077+0000","last_undegraded":"2023-06-10T21:19:27.475077+0000","last_fullsized":"2023-06-10T21:19:27.475077+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482778+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:19:27.482778+0000","last_peered":"2023-06-10T21:19:27.482778+0000","last_clean":"2023-06-10T21:19:27.482778+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:19:27.482778+0000","last_undegraded":"2023-06-10T21:19:27.482778+0000","last_fullsized":"2023-06-10T21:19:27.482778+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.484610+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:19:28.484610+0000","last_peered":"2023-06-10T21:19:28.484610+0000","last_clean":"2023-06-10T21:19:28.484610+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:19:28.484610+0000","last_undegraded":"2023-06-10T21:19:28.484610+0000","last_fullsized":"2023-06-10T21:19:28.484610+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":86,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.475914+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:19:27.475914+0000","last_peered":"2023-06-10T21:19:27.475914+0000","last_clean":"2023-06-10T21:19:27.475914+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:19:27.475914+0000","last_undegraded":"2023-06-10T21:19:27.475914+0000","last_fullsized":"2023-06-10T21:19:27.475914+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487200+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:19:28.487200+0000","last_peered":"2023-06-10T21:19:28.487200+0000","last_clean":"2023-06-10T21:19:28.487200+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:19:28.487200+0000","last_undegraded":"2023-06-10T21:19:28.487200+0000","last_fullsized":"2023-06-10T21:19:28.487200+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.470586+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:19:27.470586+0000","last_peered":"2023-06-10T21:19:27.470586+0000","last_clean":"2023-06-10T21:19:27.470586+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:19:27.470586+0000","last_undegraded":"2023-06-10T21:19:27.470586+0000","last_fullsized":"2023-06-10T21:19:27.470586+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.686543+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:19:27.686543+0000","last_peered":"2023-06-10T21:19:27.686543+0000","last_clean":"2023-06-10T21:19:27.686543+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:19:27.686543+0000","last_undegraded":"2023-06-10T21:19:27.686543+0000","last_fullsized":"2023-06-10T21:19:27.686543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.472930+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:19:27.472930+0000","last_peered":"2023-06-10T21:19:27.472930+0000","last_clean":"2023-06-10T21:19:27.472930+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:19:27.472930+0000","last_undegraded":"2023-06-10T21:19:27.472930+0000","last_fullsized":"2023-06-10T21:19:27.472930+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":68,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482973+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:19:27.482973+0000","last_peered":"2023-06-10T21:19:27.482973+0000","last_clean":"2023-06-10T21:19:27.482973+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:19:27.482973+0000","last_undegraded":"2023-06-10T21:19:27.482973+0000","last_fullsized":"2023-06-10T21:19:27.482973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482329+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:19:28.482329+0000","last_peered":"2023-06-10T21:19:28.482329+0000","last_clean":"2023-06-10T21:19:28.482329+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:19:28.482329+0000","last_undegraded":"2023-06-10T21:19:28.482329+0000","last_fullsized":"2023-06-10T21:19:28.482329+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.478090+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:19:27.478090+0000","last_peered":"2023-06-10T21:19:27.478090+0000","last_clean":"2023-06-10T21:19:27.478090+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:19:27.478090+0000","last_undegraded":"2023-06-10T21:19:27.478090+0000","last_fullsized":"2023-06-10T21:19:27.478090+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.487998+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:19:28.487998+0000","last_peered":"2023-06-10T21:19:28.487998+0000","last_clean":"2023-06-10T21:19:28.487998+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:19:28.487998+0000","last_undegraded":"2023-06-10T21:19:28.487998+0000","last_fullsized":"2023-06-10T21:19:28.487998+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.477850+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:19:27.477850+0000","last_peered":"2023-06-10T21:19:27.477850+0000","last_clean":"2023-06-10T21:19:27.477850+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:19:27.477850+0000","last_undegraded":"2023-06-10T21:19:27.477850+0000","last_fullsized":"2023-06-10T21:19:27.477850+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":107,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482280+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:19:28.482280+0000","last_peered":"2023-06-10T21:19:28.482280+0000","last_clean":"2023-06-10T21:19:28.482280+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:19:28.482280+0000","last_undegraded":"2023-06-10T21:19:28.482280+0000","last_fullsized":"2023-06-10T21:19:28.482280+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.482461+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:19:28.482461+0000","last_peered":"2023-06-10T21:19:28.482461+0000","last_clean":"2023-06-10T21:19:28.482461+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:19:28.482461+0000","last_undegraded":"2023-06-10T21:19:28.482461+0000","last_fullsized":"2023-06-10T21:19:28.482461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-06-10T21:19:28.481801+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:19:28.481801+0000","last_peered":"2023-06-10T21:19:28.481801+0000","last_clean":"2023-06-10T21:19:28.481801+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:19:28.481801+0000","last_undegraded":"2023-06-10T21:19:28.481801+0000","last_fullsized":"2023-06-10T21:19:28.481801+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":84,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.482857+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:19:27.482857+0000","last_peered":"2023-06-10T21:19:27.482857+0000","last_clean":"2023-06-10T21:19:27.482857+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:19:27.482857+0000","last_undegraded":"2023-06-10T21:19:27.482857+0000","last_fullsized":"2023-06-10T21:19:27.482857+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-10T21:19:27.471651+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:19:27.471651+0000","last_peered":"2023-06-10T21:19:27.471651+0000","last_clean":"2023-06-10T21:19:27.471651+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:19:27.471651+0000","last_undegraded":"2023-06-10T21:19:27.471651+0000","last_fullsized":"2023-06-10T21:19:27.471651+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430481,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305940,"kb_used_data":1804,"kb_used_omap":0,"kb_used_meta":304128,"kb_avail":93439212,"statfs":{"total":95995035648,"available":95681753088,"internally_reserved":0,"allocated":1847296,"data_stored":1359411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":311427072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.258,"15min":1.3480000000000001},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.68899999999999995,"5min":10.914999999999999,"15min":10.914999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.96,"15min":1.95},"min":{"1min":0.41999999999999998,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":0.75600000000000001,"5min":15.925000000000001,"15min":15.925000000000001},"last":0.61799999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:19:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.855,"15min":1.5369999999999999},"min":{"1min":0.40300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77900000000000003,"5min":10.083,"15min":10.083},"last":0.499},{"interface":"front","average":{"1min":0.52100000000000002,"5min":1.5600000000000001,"15min":1.52},"min":{"1min":0.39900000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.751,"5min":7,"15min":7},"last":0.378}]},{"osd":2,"last update":"Sat Jun 10 21:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47899999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:18:23 2023","interfaces":[{"interface":"back","average":{"1min":3.3130000000000002,"5min":2.7200000000000002,"15min":2.8069999999999999},"min":{"1min":0.51200000000000001,"5min":0.35099999999999998,"15min":0.29699999999999999},"max":{"1min":15.127000000000001,"5min":15.127000000000001,"15min":15.127000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":3.246,"5min":2.556,"15min":2.7149999999999999},"min":{"1min":0.38,"5min":0.307,"15min":0.307},"max":{"1min":15.275,"5min":15.275,"15min":15.275},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.1579999999999999,"15min":2.0960000000000001},"min":{"1min":0.309,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.67000000000000004,"5min":8.0830000000000002,"15min":219.17699999999999},"last":0.432},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4219999999999999,"15min":2.2469999999999999},"min":{"1min":0.34300000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.77200000000000002,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.439}]},{"osd":5,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.522,"15min":2.7719999999999998},"min":{"1min":0.29199999999999998,"5min":0.27100000000000002,"15min":0.254},"max":{"1min":0.77100000000000002,"5min":37.759,"15min":311.202},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4990000000000001,"15min":2.6040000000000001},"min":{"1min":0.36199999999999999,"5min":0.30199999999999999,"15min":0.29699999999999999},"max":{"1min":0.72899999999999998,"5min":30.074999999999999,"15min":311.875},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.2130000000000001,"15min":1.2769999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78500000000000003,"5min":6.8570000000000002,"15min":15.58},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.01,"15min":1.1259999999999999},"min":{"1min":0.46200000000000002,"5min":0.33400000000000002,"15min":0.32400000000000001},"max":{"1min":0.71899999999999997,"5min":7.6900000000000004,"15min":8.8829999999999991},"last":0.53300000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626709,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":281028,"kb_used_data":1788,"kb_used_omap":0,"kb_used_meta":279232,"kb_avail":93464124,"statfs":{"total":95995035648,"available":95707262976,"internally_reserved":0,"allocated":1830912,"data_stored":1358950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285933568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.4059999999999999,"15min":2.1970000000000001},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":0.80200000000000005,"5min":9.7929999999999993,"15min":119.276},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.704,"15min":2.8410000000000002},"min":{"1min":0.378,"5min":0.252,"15min":0.252},"max":{"1min":0.73099999999999998,"5min":16.324999999999999,"15min":119.22},"last":0.61099999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.788,"15min":1.823},"min":{"1min":0.37,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.70399999999999996,"5min":14.897,"15min":14.897},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.9910000000000001,"15min":1.8680000000000001},"min":{"1min":0.36499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.84899999999999998,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.66800000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.627}]},{"osd":3,"last update":"Sat Jun 10 21:18:17 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.891,"15min":2.7789999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29099999999999998},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":74.248000000000005},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.6659999999999999,"15min":2.9849999999999999},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":7.3029999999999999,"5min":15.488,"15min":74.314999999999998},"last":0.68500000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.95599999999999996,"15min":1.7909999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.66600000000000004,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.1890000000000001,"15min":1.994},"min":{"1min":0.379,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":0.71099999999999997,"5min":11.263999999999999,"15min":221.905},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.0149999999999999,"15min":2.2799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.72099999999999997,"5min":6.6680000000000001,"15min":313.661},"last":0.754},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1080000000000001,"15min":2.3490000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.74399999999999999,"5min":9.75,"15min":314.38900000000001},"last":0.73899999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:19:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.2689999999999999,"15min":1.1579999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.318},"max":{"1min":0.81100000000000005,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.1859999999999999,"15min":1.2130000000000001},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":0.78500000000000003,"5min":8.0980000000000008,"15min":12.583},"last":0.76300000000000001}]}]},{"osd":1,"up_from":105,"seq":450971566151,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213956,"kb_used_data":1276,"kb_used_omap":4,"kb_used_meta":212667,"kb_avail":93531196,"statfs":{"total":95995035648,"available":95775944704,"internally_reserved":0,"allocated":1306624,"data_stored":776163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":217771918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":1.9319999999999999,"15min":1.8260000000000001},"min":{"1min":0.23899999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.49099999999999999,"5min":10.898,"15min":10.898},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":2.4420000000000002,"15min":2.383},"min":{"1min":0.315,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":1.843,"5min":16.488,"15min":16.488},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.895,"5min":1.895,"15min":1.895},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:18:50 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":2.927,"15min":3.1629999999999998},"min":{"1min":0.32400000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":10.19,"5min":13.349,"15min":13.349},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.0030000000000001,"5min":2.9740000000000002,"15min":2.4500000000000002},"min":{"1min":0.35499999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.7530000000000001,"5min":22.033999999999999,"15min":22.033999999999999},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.8400000000000001,"15min":1.9299999999999999},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.9550000000000001,"5min":12.94,"15min":12.94},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.008,"15min":2.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.83299999999999996,"5min":12.603,"15min":12.603},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.2050000000000001,"15min":2.1720000000000002},"min":{"1min":0.42199999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.9529999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.73899999999999999,"5min":2.2730000000000001,"15min":2.5},"min":{"1min":0.45600000000000002,"5min":0.309,"15min":0.309},"max":{"1min":1.9870000000000001,"5min":13.602,"15min":13.602},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":2.2050000000000001,"15min":2.2130000000000001},"min":{"1min":0.504,"5min":0.379,"15min":0.379},"max":{"1min":3.0169999999999999,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.752,"5min":2.2029999999999998,"15min":2.524},"min":{"1min":0.47599999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.004,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:19:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":2.4180000000000001,"15min":3.2530000000000001},"min":{"1min":0.432,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.024,"5min":13.025,"15min":13.025},"last":0.439},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.1110000000000002,"15min":2.161},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.75900000000000001,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.54800000000000004}]}]},{"osd":0,"up_from":86,"seq":369367187574,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313180,"kb_used_data":1876,"kb_used_omap":5,"kb_used_meta":311290,"kb_avail":93431972,"statfs":{"total":95995035648,"available":95674339328,"internally_reserved":0,"allocated":1921024,"data_stored":1368213,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":318761594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.32900000000000001,"5min":1.994,"15min":2.0670000000000002},"min":{"1min":0.254,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.502,"5min":16.780999999999999,"15min":16.780999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.33400000000000002,"5min":3.2679999999999998,"15min":3.032},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48599999999999999,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.34699999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.441,"5min":0.441,"15min":0.441},"last":0.317},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":2.4489999999999998,"15min":2.7400000000000002},"min":{"1min":0.30399999999999999,"5min":0.247,"15min":0.247},"max":{"1min":5.6269999999999998,"5min":16.827000000000002,"15min":16.827000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.405,"5min":3.8439999999999999,"15min":4.8090000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.3780000000000001,"5min":22.053000000000001,"15min":22.379999999999999},"last":0.30199999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.093,"15min":2.2639999999999998},"min":{"1min":0.437,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1609999999999996,"5min":10.948,"15min":10.948},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.5780000000000001,"15min":2.0179999999999998},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":0.81799999999999995,"5min":8.8160000000000007,"15min":10.667},"last":0.63500000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.754,"15min":2.375},"min":{"1min":0.42699999999999999,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.86099999999999999,"5min":9.7119999999999997,"15min":10.693},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.5649999999999999,"15min":2.5840000000000001},"min":{"1min":0.442,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":10.964,"15min":17.460000000000001},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.873,"15min":2.4460000000000002},"min":{"1min":0.505,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":0.95799999999999996,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.6520000000000001,"15min":2.7029999999999998},"min":{"1min":0.503,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":0.94499999999999995,"5min":11.022,"15min":17.411000000000001},"last":0.624}]},{"osd":7,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.5390000000000001,"15min":2.7029999999999998},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.36399999999999999},"max":{"1min":0.81299999999999994,"5min":11.224,"15min":17.486999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.3100000000000001,"15min":2.194},"min":{"1min":0.53000000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82699999999999996,"5min":9.3960000000000008,"15min":10.867000000000001},"last":0.55700000000000005}]}]},{"osd":2,"up_from":120,"seq":515396075540,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161944,"kb_used_data":1232,"kb_used_omap":7,"kb_used_meta":160696,"kb_avail":93583208,"statfs":{"total":95995035648,"available":95829204992,"internally_reserved":0,"allocated":1261568,"data_stored":769786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":164552785},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.373},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.27800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.625},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.55600000000000005}]}]},{"osd":3,"up_from":125,"seq":536870912014,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115404,"kb_used_data":1220,"kb_used_omap":2,"kb_used_meta":114173,"kb_avail":93629748,"statfs":{"total":95995035648,"available":95876861952,"internally_reserved":0,"allocated":1249280,"data_stored":768648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":116913205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73699999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.748}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019165,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333384,"kb_used_data":1216,"kb_used_omap":0,"kb_used_meta":332160,"kb_avail":93411768,"statfs":{"total":95995035648,"available":95653650432,"internally_reserved":0,"allocated":1245184,"data_stored":768658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340131840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.2869999999999999,"15min":2.0550000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":8.9269999999999996,"15min":10.01},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48599999999999999,"5min":1.8520000000000001,"15min":2.298},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":0.71499999999999997,"5min":9.593,"15min":15.476000000000001},"last":0.40300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.6890000000000001,"15min":1.819},"min":{"1min":0.36599999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72199999999999998,"5min":15.757,"15min":15.757},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.8700000000000001,"15min":2.351},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.087,"5min":15.125,"15min":15.125},"last":0.64500000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.749}]},{"osd":5,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.81899999999999995,"15min":1.2430000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":0.70399999999999996,"5min":6.5010000000000003,"15min":92.653000000000006},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.0649999999999999,"15min":1.532},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":0.83199999999999996,"5min":6.9539999999999997,"15min":93.281999999999996},"last":0.70699999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.92200000000000004,"15min":0.97299999999999998},"min":{"1min":0.378,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.69199999999999995,"5min":5.867,"15min":6.4340000000000002},"last":0.629},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.998,"15min":0.88700000000000001},"min":{"1min":0.49199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81200000000000006,"5min":8.8420000000000005,"15min":8.8420000000000005},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:19:39 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.284,"15min":1.3440000000000001},"min":{"1min":0.35999999999999999,"5min":0.308,"15min":0.28699999999999998},"max":{"1min":0.74299999999999999,"5min":8.4459999999999997,"15min":19.989000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.55800000000000005,"5min":1.0880000000000001,"15min":1.1739999999999999},"min":{"1min":0.34000000000000002,"5min":0.314,"15min":0.312},"max":{"1min":0.90400000000000003,"5min":5.4379999999999997,"15min":19.783000000000001},"last":0.72299999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822936,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387320,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":386112,"kb_avail":93357832,"statfs":{"total":95995035648,"available":95598419968,"internally_reserved":0,"allocated":1228800,"data_stored":764420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395378688},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.3220000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.0139999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.85599999999999998,"5min":11.807,"15min":11.807},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":2.0550000000000002,"15min":2.0049999999999999},"min":{"1min":0.41999999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.75600000000000001,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.437},{"interface":"front","average":{"1min":0.55800000000000005,"5min":2.3690000000000002,"15min":2.6360000000000001},"min":{"1min":0.45100000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.78500000000000003,"5min":15.962999999999999,"15min":15.962999999999999},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Jun 10 21:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":2.508,"15min":3.6869999999999998},"min":{"1min":0.35599999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.444,"5min":13.186999999999999,"15min":237.95599999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.427,"5min":2.556,"15min":3.6200000000000001},"min":{"1min":0.41499999999999998,"5min":0.33900000000000002,"15min":0.317},"max":{"1min":7.508,"5min":14.259,"15min":237.96600000000001},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.1779999999999999,"15min":1.0229999999999999},"min":{"1min":0.32400000000000001,"5min":0.25900000000000001,"15min":0.25700000000000001},"max":{"1min":0.63,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.4279999999999999,"15min":1.2270000000000001},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.76500000000000001,"5min":12.901,"15min":20.356999999999999},"last":0.627}]},{"osd":6,"last update":"Sat Jun 10 21:20:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.107,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.72399999999999998,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0700000000000001,"15min":1.2430000000000001},"min":{"1min":0.44,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":0.84399999999999997,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:19:42 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.4279999999999999,"15min":1.365},"min":{"1min":0.434,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.78300000000000003,"5min":8.5109999999999992,"15min":9.5039999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.3280000000000001,"15min":1.1759999999999999},"min":{"1min":0.42299999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.75600000000000001,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.64100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:20:31.547 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-10T21:20:31.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump 2023-06-10T21:20:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:31 smithi152 ceph-mon[156819]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:31.910 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:32.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:32 smithi064 ceph-mon[180297]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:32.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:32 smithi064 ceph-mon[186706]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:32 smithi152 ceph-mon[156819]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:20:33.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:32] "GET /metrics HTTP/1.1" 200 220626 "" "Prometheus/2.33.4" 2023-06-10T21:20:33.626 INFO:teuthology.orchestra.run.smithi064.stdout:epoch 126 2023-06-10T21:20:33.626 INFO:teuthology.orchestra.run.smithi064.stdout:fsid b618768e-07d0-11ee-9b1e-001a4aab830c 2023-06-10T21:20:33.626 INFO:teuthology.orchestra.run.smithi064.stdout:created 2023-06-10T20:53:15.981225+0000 2023-06-10T21:20:33.626 INFO:teuthology.orchestra.run.smithi064.stdout:modified 2023-06-10T21:19:28.465658+0000 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:crush_version 22 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:full_ratio 0.95 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:backfillfull_ratio 0.9 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:nearfull_ratio 0.85 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:require_min_compat_client luminous 2023-06-10T21:20:33.627 INFO:teuthology.orchestra.run.smithi064.stdout:min_compat_client luminous 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:require_osd_release pacific 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:stretch_mode_enabled false 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-06-10T21:20:33.628 INFO:teuthology.orchestra.run.smithi064.stdout:max_osd 8 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0 up in weight 1 up_from 86 up_thru 125 down_at 83 last_clean_interval [8,82) [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149] [v2:172.21.15.64:6804/1597438149,v1:172.21.15.64:6805/1597438149] exists,up 17854ddb-2190-4089-8177-63bd7a59d79a 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1 up in weight 1 up_from 105 up_thru 125 down_at 102 last_clean_interval [13,101) [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888] [v2:172.21.15.64:6812/215784888,v1:172.21.15.64:6813/215784888] exists,up c0f26f57-1eae-44eb-b91a-24db061d2f83 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 up in weight 1 up_from 120 up_thru 125 down_at 117 last_clean_interval [18,116) [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109] [v2:172.21.15.64:6820/746809109,v1:172.21.15.64:6821/746809109] exists,up d4a979da-bdc4-46f7-aaa3-e912753f1272 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 up in weight 1 up_from 125 up_thru 125 down_at 122 last_clean_interval [25,121) [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225] [v2:172.21.15.64:6828/4028577225,v1:172.21.15.64:6829/4028577225] exists,up efd43bfa-889e-4c76-997c-0fd3570fbc6e 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4 up in weight 1 up_from 30 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563] [v2:172.21.15.152:6802/3009869563,v1:172.21.15.152:6803/3009869563] exists,up 9e5f714f-9f72-4bd3-9e4d-97286fbfb88e 2023-06-10T21:20:33.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 up in weight 1 up_from 36 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943] [v2:172.21.15.152:6810/1122141943,v1:172.21.15.152:6811/1122141943] exists,up eb69283f-ccc4-4166-8c9a-05522cd8fd5c 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6 up in weight 1 up_from 42 up_thru 122 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000] [v2:172.21.15.152:6818/3361850000,v1:172.21.15.152:6819/3361850000] exists,up 834d3ada-5c5e-40f0-8520-ad53ff0648ce 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 up in weight 1 up_from 48 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648] [v2:172.21.15.152:6826/769883648,v1:172.21.15.152:6827/769883648] exists,up 8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.2 [1,2] 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.18 [1,2] 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:pg_upmap_items 4.1c [1,5] 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1656599775 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1184928448 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.630 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1079194554 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1551224238 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1338325282 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3424306201 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:6835/3978239224 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1079194554 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/530031084 expires 2023-06-11T20:53:43.436289+0000 2023-06-10T21:20:33.631 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/1928672739 expires 2023-06-11T20:54:08.425389+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:0/3850833042 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3566875446 expires 2023-06-11T20:58:13.320243+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:6834/3978239224 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:0/1248818928 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3632777501 expires 2023-06-11T20:53:43.436289+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/3618131890 expires 2023-06-11T21:02:16.546685+0000 2023-06-10T21:20:33.632 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1955159746 expires 2023-06-11T21:02:16.546685+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/703163662 expires 2023-06-11T21:02:16.546685+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/1928672739 expires 2023-06-11T20:54:08.425389+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:0/3731648028 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3307901273 expires 2023-06-11T20:53:43.436289+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/3450946694 expires 2023-06-11T20:58:13.320243+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/1719072918 expires 2023-06-11T20:54:08.425389+0000 2023-06-10T21:20:33.633 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.152:0/1618902179 expires 2023-06-11T21:04:32.578278+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/790526867 expires 2023-06-11T21:05:32.902880+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/3618131890 expires 2023-06-11T21:02:16.546685+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6801/2180524494 expires 2023-06-11T20:58:13.320243+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/530031084 expires 2023-06-11T20:53:43.436289+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:0/2757723192 expires 2023-06-11T20:54:08.425389+0000 2023-06-10T21:20:33.634 INFO:teuthology.orchestra.run.smithi064.stdout:blocklist 172.21.15.64:6800/2180524494 expires 2023-06-11T20:58:13.320243+0000 2023-06-10T21:20:33.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:33 smithi064 ceph-mon[180297]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:33.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:33.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:33 smithi064 ceph-mon[186706]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:33.654 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:33 smithi152 ceph-mon[156819]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:34.397 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 86 up_thru 125 down_at 83 last_clean_interval [8,82) [v2:172.21.15.64:6802/1597438149,v1:172.21.15.64:6803/1597438149] [v2:172.21.15.64:6804/1597438149,v1:172.21.15.64:6805/1597438149] exists,up 17854ddb-2190-4089-8177-63bd7a59d79a', 'osd.1 up in weight 1 up_from 105 up_thru 125 down_at 102 last_clean_interval [13,101) [v2:172.21.15.64:6810/215784888,v1:172.21.15.64:6811/215784888] [v2:172.21.15.64:6812/215784888,v1:172.21.15.64:6813/215784888] exists,up c0f26f57-1eae-44eb-b91a-24db061d2f83', 'osd.2 up in weight 1 up_from 120 up_thru 125 down_at 117 last_clean_interval [18,116) [v2:172.21.15.64:6818/746809109,v1:172.21.15.64:6819/746809109] [v2:172.21.15.64:6820/746809109,v1:172.21.15.64:6821/746809109] exists,up d4a979da-bdc4-46f7-aaa3-e912753f1272', 'osd.3 up in weight 1 up_from 125 up_thru 125 down_at 122 last_clean_interval [25,121) [v2:172.21.15.64:6826/4028577225,v1:172.21.15.64:6827/4028577225] [v2:172.21.15.64:6828/4028577225,v1:172.21.15.64:6829/4028577225] exists,up efd43bfa-889e-4c76-997c-0fd3570fbc6e', 'osd.4 up in weight 1 up_from 30 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6800/3009869563,v1:172.21.15.152:6801/3009869563] [v2:172.21.15.152:6802/3009869563,v1:172.21.15.152:6803/3009869563] exists,up 9e5f714f-9f72-4bd3-9e4d-97286fbfb88e', 'osd.5 up in weight 1 up_from 36 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6808/1122141943,v1:172.21.15.152:6809/1122141943] [v2:172.21.15.152:6810/1122141943,v1:172.21.15.152:6811/1122141943] exists,up eb69283f-ccc4-4166-8c9a-05522cd8fd5c', 'osd.6 up in weight 1 up_from 42 up_thru 122 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6816/3361850000,v1:172.21.15.152:6817/3361850000] [v2:172.21.15.152:6818/3361850000,v1:172.21.15.152:6819/3361850000] exists,up 834d3ada-5c5e-40f0-8520-ad53ff0648ce', 'osd.7 up in weight 1 up_from 48 up_thru 125 down_at 0 last_clean_interval [0,0) [v2:172.21.15.152:6824/769883648,v1:172.21.15.152:6825/769883648] [v2:172.21.15.152:6826/769883648,v1:172.21.15.152:6827/769883648] exists,up 8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91'] 2023-06-10T21:20:34.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon mon.a config show 2023-06-10T21:20:34.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1459540252' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-10T21:20:34.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1459540252' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-10T21:20:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1459540252' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-10T21:20:34.760 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:35 smithi064 ceph-mon[186706]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:35 smithi064 ceph-mon[180297]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:35 smithi152 ceph-mon[156819]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:36.277 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:20:36.277 INFO:teuthology.orchestra.run.smithi064.stdout: "name": "mon.a", 2023-06-10T21:20:36.277 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster": "ceph", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "admin_socket_mode": "", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_client_required": "cephx, none", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_cluster_required": "cephx", 2023-06-10T21:20:36.278 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_debug": "true", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_required": "cephx", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_service_ticket_ttl": "240.000000", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "auth_supported": "", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio": "true", 2023-06-10T21:20:36.279 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_max_queue_depth": "1024", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_poll_ms": "250", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_aio_reap_max": "16", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_async_discard": "false", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_block_size": "4096", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio": "false", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-06-10T21:20:36.280 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_debug_inflight_ios": "false", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_enable_discard": "false", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry": "3", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_flock_retry_interval": "0.100000", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash": "0", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_inject_crash_flush_delay": "2", 2023-06-10T21:20:36.281 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring": "false", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_hipri": "false", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_ioring_sqthread_poll": "false", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_buffer_alignment": "4096", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-06-10T21:20:36.282 INFO:teuthology.orchestra.run.smithi064.stdout: "bdev_type": "", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_alloc_size": "1048576", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_allocator": "hybrid", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_buffered_io": "true", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_for_zeros": "false", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_often": "false", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-06-10T21:20:36.283 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_compact_log_sync": "false", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_compact_min_size": "16777216", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_log_replay_check_allocations": "true", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_log_runway": "4194304", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_max_prefetch": "1048576", 2023-06-10T21:20:36.284 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_flush_size": "524288", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_min_log_runway": "1048576", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery": "false", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_shared_alloc_size": "65536", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluefs_sync_write": "false", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-06-10T21:20:36.285 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocation_from_file": "true", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_allocator": "hybrid", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-06-10T21:20:36.286 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_blobid_prealloc": "10240", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_create": "true", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_create": "false", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_path": "", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_db_size": "0", 2023-06-10T21:20:36.287 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_path": "", 2023-06-10T21:20:36.288 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_preallocate_file": "false", 2023-06-10T21:20:36.288 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_size": "107374182400", 2023-06-10T21:20:36.288 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_create": "false", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_path": "", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_block_wal_size": "100663296", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs": "true", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_env_mirror": "false", 2023-06-10T21:20:36.289 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune": "true", 2023-06-10T21:20:36.290 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-06-10T21:20:36.291 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-06-10T21:20:36.291 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-06-10T21:20:36.291 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-06-10T21:20:36.291 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size": "0", 2023-06-10T21:20:36.292 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-06-10T21:20:36.292 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-06-10T21:20:36.292 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-06-10T21:20:36.292 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-06-10T21:20:36.293 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cache_type": "2q", 2023-06-10T21:20:36.293 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-06-10T21:20:36.293 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_clone_cow": "true", 2023-06-10T21:20:36.294 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_algorithm": "snappy", 2023-06-10T21:20:36.294 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size": "0", 2023-06-10T21:20:36.294 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-06-10T21:20:36.294 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-06-10T21:20:36.295 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size": "0", 2023-06-10T21:20:36.295 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-06-10T21:20:36.295 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-06-10T21:20:36.296 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_mode": "none", 2023-06-10T21:20:36.296 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-06-10T21:20:36.296 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_csum_type": "crc32c", 2023-06-10T21:20:36.297 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_enforce_settings": "default", 2023-06-10T21:20:36.297 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_freelist": "false", 2023-06-10T21:20:36.297 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_fsck_abort": "false", 2023-06-10T21:20:36.298 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-06-10T21:20:36.298 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_bug21040": "false", 2023-06-10T21:20:36.298 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-06-10T21:20:36.299 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_inject_read_err": "false", 2023-06-10T21:20:36.299 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_legacy_omap": "false", 2023-06-10T21:20:36.299 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-06-10T21:20:36.299 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-06-10T21:20:36.300 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-06-10T21:20:36.300 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-06-10T21:20:36.300 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_prefill": "0.000000", 2023-06-10T21:20:36.300 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-06-10T21:20:36.301 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-06-10T21:20:36.301 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-06-10T21:20:36.301 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_small_allocations": "0", 2023-06-10T21:20:36.302 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-06-10T21:20:36.302 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_read": "true", 2023-06-10T21:20:36.302 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_default_buffered_write": "false", 2023-06-10T21:20:36.302 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops": "0", 2023-06-10T21:20:36.302 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-06-10T21:20:36.303 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fail_eio": "false", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs": "true", 2023-06-10T21:20:36.304 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-06-10T21:20:36.305 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_ignore_data_csum": "false", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-06-10T21:20:36.306 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_kvbackend": "rocksdb", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_log_op_age": "5.000000", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_alloc_size": "0", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size": "0", 2023-06-10T21:20:36.307 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_defer_interval": "3.000000", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_max_deferred_txc": "32", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size": "0", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-06-10T21:20:36.308 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_nid_prealloc": "1024", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size": "0", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_qfsck_on_mount": "true", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_retry_disk_reads": "3", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cf": "true", 2023-06-10T21:20:36.309 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-06-10T21:20:36.310 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-06-10T21:20:36.310 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_rocksdb_options_annex": "", 2023-06-10T21:20:36.310 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_coremask": "0x1", 2023-06-10T21:20:36.310 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_io_sleep": "5", 2023-06-10T21:20:36.311 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_max_io_completion": "0", 2023-06-10T21:20:36.311 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_spdk_mem": "512", 2023-06-10T21:20:36.311 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_sync_submit_transaction": "false", 2023-06-10T21:20:36.312 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_bytes": "67108864", 2023-06-10T21:20:36.312 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io": "0", 2023-06-10T21:20:36.312 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-06-10T21:20:36.312 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-06-10T21:20:36.313 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-06-10T21:20:36.313 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-06-10T21:20:36.313 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_tracing": "false", 2023-06-10T21:20:36.313 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-06-10T21:20:36.314 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-06-10T21:20:36.314 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved": "0", 2023-06-10T21:20:36.314 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-06-10T21:20:36.315 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-06-10T21:20:36.315 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-06-10T21:20:36.315 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-06-10T21:20:36.315 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-06-10T21:20:36.316 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-06-10T21:20:36.316 INFO:teuthology.orchestra.run.smithi064.stdout: "bluestore_zero_block_detection": "false", 2023-06-10T21:20:36.316 INFO:teuthology.orchestra.run.smithi064.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-06-10T21:20:36.317 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_action_update_interval": "2", 2023-06-10T21:20:36.317 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-06-10T21:20:36.317 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-06-10T21:20:36.317 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-06-10T21:20:36.318 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-06-10T21:20:36.318 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_mount_timeout": "10", 2023-06-10T21:20:36.318 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-06-10T21:20:36.318 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-06-10T21:20:36.319 INFO:teuthology.orchestra.run.smithi064.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-06-10T21:20:36.319 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-06-10T21:20:36.319 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-06-10T21:20:36.320 INFO:teuthology.orchestra.run.smithi064.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-06-10T21:20:36.320 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_signatures": "false", 2023-06-10T21:20:36.320 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_cluster_require_version": "2", 2023-06-10T21:20:36.320 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_signatures": "false", 2023-06-10T21:20:36.320 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_require_version": "2", 2023-06-10T21:20:36.321 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_signatures": "false", 2023-06-10T21:20:36.321 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_service_require_version": "2", 2023-06-10T21:20:36.321 INFO:teuthology.orchestra.run.smithi064.stdout: "cephx_sign_messages": "true", 2023-06-10T21:20:36.322 INFO:teuthology.orchestra.run.smithi064.stdout: "chdir": "", 2023-06-10T21:20:36.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client_acl_type": "", 2023-06-10T21:20:36.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client_asio_thread_count": "2", 2023-06-10T21:20:36.322 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_mid": "0.750000", 2023-06-10T21:20:36.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_cache_size": "16384", 2023-06-10T21:20:36.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_caps_release_delay": "5", 2023-06-10T21:20:36.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_check_pool_perm": "true", 2023-06-10T21:20:36.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_collect_and_send_global_metrics": "false", 2023-06-10T21:20:36.324 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_force_sync_read": "false", 2023-06-10T21:20:36.324 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_getattr_caps": "false", 2023-06-10T21:20:36.324 INFO:teuthology.orchestra.run.smithi064.stdout: "client_debug_inject_tick_delay": "0", 2023-06-10T21:20:36.324 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-06-10T21:20:36.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_die_on_failed_remount": "false", 2023-06-10T21:20:36.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_dirsize_rbytes": "true", 2023-06-10T21:20:36.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_force_lazyio": "false", 2023-06-10T21:20:36.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_fs": "", 2023-06-10T21:20:36.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_fixed_oldest_tid": "false", 2023-06-10T21:20:36.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_inject_release_failure": "false", 2023-06-10T21:20:36.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_inline_size": "4096", 2023-06-10T21:20:36.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_max_retries_on_remount_failure": "5", 2023-06-10T21:20:36.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mds_namespace": "", 2023-06-10T21:20:36.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_metadata": "", 2023-06-10T21:20:36.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_gid": "-1", 2023-06-10T21:20:36.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_timeout": "300", 2023-06-10T21:20:36.328 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mount_uid": "-1", 2023-06-10T21:20:36.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_mountpoint": "/", 2023-06-10T21:20:36.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_notify_timeout": "10", 2023-06-10T21:20:36.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc": "true", 2023-06-10T21:20:36.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty": "104857600", 2023-06-10T21:20:36.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_dirty_age": "5.000000", 2023-06-10T21:20:36.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_max_objects": "1000", 2023-06-10T21:20:36.330 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_size": "209715200", 2023-06-10T21:20:36.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_oc_target_dirty": "8388608", 2023-06-10T21:20:36.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_permissions": "true", 2023-06-10T21:20:36.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota": "true", 2023-06-10T21:20:36.331 INFO:teuthology.orchestra.run.smithi064.stdout: "client_quota_df": "true", 2023-06-10T21:20:36.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_bytes": "0", 2023-06-10T21:20:36.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_max_periods": "4", 2023-06-10T21:20:36.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_readahead_min": "131072", 2023-06-10T21:20:36.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_reconnect_stale": "false", 2023-06-10T21:20:36.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_shutdown_timeout": "30", 2023-06-10T21:20:36.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_snapdir": ".snap", 2023-06-10T21:20:36.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_tick_interval": "1", 2023-06-10T21:20:36.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_trace": "", 2023-06-10T21:20:36.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_try_dentry_invalidate": "false", 2023-06-10T21:20:36.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_faked_inos": "false", 2023-06-10T21:20:36.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_use_random_mds": "false", 2023-06-10T21:20:36.335 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog": "false", 2023-06-10T21:20:36.335 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-06-10T21:20:36.335 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_graylog_port": "12201", 2023-06-10T21:20:36.336 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_monitors": "default=true", 2023-06-10T21:20:36.336 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog": "false", 2023-06-10T21:20:36.336 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-06-10T21:20:36.336 INFO:teuthology.orchestra.run.smithi064.stdout: "clog_to_syslog_level": "info", 2023-06-10T21:20:36.337 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_addr": "-", 2023-06-10T21:20:36.337 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network": "", 2023-06-10T21:20:36.337 INFO:teuthology.orchestra.run.smithi064.stdout: "cluster_network_interface": "", 2023-06-10T21:20:36.337 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_isal": "false", 2023-06-10T21:20:36.338 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_level": "5", 2023-06-10T21:20:36.338 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zlib_winsize": "-15", 2023-06-10T21:20:36.338 INFO:teuthology.orchestra.run.smithi064.stdout: "compressor_zstd_level": "1", 2023-06-10T21:20:36.339 INFO:teuthology.orchestra.run.smithi064.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:0e5b060247566ee1047f2701e921ccfa9603f8b45612cb9fe16a73656ee83143", 2023-06-10T21:20:36.339 INFO:teuthology.orchestra.run.smithi064.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-06-10T21:20:36.339 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_op_num_threads": "6", 2023-06-10T21:20:36.339 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_alien_thread_cpu_cores": "", 2023-06-10T21:20:36.340 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_obc_lru_size": "10", 2023-06-10T21:20:36.340 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-06-10T21:20:36.340 INFO:teuthology.orchestra.run.smithi064.stdout: "crimson_seastar_smp": "1", 2023-06-10T21:20:36.341 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location": "", 2023-06-10T21:20:36.341 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook": "", 2023-06-10T21:20:36.341 INFO:teuthology.orchestra.run.smithi064.stdout: "crush_location_hook_timeout": "10", 2023-06-10T21:20:36.341 INFO:teuthology.orchestra.run.smithi064.stdout: "daemonize": "false", 2023-06-10T21:20:36.342 INFO:teuthology.orchestra.run.smithi064.stdout: "daos_pool": "tank", 2023-06-10T21:20:36.342 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-06-10T21:20:36.342 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-06-10T21:20:36.342 INFO:teuthology.orchestra.run.smithi064.stdout: "dbstore_db_name_prefix": "dbstore", 2023-06-10T21:20:36.343 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_alienstore": "0/5", 2023-06-10T21:20:36.343 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_allow_any_pool_priority": "false", 2023-06-10T21:20:36.343 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok": "1/5", 2023-06-10T21:20:36.344 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asok_assert_abort": "false", 2023-06-10T21:20:36.344 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_asserts_on_shutdown": "true", 2023-06-10T21:20:36.344 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_auth": "1/5", 2023-06-10T21:20:36.344 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bdev": "1/3", 2023-06-10T21:20:36.345 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluefs": "1/5", 2023-06-10T21:20:36.345 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_bluestore": "1/5", 2023-06-10T21:20:36.345 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_buffer": "0/1", 2023-06-10T21:20:36.346 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ceph_exporter": "1/5", 2023-06-10T21:20:36.346 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephfs_mirror": "0/5", 2023-06-10T21:20:36.346 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cephsqlite": "0/5", 2023-06-10T21:20:36.346 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_client": "0/5", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_compressor": "1/5", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_context": "0/1", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crush": "1/1", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_crypto": "1/5", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_cyanstore": "0/5", 2023-06-10T21:20:36.347 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_deliberately_leak_memory": "false", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_disable_randomized_ping": "false", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_dpdk": "1/5", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_eventtrace": "1/5", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filer": "0/1", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_filestore": "1/3", 2023-06-10T21:20:36.348 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_finisher": "1/1", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_fuse": "1/5", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeat_testing_span": "0", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_heartbeatmap": "1/5", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_immutable_obj_cache": "0/5", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_javaclient": "1/5", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journal": "1/3", 2023-06-10T21:20:36.349 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_journaler": "0/5", 2023-06-10T21:20:36.350 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_kstore": "1/5", 2023-06-10T21:20:36.350 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_leveldb": "4/5", 2023-06-10T21:20:36.350 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_lockdep": "0/1", 2023-06-10T21:20:36.350 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mclock": "1/5", 2023-06-10T21:20:36.350 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_balancer": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_locker": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_log_expire": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mds_migrator": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_memstore": "1/5", 2023-06-10T21:20:36.351 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgr": "2/5", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mgrc": "1/5", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_mon": "20/20", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_monc": "0/10", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_ms": "1/1", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_none": "0/5", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objclass": "0/5", 2023-06-10T21:20:36.352 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objectcacher": "0/5", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_objecter": "0/1", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_optracker": "0/5", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_osd": "1/5", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_paxos": "20/20", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_perfcounter": "1/5", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_prioritycache": "1/5", 2023-06-10T21:20:36.353 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rados": "0/5", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd": "0/5", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_mirror": "0/5", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_pwl": "0/5", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rbd_replay": "0/5", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_refs": "0/0", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_reserver": "1/1", 2023-06-10T21:20:36.354 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_access": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_datacache": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_dbstore": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_flight": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rgw_sync": "1/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_rocksdb": "4/5", 2023-06-10T21:20:36.355 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_backref": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cache": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_cleaner": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_device": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_epm": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-06-10T21:20:36.356 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_journal": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_lba": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_odata": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_omap": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_onode": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_t": "0/5", 2023-06-10T21:20:36.357 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_seastore_tm": "0/5", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_striper": "0/1", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_test": "0/5", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_throttle": "1/1", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_timer": "0/1", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "debug_tp": "0/5", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "device_failure_prediction_mode": "none", 2023-06-10T21:20:36.358 INFO:teuthology.orchestra.run.smithi064.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_graylog": "false", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_journald": "false", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_stderr": "true", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "err_to_syslog": "false", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event_tracing": "false", 2023-06-10T21:20:36.359 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_addr": "0.0.0.0", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_http_port": "9926", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_prio_limit": "5", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_sort_metrics": "true", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "exporter_stats_period": "5", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "fake_statfs_for_testing": "0", 2023-06-10T21:20:36.360 INFO:teuthology.orchestra.run.smithi064.stdout: "fatal_signal_handlers": "true", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_purge_ops": "10", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filer_max_truncate_ops": "128", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_apply_finisher_threads": "1", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_blackhole": "false", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_clone_range": "true", 2023-06-10T21:20:36.361 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_btrfs_snap": "true", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_caller_concurrency": "10", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_collect_device_partition_information": "true", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_commit_timeout": "600.000000", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_inject_read_err": "false", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_omap_check": "false", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_random_read_err": "0.000000", 2023-06-10T21:20:36.362 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_debug_verify_split": "false", 2023-06-10T21:20:36.363 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_dump_file": "", 2023-06-10T21:20:36.363 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-06-10T21:20:36.363 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-06-10T21:20:36.364 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fadvise": "true", 2023-06-10T21:20:36.364 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fail_eio": "true", 2023-06-10T21:20:36.364 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_shards": "16", 2023-06-10T21:20:36.364 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fd_cache_size": "128", 2023-06-10T21:20:36.365 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap": "false", 2023-06-10T21:20:36.365 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fiemap_threshold": "4096", 2023-06-10T21:20:36.365 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-06-10T21:20:36.365 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_index_retry_probability": "0.000000", 2023-06-10T21:20:36.366 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_inject_stall": "0", 2023-06-10T21:20:36.366 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_parallel": "false", 2023-06-10T21:20:36.366 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_trailing": "false", 2023-06-10T21:20:36.367 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_journal_writeahead": "false", 2023-06-10T21:20:36.367 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_kill_at": "0", 2023-06-10T21:20:36.367 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-06-10T21:20:36.367 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size": "0", 2023-06-10T21:20:36.368 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-06-10T21:20:36.368 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-06-10T21:20:36.368 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-06-10T21:20:36.368 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs": "0", 2023-06-10T21:20:36.369 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-06-10T21:20:36.369 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_other": "2", 2023-06-10T21:20:36.369 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_sync_interval": "5.000000", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size": "0", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-06-10T21:20:36.370 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_merge_threshold": "-10", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_min_sync_interval": "0.010000", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_odsync_write": "false", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend": "rocksdb", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_backend_path": "", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_omap_header_cache_size": "1024", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_ondisk_finisher_threads": "1", 2023-06-10T21:20:36.371 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_thread_timeout": "60", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_op_threads": "2", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_punch_hole": "false", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-06-10T21:20:36.372 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_bytes": "104857600", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_queue_max_ops": "50", 2023-06-10T21:20:36.373 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-06-10T21:20:36.374 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_seek_data_hole": "false", 2023-06-10T21:20:36.374 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc": "false", 2023-06-10T21:20:36.374 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-06-10T21:20:36.374 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_splice": "false", 2023-06-10T21:20:36.374 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_multiple": "2", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_split_rand_factor": "20", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_update_to": "1000", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-06-10T21:20:36.375 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_enable": "true", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-06-10T21:20:36.376 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_xfs_extsize": "false", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "filestore_zfs_snap": "false", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "fio_dir": "/tmp/fio", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "fsid": "b618768e-07d0-11ee-9b1e-001a4aab830c", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_allow_other": "true", 2023-06-10T21:20:36.377 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_atomic_o_trunc": "true", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_big_writes": "true", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_debug": "false", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_default_permissions": "false", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_disable_pagecache": "false", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_max_write": "0", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_multithreaded": "true", 2023-06-10T21:20:36.378 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_require_active_mds": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_set_user_groups": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_move": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_read": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_splice_write": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_syncfs_on_mksnap": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "fuse_use_invalidate_cb": "true", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-06-10T21:20:36.379 INFO:teuthology.orchestra.run.smithi064.stdout: "gss_target_name": "ceph", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_file": "", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_inject_failure": "0", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "heartbeat_interval": "5", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "host": "smithi064", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-06-10T21:20:36.380 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_max_size": "1073741824", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-06-10T21:20:36.381 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "immutable_object_cache_watermark": "0.900000", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "inject_early_sigterm": "false", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_agent_port": "6799", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "jaeger_tracing_enable": "false", 2023-06-10T21:20:36.382 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_aio": "true", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_align_min_size": "65536", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_align": "true", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_block_size": "4096", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_dio": "true", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_discard": "false", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_force_aio": "false", 2023-06-10T21:20:36.383 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_ignore_corruption": "false", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_bytes": "10485760", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_max_write_entries": "100", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_replay_from": "0", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_multiple": "0.000000", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-06-10T21:20:36.384 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_throttle_max_multiple": "0.000000", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_write_header_frequency": "0", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "journal_zero_on_create": "false", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prefetch_periods": "10", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_prezero_periods": "5", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "journaler_write_head_interval": "15", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "key": "", 2023-06-10T21:20:36.385 INFO:teuthology.orchestra.run.smithi064.stdout: "keyfile": "", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_backend": "rocksdb", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_default_stripe_size": "65536", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount": "false", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_fsck_on_mount_deep": "true", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_bytes": "67108864", 2023-06-10T21:20:36.386 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_max_ops": "512", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_nid_prealloc": "1024", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_onode_map_size": "1024", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_submit_transaction": "false", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "kstore_sync_transaction": "false", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_block_size": "65536", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_bloom_size": "0", 2023-06-10T21:20:36.387 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_cache_size": "536870912", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_compact_on_mount": "false", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_compression": "false", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_log": "", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_log_to_ceph_log": "true", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_max_open_files": "0", 2023-06-10T21:20:36.388 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_paranoid": "false", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "leveldb_write_buffer_size": "33554432", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "librados_thread_count": "2", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep": "false", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "lockdep_force_backtrace": "false", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "log_coarse_timestamps": "true", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-06-10T21:20:36.389 INFO:teuthology.orchestra.run.smithi064.stdout: "log_flush_on_exit": "false", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_host": "127.0.0.1", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_graylog_port": "12201", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_new": "1000", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_max_recent": "10000", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stderr_prefix": "", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_stop_at_utilization": "0.970000", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_file": "true", 2023-06-10T21:20:36.390 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_graylog": "false", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_journald": "false", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_stderr": "false", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "log_to_syslog": "false", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "max_rotating_auth_attempts": "10", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_action_on_write_error": "1", 2023-06-10T21:20:36.391 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_alternate_name_max": "8192", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_asio_thread_count": "2", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_export_pin": "true", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_dirs": "true", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_interval": "5", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_fragment_size_max": "100000", 2023-06-10T21:20:36.392 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_idle_threshold": "0.000000", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_interval": "10", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max": "-1", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_max_until": "-1", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_merge_size": "50", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_midchunk": "0.300000", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_rebalance": "0.100000", 2023-06-10T21:20:36.393 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_min_start": "0.200000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_minchunk": "0.001000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_mode": "0", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_max": "1.200000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_need_min": "0.800000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_sample_interval": "3.000000", 2023-06-10T21:20:36.394 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_bits": "3", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_rd": "25000.000000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_size": "10000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_split_wr": "10000.000000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_target_decay": "10.000000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_grace": "15.000000", 2023-06-10T21:20:36.395 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_interval": "4.000000", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_beacon_mon_down_grace": "60", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_memory_limit": "4294967296", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_mid": "0.700000", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_release_free_interval": "10", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_reservation": "0.050000", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-06-10T21:20:36.396 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_interval": "1", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cache_trim_threshold": "262144", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_delegate_inos_pct": "50", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_prealloc_inos": "1000", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-06-10T21:20:36.397 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_connect_bootstrapping": "false", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_damage_table_max_entries": "10000", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_auth_pins": "false", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_frag": "false", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_scatterstat": "false", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_debug_subtrees": "false", 2023-06-10T21:20:36.398 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_decay_halflife": "5.000000", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_default_dir_hash": "2", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_defer_session_stale": "true", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_deny_all_reconnect": "false", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_keys_per_op": "16384", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_commit_size": "10", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_max_entries": "0", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dir_prefetch": "true", 2023-06-10T21:20:36.399 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dirstat_min_interval": "1.000000", 2023-06-10T21:20:36.400 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_after_rejoin": "false", 2023-06-10T21:20:36.400 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_on_map": "false", 2023-06-10T21:20:36.400 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_file": "0", 2023-06-10T21:20:36.400 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_early_reply": "true", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enable_op_tracker": "true", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_enforce_unique_name": "true", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed": "true", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-06-10T21:20:36.401 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random": "true", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_forward_all_requests_to_auth": "false", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-06-10T21:20:36.402 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_cache_threshold": "1.500000", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_health_summarize_threshold": "10", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_grace": "15.000000", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_heartbeat_reset_grace": "1000", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_health_dummy": "false", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-06-10T21:20:36.403 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_migrator_session_race": "false", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_join_fs": "", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_journal_format": "1", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_create_at": "0", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_export_at": "0", 2023-06-10T21:20:36.404 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_import_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_expire_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_journal_replay_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_link_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_mdstable_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_openc_at": "0", 2023-06-10T21:20:36.405 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_kill_rename_at": "0", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_events_per_segment": "1024", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_events": "-1", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_max_segments": "128", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_pause": "false", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_segment_size": "0", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_skip_corrupt_events": "false", 2023-06-10T21:20:36.406 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_log_warn_factor": "2.000000", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_caps_per_client": "1048576", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_flushes": "100000", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_completed_requests": "100000", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_export_size": "20971520", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_file_recover": "32", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_files": "64", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops": "8192", 2023-06-10T21:20:36.407 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_snaps_per_dir": "100", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_max_xattr_pairs_size": "65536", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_metrics_update_interval": "2", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_per_client": "100", 2023-06-10T21:20:36.408 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_min_caps_working_set": "10000", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_numa_node": "-1", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_complaint_time": "30.000000", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_duration": "600", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_history_size": "20", 2023-06-10T21:20:36.409 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_op_log_threshold": "5", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_grace": "15", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_ping_interval": "5", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_caps": "30000", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_max_decay_threshold": "131072", 2023-06-10T21:20:36.410 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_recall_warning_threshold": "262144", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_reconnect_timeout": "45.000000", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_interval": "1.000000", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-06-10T21:20:36.411 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_gid": "0", 2023-06-10T21:20:36.412 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_root_ino_uid": "0", 2023-06-10T21:20:36.412 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-06-10T21:20:36.412 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_evict": "true", 2023-06-10T21:20:36.413 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_blocklist_on_timeout": "true", 2023-06-10T21:20:36.413 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-06-10T21:20:36.413 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-06-10T21:20:36.413 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_decay_rate": "10.000000", 2023-06-10T21:20:36.414 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_cap_acquisition_throttle": "500000", 2023-06-10T21:20:36.414 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-06-10T21:20:36.414 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-06-10T21:20:36.415 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_shutdown_check": "0", 2023-06-10T21:20:36.415 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_skip_ino": "0", 2023-06-10T21:20:36.415 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_sleep_rank_change": "0.000000", 2023-06-10T21:20:36.415 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_max_uid": "4294967294", 2023-06-10T21:20:36.416 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_min_uid": "0", 2023-06-10T21:20:36.416 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_snap_rstat": "false", 2023-06-10T21:20:36.416 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_standby_replay_damaged": "false", 2023-06-10T21:20:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_symlink_recovery": "true", 2023-06-10T21:20:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_task_status_update_interval": "2.000000", 2023-06-10T21:20:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_exports": "0", 2023-06-10T21:20:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_thrash_fragments": "0", 2023-06-10T21:20:36.417 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_tick_interval": "5.000000", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_valgrind_exit": "false", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_backtrace": "1", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_verify_scatter": "false", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_ino_prealloc": "false", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mds_wipe_sessions": "false", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "mempool_debug": "false", 2023-06-10T21:20:36.418 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_debug_omit_block_device_write": "false", 2023-06-10T21:20:36.419 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_device_bytes": "1073741824", 2023-06-10T21:20:36.419 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_set": "false", 2023-06-10T21:20:36.419 INFO:teuthology.orchestra.run.smithi064.stdout: "memstore_page_size": "65536", 2023-06-10T21:20:36.419 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_bytes": "134217728", 2023-06-10T21:20:36.419 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_messages": "512", 2023-06-10T21:20:36.420 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-06-10T21:20:36.420 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_connect_retry_interval": "1.000000", 2023-06-10T21:20:36.420 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-06-10T21:20:36.420 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-06-10T21:20:36.421 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_disabled_modules": "", 2023-06-10T21:20:36.421 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-06-10T21:20:36.421 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_max_pg_num_change": "128", 2023-06-10T21:20:36.422 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_bytes": "134217728", 2023-06-10T21:20:36.422 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mds_messages": "128", 2023-06-10T21:20:36.422 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-06-10T21:20:36.422 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_bytes": "134217728", 2023-06-10T21:20:36.423 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_mon_messages": "128", 2023-06-10T21:20:36.423 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_bytes": "536870912", 2023-06-10T21:20:36.423 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_osd_messages": "8192", 2023-06-10T21:20:36.424 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_pool": "true", 2023-06-10T21:20:36.424 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_service_beacon_grace": "60.000000", 2023-06-10T21:20:36.424 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_standby_modules": "true", 2023-06-10T21:20:36.425 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_period": "5", 2023-06-10T21:20:36.425 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_stats_threshold": "5", 2023-06-10T21:20:36.425 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_tick_period": "2", 2023-06-10T21:20:36.426 INFO:teuthology.orchestra.run.smithi064.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-06-10T21:20:36.426 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_accept_timeout_factor": "2.000000", 2023-06-10T21:20:36.426 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_delete": "true", 2023-06-10T21:20:36.427 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_allow_pool_size_one": "false", 2023-06-10T21:20:36.427 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_auth_validate_all_caps": "true", 2023-06-10T21:20:36.427 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-06-10T21:20:36.427 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-06-10T21:20:36.428 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_bytes": "104857600", 2023-06-10T21:20:36.428 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_directed_command_retry": "2", 2023-06-10T21:20:36.428 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval": "3.000000", 2023-06-10T21:20:36.428 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-06-10T21:20:36.429 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-06-10T21:20:36.429 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-06-10T21:20:36.429 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_hunt_parallel": "3", 2023-06-10T21:20:36.429 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_log_interval": "1.000000", 2023-06-10T21:20:36.430 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-06-10T21:20:36.430 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_interval": "10.000000", 2023-06-10T21:20:36.430 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_client_ping_timeout": "30.000000", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_allowed": "1.000000", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_file_level": "debug", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_file": "false", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog": "false", 2023-06-10T21:20:36.431 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_journald": "true", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_stderr": "false", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-06-10T21:20:36.432 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_bootstrap": "false", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_start": "false", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_compact_on_trim": "true", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_persist_interval": "10", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_con_tracker_score_halflife": "43200", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_config_key_max_entry_size": "65536", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_cpu_threads": "4", 2023-06-10T21:20:36.433 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_crush_min_required_version": "hammer", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_daemon_bytes": "419430400", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_crit": "5", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_avail_warn": "5", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_data_size_warn": "16106127360", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_block_osdmap_trim": "false", 2023-06-10T21:20:36.434 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-06-10T21:20:36.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_json": "false", 2023-06-10T21:20:36.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-06-10T21:20:36.435 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_dump_transactions": "false", 2023-06-10T21:20:36.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_extra_checks": "false", 2023-06-10T21:20:36.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-06-10T21:20:36.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-06-10T21:20:36.436 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_quincy": "false", 2023-06-10T21:20:36.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_no_require_reef": "false", 2023-06-10T21:20:36.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-06-10T21:20:36.437 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_delta_reset_interval": "10.000000", 2023-06-10T21:20:36.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_dns_srv_name": "ceph-mon", 2023-06-10T21:20:36.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_down_mkfs_grace": "60", 2023-06-10T21:20:36.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_default_strategy": "3", 2023-06-10T21:20:36.438 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_election_timeout": "5.000000", 2023-06-10T21:20:36.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-06-10T21:20:36.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_divisor": "2", 2023-06-10T21:20:36.439 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_elector_ping_timeout": "2.000000", 2023-06-10T21:20:36.440 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_enable_op_tracker": "true", 2023-06-10T21:20:36.440 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_fake_pool_delete": "false", 2023-06-10T21:20:36.440 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_force_quorum_join": "false", 2023-06-10T21:20:36.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_globalid_prealloc": "10000", 2023-06-10T21:20:36.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_detail_to_clog": "true", 2023-06-10T21:20:36.441 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_log_update_period": "5", 2023-06-10T21:20:36.442 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_max_detail": "50", 2023-06-10T21:20:36.442 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog": "true", 2023-06-10T21:20:36.442 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_interval": "600", 2023-06-10T21:20:36.442 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-06-10T21:20:36.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host": "[v2:172.21.15.64:3300/0,v1:172.21.15.64:6789/0] [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.64:3301/0,v1:172.21.15.64:6790/0]", 2023-06-10T21:20:36.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_host_override": "", 2023-06-10T21:20:36.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_initial_members": "", 2023-06-10T21:20:36.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-06-10T21:20:36.443 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_keyvaluedb": "rocksdb", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease": "5.000000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-06-10T21:20:36.444 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_full_interval": "50", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max": "10000", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_log_max_summary": "50", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_entries_per_event": "4096", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_log_epochs": "500", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mdsmap_epochs": "500", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_mgrmap_epochs": "500", 2023-06-10T21:20:36.445 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_osd": "10000", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pg_per_osd": "10000", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_pool_pg_num": "65536", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_force_trim_to": "0", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mds_skip_sanity": "true", 2023-06-10T21:20:36.446 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_autotune": "true", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_memory_target": "2147483648", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_beacon_grace": "30", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_digest_period": "5", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_inactive_grace": "60", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_mkfs_grace": "240", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-06-10T21:20:36.447 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_min_osdmap_epochs": "500", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_complaint_time": "30", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_duration": "600", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_size": "20", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_size": "20", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_history_slow_op_threshold": "10", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_op_log_threshold": "5", 2023-06-10T21:20:36.448 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_in": "false", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_auto_mark_new_in": "true", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-06-10T21:20:36.449 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size": "500", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_cache_size_min": "134217728", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_crush_smoke_test": "true", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_destroyed_out_interval": "600", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_interval": "600", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-06-10T21:20:36.450 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_force_trim_to": "0", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_full_ratio": "0.950000", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_halflife": "3600", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_max_interval": "300", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_laggy_weight": "0.300000", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-06-10T21:20:36.451 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_max_creating_pgs": "1024", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_max_initial_pgs": "1024", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_down_reporters": "2", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp": "true", 2023-06-10T21:20:36.452 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_report_timeout": "900", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_num_repaired": "10", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osd_warn_op_age": "32.000000", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-06-10T21:20:36.453 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_interval": "10", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_min": "10000", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_stuck_threshold": "60", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_objects": "10000", 2023-06-10T21:20:36.454 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_per_osd": "0", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_crit_threshold": "0", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_pool_quota_warn_threshold": "0", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_probe_timeout": "2.000000", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_change": "0.050000", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_max_osds": "4", 2023-06-10T21:20:36.455 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_interval": "86400", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_max_keys": "100", 2023-06-10T21:20:36.456 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_scrub_timeout": "300", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_session_timeout": "300", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_smart_report_timeout": "5", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stat_smooth_intervals": "6", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_min_size": "2", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_pool_size": "4", 2023-06-10T21:20:36.457 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_subscribe_interval": "86400.000000", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_debug": "false", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_keys": "2000", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_max_payload_size": "1048576", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_provider_kill_at": "0", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_requester_kill_at": "0", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_sync_timeout": "60.000000", 2023-06-10T21:20:36.458 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_target_pg_per_osd": "100", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_tick_interval": "5", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_interval": "300.000000", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_use_min_delay_socket": "false", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_older_version_delay": "604800", 2023-06-10T21:20:36.459 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_filestore_osds": "true", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-06-10T21:20:36.460 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_misplaced": "false", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_older_version": "true", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_app": "true", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-06-10T21:20:36.461 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_on_too_few_osds": "false", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "monmap": "", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-06-10T21:20:36.462 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "motr_tracing_enabled": "false", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_op_threads": "3", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_buffer_size": "131072", 2023-06-10T21:20:36.463 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_cm": "false", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_device_name": "", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_dscp": "96", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_gid_idx": "0", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_local_gid": "", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_polling_us": "1000", 2023-06-10T21:20:36.464 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_port_num": "1", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_roce_ver": "1", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_send_buffers": "1024", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_sl": "3", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_support_srq": "true", 2023-06-10T21:20:36.465 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_rdma_type": "ib", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_async_reap_threshold": "5", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_before_connect": "false", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv4": "true", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_ipv6": "false", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr1": "true", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_msgr2": "true", 2023-06-10T21:20:36.466 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_max": "7568", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_port_min": "6800", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_prefer_ipv4": "false", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_count": "3", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_bind_retry_delay": "5", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_client": "false", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mds": "false", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mgr": "false", 2023-06-10T21:20:36.467 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_mon": "false", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_blackhole_osd": "false", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_client_mode": "crc secure", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_mode": "crc secure", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_cluster_type": "", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_compress_secure": "false", 2023-06-10T21:20:36.468 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_idle_timeout": "900", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_connection_ready_timeout": "10", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_data": "true", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_crc_header": "true", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bad_msg": "false", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_bug": "true", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_old_message": "true", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_skipped_message": "false", 2023-06-10T21:20:36.469 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_die_on_unhandled_msg": "false", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_coremask": "0xF", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_devs_allowlist": "", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_enable_tso": "true", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-06-10T21:20:36.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hugepages": "", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_flow_control": "true", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_lro": "true", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_memory_channel": "4", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-06-10T21:20:36.471 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_pmd": "", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_port_id": "0", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_corrupt_message_level": "1", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_dump_on_send": "false", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_initial_backoff": "0.200000", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_max": "1.000000", 2023-06-10T21:20:36.472 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_probability": "0.000000", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_delay_type": "", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_internal_delays": "0.000000", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_network_congestion": "0", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_inject_socket_failures": "0", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_learn_addr_from_peer": "true", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_accept_failures": "4", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_max_backoff": "15.000000", 2023-06-10T21:20:36.473 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_client_mode": "secure crc", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_cluster_mode": "secure crc", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_mon_service_mode": "secure crc", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_min_size": "1024", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compress_mode": "none", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_osd_compression_algorithm": "snappy", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-06-10T21:20:36.474 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_pq_min_cost": "65536", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_public_type": "", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_service_mode": "crc secure", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_listen_backlog": "512", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_nodelay": "true", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_tcp_rcvbuf": "0", 2023-06-10T21:20:36.475 INFO:teuthology.orchestra.run.smithi064.stdout: "ms_type": "async+posix", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "no_config_file": "false", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_completion_locks_per_session": "32", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_debug_inject_relock_delay": "false", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_op_bytes": "104857600", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inflight_ops": "1024", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_inject_no_watch_ping": "false", 2023-06-10T21:20:36.476 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_tick_interval": "5.000000", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "objecter_timeout": "10.000000", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "objectstore_blackhole": "false", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "openssl_engine_opts": "", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_delay_time": "5.000000", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_hist_halflife": "1000", 2023-06-10T21:20:36.477 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_low_ops": "2", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_max_ops": "4", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_quantize_effort": "0.100000", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_agent_slop": "0.020000", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_allow_recovery_below_min_size": "true", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_asio_thread_count": "2", 2023-06-10T21:20:36.478 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_async_recovery_min_cost": "100", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_auto_mark_unfound_lost": "false", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_retry_interval": "30.000000", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_max": "512", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backfill_scan_min": "64", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_degraded": "false", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_peering": "false", 2023-06-10T21:20:36.479 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_backoff_on_unfound": "true", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_beacon_report_interval": "300", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_duration": "30", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_max_block_size": "67108864", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_bench_small_size_max_iops": "100", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blkin_trace_all": "false", 2023-06-10T21:20:36.480 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_blocked_scrub_grace_period": "120", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_for_log_corruption": "false", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-06-10T21:20:36.481 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_class_update_on_start": "true", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_cap": "256", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_message_size_cap": "524288000", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_op_priority": "63", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_client_watch_timeout": "30", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_command_max_records": "256", 2023-06-10T21:20:36.482 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_compact_on_start": "false", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_chooseleaf_type": "1", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_initial_weight": "-1.000000", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_on_start": "true", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_crush_update_weight_set": "true", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-06-10T21:20:36.483 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_duration": "0", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_feed_pullee": "-1", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-06-10T21:20:36.484 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_misdirected_ops": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_acting_change": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_no_purge_strays": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_op_order": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pg_log_writeout": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_pretend_recovery_active": "false", 2023-06-10T21:20:36.485 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_shutdown": "false", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_cached_snaps": "false", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_missing_on_start": "false", 2023-06-10T21:20:36.486 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_snaps": "false", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_keys": "1024", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-06-10T21:20:36.487 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_stride": "524288", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_data_pool_replay_window": "45", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_default_notify_timeout": "30", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep": "0.000000", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-06-10T21:20:36.488 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_discard_disconnected_ops": "true", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_enable_op_tracker": "true", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_extblkdev_plugins": "vdo", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_info": "true", 2023-06-10T21:20:36.489 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown": "true", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_fast_shutdown_timeout": "15", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_function_tracing": "false", 2023-06-10T21:20:36.490 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_grace": "20", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_interval": "6", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_peers": "10", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_min_size": "2000", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_stale": "600", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-06-10T21:20:36.491 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_max_size": "100000", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_min_size": "1000", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-06-10T21:20:36.492 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_flush_on_shutdown": "true", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_journal_size": "5120", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_kill_backfill_at": "0", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_loop_before_reset_tphandle": "64", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_cache_size": "50", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_dedup": "true", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max": "40", 2023-06-10T21:20:36.493 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_message_max_bytes": "10485760", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_map_share_max_epochs": "40", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_name_len": "100", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_attr_size": "0", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_backfills": "1", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_count": "5", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_markdown_period": "600", 2023-06-10T21:20:36.494 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_name_len": "2048", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_namespace_len": "256", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_object_size": "134217728", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_omap_entries_per_request": "1024", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_blocked_by": "16", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_log_entries": "10000", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-06-10T21:20:36.495 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_pgls": "1024", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_cost": "8388608", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_push_objects": "10", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_scrubs": "1", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_trimming_pgs": "2", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_op_reply_len": "64", 2023-06-10T21:20:36.496 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_max_write_size": "90", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-06-10T21:20:36.497 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_override_recovery_settings": "false", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_profile": "balanced", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-06-10T21:20:36.498 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mclock_skip_benchmark": "false", 2023-06-10T21:20:36.499 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_base": "805306368", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_min": "134217728", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target": "4294967296", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_autotune": "false", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-06-10T21:20:36.500 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_pg_log_entries": "250", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_min_recovery_priority": "0", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_interval": "30", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_interval": "5", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_report_max_in_flight": "2", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_cache_shards": "32", 2023-06-10T21:20:36.501 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_num_op_tracker_shard": "32", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_auto_affinity": "true", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_node": "-1", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_numa_prefer_iface": "true", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objecter_finishers": "1", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore": "bluestore", 2023-06-10T21:20:36.502 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_fuse": "false", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_objectstore_tracing": "false", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_complaint_time": "30.000000", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_duration": "600", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_size": "20", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_size": "20", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-06-10T21:20:36.503 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_log_threshold": "5", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards": "0", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_hdd": "5", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_shards_ssd": "8", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard": "0", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-06-10T21:20:36.504 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_pq_min_cost": "65536", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue": "mclock_scheduler", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_queue_cut_off": "high", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_suicide_timeout": "150", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_op_thread_timeout": "15", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_open_classes_on_start": "true", 2023-06-10T21:20:36.505 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_os_flags": "0", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_peering_op_priority": "255", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_cost": "1048576", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_delete_priority": "5", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_dups_tracked": "3000", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_max": "10000", 2023-06-10T21:20:36.506 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_log_trim_min": "100", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_object_context_cache_count": "64", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pg_stat_report_interval_max": "500", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-06-10T21:20:36.507 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crimson": "false", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_crush_rule": "-1", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_ec_fast_read": "false", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-06-10T21:20:36.508 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_bulk": "false", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nodelete": "false", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_flags": "0", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-06-10T21:20:36.509 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_min_size": "0", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pg_num": "32", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_pgp_num": "0", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_size": "3", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_default_type": "replicated", 2023-06-10T21:20:36.510 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_pool_use_gmt_hitset": "true", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_push_per_object_cost": "1000", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_read_ec_check_for_errors": "false", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap": "true", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recover_clone_overlap_limit": "10", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_cost": "20971520", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_delay_start": "0.000000", 2023-06-10T21:20:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active": "0", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_hdd": "3", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_active_ssd": "10", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_chunk": "8388608", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_max_single_start": "1", 2023-06-10T21:20:36.512 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_priority": "3", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_op_warn_multiple": "16", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_priority": "5", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_retry_interval": "30.000000", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep": "0.000000", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-06-10T21:20:36.513 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_repair_during_recovery": "false", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_requested_scrub_priority": "120", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_rollback_to_cluster_snap": "", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair": "false", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-06-10T21:20:36.514 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_hour": "0", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_begin_week_day": "0", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_max": "25", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_chunk_min": "5", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_cost": "52428800", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_during_recovery": "false", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_hour": "0", 2023-06-10T21:20:36.515 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_end_week_day": "0", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_event_cost": "4096", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_invalid_stats": "true", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_load_threshold": "0.500000", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_interval": "604800.000000", 2023-06-10T21:20:36.516 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_max_preemptions": "5", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_min_interval": "86400.000000", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_priority": "5", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_reservation_timeout": "5000", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_sleep": "0.000000", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-06-10T21:20:36.517 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_shutdown_pgref_assert": "false", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_skip_data_digest": "false", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_smart_report_timeout": "5", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_cost": "1048576", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_priority": "5", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep": "0.000000", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-06-10T21:20:36.518 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_stats_update_period_scrubbing": "15", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_target_transaction_size": "30", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-06-10T21:20:36.519 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_default_cache_mode": "writeback", 2023-06-10T21:20:36.520 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_tracing": "false", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_use_stale_snap": "false", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "osdc_blkin_trace_all": "false", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_kill_at": "0", 2023-06-10T21:20:36.521 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_max_join_drift": "10", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min": "500", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_min_wait": "0.050000", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_propose_interval": "1.000000", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max": "500", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_max_multiplier": "20", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_service_trim_min": "250", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_stash_full_interval": "25", 2023-06-10T21:20:36.522 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_max": "500", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "paxos_trim_min": "250", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "perf": "true", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "pid_file": "", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addr": "-", 2023-06-10T21:20:36.523 INFO:teuthology.orchestra.run.smithi064.stdout: "public_addrv": "", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "public_bind_addr": "-", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network": "172.21.0.0/20", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "public_network_interface": "", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_enabled": "false", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "qat_compressor_session_max_number": "256", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_mon_op_timeout": "0", 2023-06-10T21:20:36.524 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_osd_op_timeout": "0", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rados_tracing": "false", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_atime_update_interval": "60", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_parent_reads": "false", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_balance_snap_reads": "false", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blkin_trace_all": "false", 2023-06-10T21:20:36.525 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_expire_seconds": "0", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_blocklist_on_break_lock": "true", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache": "true", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_block_writes_upfront": "false", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty": "25165824", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_max_dirty_object": "0", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_policy": "writearound", 2023-06-10T21:20:36.526 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_size": "33554432", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_target_dirty": "16777216", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_clone_copy_on_read": "false", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_compression_hint": "none", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_concurrent_management_ops": "10", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-06-10T21:20:36.527 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_clone_format": "auto", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_data_pool": "", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_features": "61", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_format": "2", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_map_options": "", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_order": "22", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_pool": "rbd", 2023-06-10T21:20:36.528 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_count": "0", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_default_stripe_unit": "0", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_disable_zero_copy_writes": "true", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_granularity_bytes": "65536", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_enable_alloc_hint": "true", 2023-06-10T21:20:36.529 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler": "simple", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_commit_age": "5.000000", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-06-10T21:20:36.530 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_flush_interval": "0", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_order": "24", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_pool": "", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_journal_splay_width": "4", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_parent_reads": "false", 2023-06-10T21:20:36.531 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_localize_snap_reads": "false", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-06-10T21:20:36.532 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_type": "simple", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-06-10T21:20:36.533 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_autotune": "true", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_base": "805306368", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-06-10T21:20:36.534 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_memory_target": "4294967296", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_delete_delay": "0", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_replay_delay": "0", 2023-06-10T21:20:36.535 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove": "false", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_mtime_update_interval": "60", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_non_blocking_aio": "true", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_thread_timeout": "60", 2023-06-10T21:20:36.536 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_op_threads": "1", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_parent_cache_enabled": "false", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_mode": "disabled", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_path": "/tmp", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_persistent_cache_size": "1073741824", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_plugins": "", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst": "0", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-06-10T21:20:36.537 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_bps_limit": "0", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_exclude_ops": "0", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst": "0", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_iops_limit": "0", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst": "0", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-06-10T21:20:36.538 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_bps_limit": "0", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst": "0", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_read_iops_limit": "0", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_schedule_tick_min": "50", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst": "0", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-06-10T21:20:36.539 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_bps_limit": "0", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst": "0", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_qos_write_iops_limit": "0", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_quiesce_notification_attempts": "10", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_read_from_replica_policy": "default", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-06-10T21:20:36.540 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_max_bytes": "524288", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_readahead_trigger_requests": "10", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_request_timed_out_seconds": "30", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_skip_partial_discard": "true", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_tracing": "false", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_names": "true", 2023-06-10T21:20:36.541 INFO:teuthology.orchestra.run.smithi064.stdout: "rbd_validate_pool": "true", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_base_url": "", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "restapi_log_level": "", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_acl_grants_max_num": "100", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_admin_entry": "admin", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_backend_store": "rados", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_barbican_url": "", 2023-06-10T21:20:36.542 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_beast_enable_async": "true", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_max_aio": "128", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_quota_ttl": "600", 2023-06-10T21:20:36.543 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_enabled": "true", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_expiry_interval": "900", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cache_lru_size": "10000", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_config_store": "rados", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_content_length_compat": "false", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress": "true", 2023-06-10T21:20:36.544 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cors_rules_max_num": "100", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_cross_domain_policy": "", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_default_encryption_key": "", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_addr": "", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_ca_path": "", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_cert": "", 2023-06-10T21:20:36.545 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_client_key": "", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_password": "", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_kmip_username": "", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_require_ssl": "true", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-06-10T21:20:36.546 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-06-10T21:20:36.547 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_suppress_logs": "true", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_addr": "", 2023-06-10T21:20:36.548 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_auth": "token", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_namespace": "", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_prefix": "", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-06-10T21:20:36.549 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_token_file": "", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_buffersize": "524288", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_limit": "1024", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_low_speed_time": "300", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_tcp_keepalive": "0", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-06-10T21:20:36.550 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_fadvise": "4", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-06-10T21:20:36.551 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_changes_size": "1000", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_num_shards": "128", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_log_window": "30", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_notify_interval_msec": "0", 2023-06-10T21:20:36.552 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_poll_interval": "20", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_data_sync_spawn_window": "20", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_data_log_backing": "fifo", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_region_info_oid": "default.region", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_defer_to_bucket_acls": "", 2023-06-10T21:20:36.553 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-06-10T21:20:36.554 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_res": "500.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_name": "", 2023-06-10T21:20:36.555 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dns_s3website_name": "", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_dynamic_resharding": "true", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_gc_threads": "true", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_lc_threads": "true", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_ops_log": "false", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_quota_threads": "true", 2023-06-10T21:20:36.556 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_static_website": "false", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enable_usage_log": "false", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_enforce_swift_acls": "true", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_exit_timeout_secs": "120", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_expose_bucket": "false", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_extended_http_attrs": "", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_filter": "none", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-06-10T21:20:36.557 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_frontends": "beast port=7480", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_concurrent_io": "10", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred": "50", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_objs": "32", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_queue_size": "134213632", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_max_trim_chunk": "16", 2023-06-10T21:20:36.558 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_obj_min_wait": "7200", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_max_time": "3600", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_gc_processor_period": "3600", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_get_obj_window_size": "16777216", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_healthcheck_disabling_path": "", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ignore_get_invalid_range": "false", 2023-06-10T21:20:36.559 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_init_timeout": "300", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_domain": "", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password": "", 2023-06-10T21:20:36.560 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_password_path": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_project": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_tenant": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_token_path": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_admin_user": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_api_version": "2", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_domain": "", 2023-06-10T21:20:36.561 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_password": "", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_project": "", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_tenant": "", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_barbican_user": "", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_implicit_tenants": "false", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-06-10T21:20:36.562 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_service_token_enabled": "false", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_token_cache_size": "10000", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_url": "", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_keystone_verify_ssl": "true", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_debug_interval": "-1", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_lock_max_time": "90", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_objs": "32", 2023-06-10T21:20:36.563 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_rules": "1000", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_worker": "3", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_max_wp_worker": "3", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lc_thread_delay": "0", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_dnattr": "uid", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-06-10T21:20:36.564 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_searchfilter": "", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ldap_uri": "ldaps://", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_http_headers": "", 2023-06-10T21:20:36.565 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_nonexistent_bucket": "false", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_log_object_name_utc": "false", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_name_len": "0", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attr_size": "0", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_attrs_num_in_req": "0", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_chunk_size": "4194304", 2023-06-10T21:20:36.566 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_concurrent_requests": "1024", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_dynamic_shards": "1999", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_listing_results": "1000", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_notify_retries": "3", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_objs_per_shard": "100000", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_param_size": "1048576", 2023-06-10T21:20:36.567 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_put_size": "5368709120", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_max_slo_entries": "1000", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_log_max_shards": "64", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_md_notify_interval_msec": "200", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_poll_interval": "20", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_meta_sync_spawn_window": "20", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-06-10T21:20:36.568 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_mp_lock_max_time": "600", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_min_part_size": "5242880", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_fhcache_size": "2017", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-06-10T21:20:36.569 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_lru_lanes": "5", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_max_gc": "300", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_gc_threads": "false", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_lc_threads": "false", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_quota_threads": "false", 2023-06-10T21:20:36.570 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_run_sync_thread": "false", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_async_rados_threads": "32", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_num_control_oids": "8", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_numa_node": "-1", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_stripe_size": "4194304", 2023-06-10T21:20:36.571 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_chunk_size": "100", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_gc_interval": "600", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_objexp_hints_num_shards": "127", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_thread_timeout": "600", 2023-06-10T21:20:36.572 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_op_tracing": "false", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_token": "", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_url": "", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_opa_verify_ssl": "true", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_rados": "false", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_ops_log_socket_path": "", 2023-06-10T21:20:36.573 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval": "2.000000", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_push_interval_max": "30.000000", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_period_root_pool": ".rgw.root", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-06-10T21:20:36.574 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_continue": "true", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_print_prohibited_content_length": "false", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rados_tracing": "false", 2023-06-10T21:20:36.575 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm": "", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_id": "", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region": "", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_region_root_pool": ".rgw.root", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_region_enforcement": "false", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-06-10T21:20:36.576 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_request_uri": "", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_batch_size": "64", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_max_aio": "128", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_num_logs": "16", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_reshard_thread_interval": "600", 2023-06-10T21:20:36.577 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_resolve_cname": "false", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_rest_getusage_op_compat": "false", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_run_sync_thread": "true", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_keystone": "false", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_ldap": "false", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_rados": "true", 2023-06-10T21:20:36.578 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_auth_use_sts": "false", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_s3_success_create_obj_status": "0", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_scheduler_type": "throttler", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_script_uri": "", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_service_provider_name": "", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-06-10T21:20:36.579 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_id": "", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_client_secret": "", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_entry": "sts", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_key": "sts", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_max_session_duration": "43200", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_min_session_duration": "900", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sts_token_introspection_url": "", 2023-06-10T21:20:36.580 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_account_in_url": "false", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_entry": "auth", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_auth_url": "", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_custom_header": "", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_enforce_content_length": "false", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_need_stats": "true", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_tenant_name": "", 2023-06-10T21:20:36.581 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_token_expiration": "86400", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url": "", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_url_prefix": "swift", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_swift_versioning_enabled": "false", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_lease_period": "120", 2023-06-10T21:20:36.582 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_interval": "1200", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_obj_etag_verify": "false", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_history_size": "4096", 2023-06-10T21:20:36.583 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_thread_pool_size": "512", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_comment": "", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_createby": "", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_encoding": "", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_flag": "false", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_origin": "", 2023-06-10T21:20:36.584 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_sha_unit": "524288", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_torrent_tracker": "", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_trust_forwarded_https": "false", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_log_tick_interval": "30", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_shards": "32", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_usage_max_user_shards": "1", 2023-06-10T21:20:36.585 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_use_opa_authz": "false", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_default_quota_max_size": "-1", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_max_buckets": "1000", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_policies_max_num": "100", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-06-10T21:20:36.586 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_interval": "86400", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_user_unique_email": "true", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_verify_ssl": "true", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_website_routing_rules_max_num": "50", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone": "", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_id": "", 2023-06-10T21:20:36.587 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup": "", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_id": "", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_auto_session_clear": "true", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rgwlc_skip_bucket_step": "false", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_block_size": "4096", 2023-06-10T21:20:36.588 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_shard_bits": "4", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_size": "536870912", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cache_type": "binned_lru", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-06-10T21:20:36.589 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_compaction_stats": "false", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_extended_stats": "false", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_collect_memory_stats": "false", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_index_type": "binary_search", 2023-06-10T21:20:36.590 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_log_to_ceph_log": "true", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_metadata_block_size": "4096", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_partition_filters": "false", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_perf": "false", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "rotating_keys_renewal_timeout": "10", 2023-06-10T21:20:36.591 INFO:teuthology.orchestra.run.smithi064.stdout: "run_dir": "/var/run/ceph", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_block_create": "true", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cache_lru_size": "67108864", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_cbjournal_size": "5368709120", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_max_object_size": "16777216", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_device_size": "53687091200", 2023-06-10T21:20:36.592 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_capacity": "16", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_journal_iodepth_limit": "5", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_main_device_type": "SSD", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_max_concurrent_transactions": "8", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-06-10T21:20:36.593 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "seastore_segment_size": "67108864", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "set_keepcaps": "false", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "setgroup": "ceph", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser": "ceph", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "setuser_match_path": "", 2023-06-10T21:20:36.594 INFO:teuthology.orchestra.run.smithi064.stdout: "target_max_misplaced_ratio": "0.050000", 2023-06-10T21:20:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "thp": "false", 2023-06-10T21:20:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_default_timeout": "60", 2023-06-10T21:20:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "threadpool_empty_queue_max_wait": "2", 2023-06-10T21:20:36.595 INFO:teuthology.orchestra.run.smithi064.stdout: "throttler_perf_counter": "true" 2023-06-10T21:20:36.595 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:20:37.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-06-10T21:20:37.396 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:37.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:37 smithi064 ceph-mon[186706]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:37.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:37 smithi064 ceph-mon[180297]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:37 smithi152 ceph-mon[156819]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:38.893 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2023-06-10T21:20:38.903 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2023-06-10T21:20:38.913 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2023-06-10T21:20:38.914 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-10T21:20:38.914 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-10T21:20:38.914 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-10T21:20:39.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:39 smithi064 ceph-mon[180297]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1826347423' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:20:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:39 smithi064 ceph-mon[186706]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1826347423' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:20:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:39 smithi152 ceph-mon[156819]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1826347423' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:20:39.705 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-06-10T21:20:39.705 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-06-10T21:20:39.705 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:20:39.706 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:20:39.706 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-06-10T21:20:39.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_4 16 2023-06-10T21:20:39.707 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-06-10T21:20:39.708 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:20:39.708 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:20:39.708 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-06-10T21:20:39.708 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:39.709 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-06-10T21:20:39.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:20:39.709 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-06-10T21:20:39.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:20:39.710 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-06-10T21:20:39.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:20:39.768 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:39.849 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:39.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:39.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:40.002 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:40.051 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:40.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:40.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:40.185 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:40.233 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:40.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:40.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:20:40.369 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:40.412 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:40.413 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:40.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:20:40.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:40.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:20:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:20:40.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:20:40.531 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:40.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:20:40.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:20:40.555 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:40.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:20:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:40.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:20:40.666 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:40.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:40.789 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:40.833 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:40.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:40.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:40.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:40.966 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:41.010 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:41.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:41.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:41.141 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:41.187 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:41.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:41.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:20:41.323 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:41.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:41 smithi064 ceph-mon[180297]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:41.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:41 smithi064 ceph-mon[186706]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:41 smithi152 ceph-mon[156819]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:42.325 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:20:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:42.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:20:42.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:42.460 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:42] "GET /metrics HTTP/1.1" 200 220622 "" "Prometheus/2.33.4" 2023-06-10T21:20:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[186706]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[180297]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:43 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:43 smithi152 ceph-mon[156819]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:43 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:43 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:43.713 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:43.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:43.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:43.848 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:43.894 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:43.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:44.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:20:44.033 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:44.256 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' created 2023-06-10T21:20:44.284 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-10T21:20:44.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-10T21:20:44.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:20:44.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:44.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:20:44.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:44.448 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-10T21:20:44.955 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:20:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:20:44.956 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:20:44.956 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:20:45.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:45.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:45.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-06-10T21:20:45.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:45.498 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:45.547 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:45.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:45.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-10T21:20:45.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: osdmap e127: 8 total, 8 up, 8 in 2023-06-10T21:20:45.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:45 smithi152 ceph-mon[156819]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:45.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:45.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:45.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: osdmap e127: 8 total, 8 up, 8 in 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[180297]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/740144735' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-10T21:20:45.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: osdmap e127: 8 total, 8 up, 8 in 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:45 smithi064 ceph-mon[186706]: failed to encode map e127 with expected crc 2023-06-10T21:20:45.682 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:45.727 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:45.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:45.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:45.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:45.869 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:46.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:20:46.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:20:46.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:20:46.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:46.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:46.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:46.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:20:46.182 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:20:46.201 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:20:46.220 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:20:46.239 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:20:46.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:20:46.264 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.264 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.264 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:20:46.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:20:46.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: osdmap e128: 8 total, 8 up, 8 in 2023-06-10T21:20:46.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4292809675' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:46.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: osdmap e128: 8 total, 8 up, 8 in 2023-06-10T21:20:46.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4292809675' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:46.323 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: osdmap e128: 8 total, 8 up, 8 in 2023-06-10T21:20:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: failed to encode map e128 with expected crc 2023-06-10T21:20:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4292809675' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:46.932 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:47.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:20:47.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:47.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:20:47.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:47.141 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:47.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:47.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:47.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:47.283 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:20:47.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:20:47.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:47.495 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: pgmap v510: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:47.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: osdmap e129: 8 total, 8 up, 8 in 2023-06-10T21:20:47.497 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3463149450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.497 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: pgmap v510: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:47.497 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: osdmap e129: 8 total, 8 up, 8 in 2023-06-10T21:20:47.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3463149450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.499 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:47 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: pgmap v510: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: failed to encode map e129 with expected crc 2023-06-10T21:20:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: osdmap e129: 8 total, 8 up, 8 in 2023-06-10T21:20:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3463149450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:47 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:20:47.799 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:20:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:47.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:20:47.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:47.937 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:47.984 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:47.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:48.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:48.120 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:48.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:48.164 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:48.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:48.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:48.299 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:48.342 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:48.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:48.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:48.482 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:48.523 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:48.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:48.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:20:48.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:20:48.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: osdmap e130: 8 total, 8 up, 8 in 2023-06-10T21:20:48.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: pgmap v513: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:48.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:48.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:48.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: osdmap e130: 8 total, 8 up, 8 in 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: pgmap v513: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:48.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:48.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:48.659 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:20:48.659 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:20:48.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: osdmap e130: 8 total, 8 up, 8 in 2023-06-10T21:20:48.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: pgmap v513: 89 pgs: 4 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:48.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: failed to encode map e130 with expected crc 2023-06-10T21:20:48.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:48.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:20:48.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:48.702 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:48.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:48.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:48.809 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:48.836 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:48.879 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:48.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:48.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:48.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:20:48.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:20:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:49.058 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:20:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:49.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:20:49.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:20:49.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:49 smithi064 ceph-mon[186706]: failed to encode map e130 with expected crc 2023-06-10T21:20:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:49 smithi064 ceph-mon[180297]: failed to encode map e130 with expected crc 2023-06-10T21:20:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:49 smithi152 ceph-mon[156819]: failed to encode map e130 with expected crc 2023-06-10T21:20:50.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:20:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:50.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:20:50.210 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:50.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:50.307 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-06-10T21:20:50.311 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:50.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[186706]: pgmap v514: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/924845223' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:50.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[180297]: pgmap v514: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/924845223' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:50.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:50 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:50.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:20:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:50 smithi152 ceph-mon[156819]: pgmap v514: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/924845223' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:50 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:20:51.306 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:51.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:51.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:51.444 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:51.517 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:20:51.517 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:20:51.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[186706]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[186706]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:20:51.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[186706]: osdmap e131: 8 total, 8 up, 8 in 2023-06-10T21:20:51.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[180297]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[180297]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:20:51.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:51 smithi064 ceph-mon[180297]: osdmap e131: 8 total, 8 up, 8 in 2023-06-10T21:20:51.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:51.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:51.655 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:51.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:51 smithi152 ceph-mon[156819]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.686 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:51 smithi152 ceph-mon[156819]: failed to encode map e131 with expected crc 2023-06-10T21:20:51.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:20:51.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:51 smithi152 ceph-mon[156819]: osdmap e131: 8 total, 8 up, 8 in 2023-06-10T21:20:51.689 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:51.737 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:51.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:51.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:51.873 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:51.916 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:51.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:51.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:52.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:20:52.056 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:52.360 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-06-10T21:20:52.361 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-06-10T21:20:52.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:20:52.364 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:20:52.365 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:20:52.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[186706]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[186706]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[186706]: pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[180297]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[180297]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[180297]: pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:52.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.521 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:20:52.522 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:20:52.522 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_247471 2023-06-10T21:20:52.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:20:52.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:20:52.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:20:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:52 smithi152 ceph-mon[156819]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:52 smithi152 ceph-mon[156819]: failed to encode map e131 with expected crc 2023-06-10T21:20:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:52 smithi152 ceph-mon[156819]: pgmap v516: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-10T21:20:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:52.723 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:20:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:52.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:20:52.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:20:52.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:20:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:20:52] "GET /metrics HTTP/1.1" 200 222967 "" "Prometheus/2.33.4" 2023-06-10T21:20:53.183 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:53.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:20:53.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:53.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:20:53.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:53.411 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:53.475 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:53.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:53.522 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2794 2778 173.616 173.625 0.00364924 0.00574328 2023-06-10T21:20:53.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:53.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:20:53.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:20:53.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:53.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:20:53.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:20:53.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:53.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:20:53.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:20:53.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:20:53.623 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:53.703 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:53.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:53.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:53.718 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:53.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:53.862 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:53.927 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:53.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:54.034 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:54.069 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:54.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:20:54.147 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:54.151 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:20:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:54.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:54.192 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:20:54.240 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:20:54.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:20:54.280 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:20:54.317 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:54.318 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:20:54.357 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:20:54.395 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:20:54.400 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.400 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.401 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:20:54.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:54 smithi064 ceph-mon[180297]: pgmap v517: 89 pgs: 89 active+clean; 5.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 699 KiB/s wr, 10 op/s 2023-06-10T21:20:54.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2690849514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:54.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:54 smithi064 ceph-mon[186706]: pgmap v517: 89 pgs: 89 active+clean; 5.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 699 KiB/s wr, 10 op/s 2023-06-10T21:20:54.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2690849514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:54.522 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5893 5877 183.643 193.688 0.00219051 0.00543755 2023-06-10T21:20:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:54 smithi152 ceph-mon[156819]: pgmap v517: 89 pgs: 89 active+clean; 5.4 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 699 KiB/s wr, 10 op/s 2023-06-10T21:20:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2690849514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:20:55.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8776 8760 182.468 180.188 0.00498594 0.00547246 2023-06-10T21:20:56.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 12066 12050 188.252 205.625 0.00463915 0.00530604 2023-06-10T21:20:56.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:56 smithi064 ceph-mon[186706]: pgmap v518: 89 pgs: 89 active+clean; 155 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s wr, 303 op/s 2023-06-10T21:20:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:56 smithi064 ceph-mon[180297]: pgmap v518: 89 pgs: 89 active+clean; 155 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s wr, 303 op/s 2023-06-10T21:20:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:56 smithi152 ceph-mon[156819]: pgmap v518: 89 pgs: 89 active+clean; 155 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s wr, 303 op/s 2023-06-10T21:20:57.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:20:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:57.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:20:57.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:57.197 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:57.283 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:57.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:57.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:57.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:57.435 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:57.510 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:57.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:57.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15030 15014 187.648 185.25 0.00692801 0.00532447 2023-06-10T21:20:57.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:20:57.657 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:57.728 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:20:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:57.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:20:57.802 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:20:57.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:20:57.875 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:57.954 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:20:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:57.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:20:58.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:58.104 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:58.192 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:58.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:58.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:58.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:20:58.363 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:58.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17371 17355 180.755 146.312 0.00342139 0.00545615 2023-06-10T21:20:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:20:58 smithi064 ceph-mon[186706]: pgmap v519: 89 pgs: 89 active+clean; 414 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-06-10T21:20:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:20:58 smithi064 ceph-mon[180297]: pgmap v519: 89 pgs: 89 active+clean; 414 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-06-10T21:20:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:20:58 smithi152 ceph-mon[156819]: pgmap v519: 89 pgs: 89 active+clean; 414 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 41 MiB/s wr, 661 op/s 2023-06-10T21:20:59.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:20:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:59.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:20:59.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:59.228 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:59.306 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:59.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:59.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:20:59.454 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:59.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 19102 19086 170.387 108.188 0.00723813 0.00586273 2023-06-10T21:20:59.525 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:20:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:59.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:20:59.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:20:59.667 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:20:59.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:20:59.736 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:20:59.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:20:59.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:20:59.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:20:59.931 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:00.022 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:00.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:00.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:00.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:00.518 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:21:00.523 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 20689 20673 161.487 99.1875 0.00956206 0.00618515 2023-06-10T21:21:00.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[186706]: pgmap v520: 89 pgs: 89 active+clean; 752 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:00.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/267255361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:00.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:00.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[180297]: pgmap v520: 89 pgs: 89 active+clean; 752 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:00.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/267255361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:00.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:00 smithi152 ceph-mon[156819]: pgmap v520: 89 pgs: 89 active+clean; 752 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/267255361' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:01.277 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:01.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:01.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:01.424 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:01.524 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 22703 22687 157.529 125.875 0.010848 0.00634196 2023-06-10T21:21:01.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[180297]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[180297]: osdmap e132: 8 total, 8 up, 8 in 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[180297]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:01.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[186706]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[186706]: osdmap e132: 8 total, 8 up, 8 in 2023-06-10T21:21:01.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:01 smithi064 ceph-mon[186706]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.793 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:01.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:01.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:01.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:01 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:01.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:01 smithi152 ceph-mon[156819]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.933 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:01.934 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:01 smithi152 ceph-mon[156819]: osdmap e132: 8 total, 8 up, 8 in 2023-06-10T21:21:01.934 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:01 smithi152 ceph-mon[156819]: failed to encode map e132 with expected crc 2023-06-10T21:21:01.937 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:02.007 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:02.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:02.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:02.201 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:02.305 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:02.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:02.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:02.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:02.435 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:02.439 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:21:02.448 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:02.450 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:21:02.517 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:02.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:02.524 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 24710 24694 154.298 125.438 0.0117212 0.00647311 2023-06-10T21:21:02.601 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:21:02.601 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:21:02.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:21:02.601 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:130458 7.6 7:6b3e2d15:::benchmark_data_smithi064_196031_object14862:head [delete] snapc 0=[] ondisk+write+known_if_redirected e89)", 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:11:08.333406+0000", 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 594.26159278099999, 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.026406098, 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.602 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 130458 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333406+0000", 2023-06-10T21:21:02.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333406+0000", 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333404+0000", 2023-06-10T21:21:02.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333415+0000", 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0684e-05 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333433+0000", 2023-06-10T21:21:02.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8374999999999999e-05 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333443+0000", 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2909999999999996e-06 2023-06-10T21:21:02.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333505+0000", 2023-06-10T21:21:02.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2223000000000004e-05 2023-06-10T21:21:02.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.333518+0000", 2023-06-10T21:21:02.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2738000000000001e-05 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.347067+0000", 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013549923 2023-06-10T21:21:02.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.347194+0000", 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012629000000000001 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.347269+0000", 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5458999999999997e-05 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.353504+0000", 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062351029999999997 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.355507+0000", 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020027959999999998 2023-06-10T21:21:02.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359728+0000", 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042213559999999999 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359776+0000", 2023-06-10T21:21:02.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7784000000000002e-05 2023-06-10T21:21:02.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359812+0000", 2023-06-10T21:21:02.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5920000000000002e-05 2023-06-10T21:21:02.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.618 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:130478 7.6 7:6f32854f:::benchmark_data_smithi064_196031_object14882:head [delete] snapc 0=[] ondisk+write+known_if_redirected e89)", 2023-06-10T21:21:02.618 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:11:08.356955+0000", 2023-06-10T21:21:02.619 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 594.23804430899997, 2023-06-10T21:21:02.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.007631025, 2023-06-10T21:21:02.619 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.619 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.621 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 130478 2023-06-10T21:21:02.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.621 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356955+0000", 2023-06-10T21:21:02.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356955+0000", 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356953+0000", 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356962+0000", 2023-06-10T21:21:02.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4769999999999992e-06 2023-06-10T21:21:02.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356980+0000", 2023-06-10T21:21:02.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8012999999999998e-05 2023-06-10T21:21:02.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.356992+0000", 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1745e-05 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.357053+0000", 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1445000000000005e-05 2023-06-10T21:21:02.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.357067+0000", 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3706000000000001e-05 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.349647+0000", 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99257963999999999 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.349728+0000", 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1200999999999997e-05 2023-06-10T21:21:02.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.349794+0000", 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5891999999999999e-05 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.355542+0000", 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057478649999999996 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.355563+0000", 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1138999999999999e-05 2023-06-10T21:21:02.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.364522+0000", 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089586100000000005 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.364555+0000", 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3912000000000003e-05 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.364586+0000", 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0122999999999999e-05 2023-06-10T21:21:02.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:130487 7.6 7:60eee0d7:::benchmark_data_smithi064_196031_object14891:head [delete] snapc 0=[] ondisk+write+known_if_redirected e89)", 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:11:08.369553+0000", 2023-06-10T21:21:02.634 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 594.22544629499998, 2023-06-10T21:21:02.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0006163539999999, 2023-06-10T21:21:02.635 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.635 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.635 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 130487 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369553+0000", 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369553+0000", 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369550+0000", 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369563+0000", 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2584e-05 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369589+0000", 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5609000000000002e-05 2023-06-10T21:21:02.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369603+0000", 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4290999999999999e-05 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369676+0000", 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2713999999999995e-05 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.369689+0000", 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3538e-05 2023-06-10T21:21:02.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.351663+0000", 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98197377799999996 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.351736+0000", 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2720999999999997e-05 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.351799+0000", 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2935999999999995e-05 2023-06-10T21:21:02.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.357563+0000", 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057642819999999999 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359653+0000", 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020902450000000001 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370108+0000", 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010454952 2023-06-10T21:21:02.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370140+0000", 2023-06-10T21:21:02.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2051000000000002e-05 2023-06-10T21:21:02.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370169+0000", 2023-06-10T21:21:02.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8861e-05 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:130502 7.6 7:6576f59d:::benchmark_data_smithi064_196031_object14906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e89)", 2023-06-10T21:21:02.648 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:11:08.383915+0000", 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 594.21108346000005, 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98635732899999995, 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.649 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 130502 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383915+0000", 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383915+0000", 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383914+0000", 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383921+0000", 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3930000000000002e-06 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383934+0000", 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3057000000000001e-05 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.383940+0000", 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9000000000000003e-06 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.384000+0000", 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9721e-05 2023-06-10T21:21:02.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:08.384010+0000", 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0618000000000001e-05 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.353543+0000", 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96953259000000003 2023-06-10T21:21:02.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.353616+0000", 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2509999999999995e-05 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:21:02.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.353692+0000", 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6372000000000007e-05 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359660+0000", 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059685839999999999 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.359686+0000", 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5367999999999998e-05 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370208+0000", 2023-06-10T21:21:02.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010521697 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370234+0000", 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6253000000000001e-05 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:11:09.370273+0000", 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8918000000000002e-05 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:178062 7.4 7:28d2d7f1:::benchmark_data_smithi064_196031_object62466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e91)", 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:12:16.469510+0000", 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 526.12548894999998, 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.025196725, 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.662 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 178062 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469510+0000", 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469510+0000", 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469509+0000", 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469513+0000", 2023-06-10T21:21:02.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3210000000000001e-06 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469520+0000", 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4919999999999996e-06 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469527+0000", 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3779999999999998e-06 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469589+0000", 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2324999999999997e-05 2023-06-10T21:21:02.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.469599+0000", 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0576e-05 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.483069+0000", 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0134693320000001 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.483212+0000", 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000143108 2023-06-10T21:21:02.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.483354+0000", 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014214100000000001 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.491847+0000", 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084934880000000004 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494559+0000", 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002711087 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494616+0000", 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7701999999999997e-05 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494668+0000", 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1839999999999998e-05 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494707+0000", 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8516999999999997e-05 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.673 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:178074 7.4 7:24869b1b:::benchmark_data_smithi064_196031_object62478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e91)", 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:12:16.490090+0000", 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 526.10490861300002, 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.004854932, 2023-06-10T21:21:02.674 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 178074 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490090+0000", 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490090+0000", 2023-06-10T21:21:02.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490088+0000", 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490096+0000", 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2189999999999998e-06 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490114+0000", 2023-06-10T21:21:02.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8230999999999999e-05 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490124+0000", 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8199999999999992e-06 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490191+0000", 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7038000000000004e-05 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.490206+0000", 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5099999999999999e-05 2023-06-10T21:21:02.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.486363+0000", 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99615748100000001 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.486471+0000", 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010801799999999999 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.486544+0000", 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2257999999999998e-05 2023-06-10T21:21:02.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494501+0000", 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079570739999999997 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494773+0000", 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027211100000000001 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494877+0000", 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103867 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494905+0000", 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8062999999999999e-05 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494945+0000", 2023-06-10T21:21:02.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0435999999999997e-05 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.34251.0:178080 7.4 7:23714054:::benchmark_data_smithi064_196031_object62484:head [delete] snapc 0=[] ondisk+write+known_if_redirected e91)", 2023-06-10T21:21:02.686 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:12:16.503090+0000", 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 526.09190935599997, 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99797817300000002, 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.34251", 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/3335933734", 2023-06-10T21:21:02.687 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 178080 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503090+0000", 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503090+0000", 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503088+0000", 2023-06-10T21:21:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503096+0000", 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3899999999999993e-06 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503115+0000", 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9128999999999999e-05 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503125+0000", 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3049999999999998e-06 2023-06-10T21:21:02.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503192+0000", 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7373999999999996e-05 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:16.503207+0000", 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5356999999999999e-05 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.488817+0000", 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98560984600000001 2023-06-10T21:21:02.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.488951+0000", 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133523 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:21:02.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.489080+0000", 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012890300000000001 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494721+0000", 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005641386 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.494828+0000", 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010713399999999999 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.500981+0000", 2023-06-10T21:21:02.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061527459999999997 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.501027+0000", 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5819999999999998e-05 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:12:17.501068+0000", 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1007999999999998e-05 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.698 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:41969 8.1 8:874800cc:::benchmark_data_smithi064_208298_object41968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e102)", 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:18.433412+0000", 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.16158669800001, 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0107505269999999, 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.699 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41969 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433412+0000", 2023-06-10T21:21:02.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433412+0000", 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433410+0000", 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433444+0000", 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3435000000000001e-05 2023-06-10T21:21:02.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433456+0000", 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2676e-05 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433467+0000", 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0504e-05 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437557+0000", 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040901619999999996 2023-06-10T21:21:02.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437568+0000", 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0638999999999999e-05 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.443347+0000", 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005778949 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444163+0000", 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081597699999999996 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.706 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:41979 8.1 8:8642025e:::benchmark_data_smithi064_208298_object41978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e102)", 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:18.433475+0000", 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.16152384200001, 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0110103589999999, 2023-06-10T21:21:02.707 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41979 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.708 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433475+0000", 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433475+0000", 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433474+0000", 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433524+0000", 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9669999999999997e-05 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433532+0000", 2023-06-10T21:21:02.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3790000000000006e-06 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433539+0000", 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.229e-06 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437586+0000", 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040473740000000003 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437590+0000", 2023-06-10T21:21:02.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6799999999999999e-06 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444423+0000", 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0068329250000001 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444485+0000", 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2685e-05 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.715 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:41988 8.1 8:8cbe3046:::benchmark_data_smithi064_208298_object41987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e102)", 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:18.433546+0000", 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.16145325700001, 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.018683191, 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41988 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433546+0000", 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433546+0000", 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433545+0000", 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433591+0000", 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5858e-05 2023-06-10T21:21:02.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433600+0000", 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1390000000000001e-06 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.433608+0000", 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.943e-06 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437601+0000", 2023-06-10T21:21:02.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039931150000000002 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.437604+0000", 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1949999999999999e-06 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444515+0000", 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0069106649999999 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444765+0000", 2023-06-10T21:21:02.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024962200000000002 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.444966+0000", 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020170299999999999 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.449941+0000", 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049749770000000002 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.452055+0000", 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021140830000000001 2023-06-10T21:21:02.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.452115+0000", 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9691000000000002e-05 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.452229+0000", 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113688 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:41992 8.1 8:82fafa7f:::benchmark_data_smithi064_208298_object41991:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-06-10T21:21:02.727 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:18.441488+0000", 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.15351054000001, 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013645664, 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.728 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41992 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441488+0000", 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441488+0000", 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441487+0000", 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441527+0000", 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0040000000000003e-05 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441539+0000", 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2075999999999999e-05 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441546+0000", 2023-06-10T21:21:02.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3539999999999996e-06 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441584+0000", 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8210999999999997e-05 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.441593+0000", 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0769999999999997e-06 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.446574+0000", 2023-06-10T21:21:02.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0049813620000001 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.446715+0000", 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014082799999999999 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.446879+0000", 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000163852 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.449947+0000", 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030681300000000001 2023-06-10T21:21:02.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.454982+0000", 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050349610000000001 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.455044+0000", 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1593000000000005e-05 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.455134+0000", 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0345000000000003e-05 2023-06-10T21:21:02.738 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:41995 8.1 8:81248d95:::benchmark_data_smithi064_208298_object41994:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e102)", 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:18.447226+0000", 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 404.14777286200001, 2023-06-10T21:21:02.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0082461220000001, 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 41995 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.740 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447226+0000", 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447226+0000", 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447225+0000", 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:21:02.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447244+0000", 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8953e-05 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447254+0000", 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0376e-05 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447261+0000", 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1500000000000004e-06 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447309+0000", 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8288000000000003e-05 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:18.447321+0000", 2023-06-10T21:21:02.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1703e-05 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.448335+0000", 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0010143330000001 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.448500+0000", 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164721 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2", 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.448680+0000", 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000180144 2023-06-10T21:21:02.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.450256+0000", 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015762009999999999 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.455368+0000", 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051117539999999996 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.455415+0000", 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7496000000000001e-05 2023-06-10T21:21:02.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:19.455472+0000", 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6884e-05 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:75199 8.1 8:846c7e26:::benchmark_data_smithi064_208298_object75198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e106)", 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:42.422687+0000", 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 380.17231172800001, 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1275096520000001, 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:21:02.751 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 75199 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422687+0000", 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422687+0000", 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422685+0000", 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422721+0000", 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6171000000000003e-05 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422742+0000", 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0281000000000001e-05 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422755+0000", 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3089e-05 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422966+0000", 2023-06-10T21:21:02.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021104000000000001 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422980+0000", 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4729000000000001e-05 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.550080+0000", 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1270996769999999 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.550197+0000", 2023-06-10T21:21:02.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116661 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:75242 8.1 8:8535fe46:::benchmark_data_smithi064_208298_object75241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e106)", 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:14:42.422769+0000", 2023-06-10T21:21:02.759 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 380.17223016200001, 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.143851199, 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 75242 2023-06-10T21:21:02.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422769+0000", 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422769+0000", 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422768+0000", 2023-06-10T21:21:02.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422796+0000", 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8364000000000002e-05 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422805+0000", 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1900000000000001e-06 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.422812+0000", 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:21:02.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.423008+0000", 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000195649 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:21:02.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:42.423017+0000", 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2259999999999994e-06 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.550566+0000", 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1275483129999999 2023-06-10T21:21:02.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.550738+0000", 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017221900000000001 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:21:02.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.550960+0000", 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022246500000000001 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.556938+0000", 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059782079999999996 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.557521+0000", 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000582816 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.565014+0000", 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074928529999999998 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.565084+0000", 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9879e-05 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:14:43.566620+0000", 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015358640000000001 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:17349 9.9 9:9bf5ec16:::benchmark_data_smithi064_247471_object17348:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:20:58.438393+0000", 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.1566060470000004, 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30251292600000002, 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.772 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 17349 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438393+0000", 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438393+0000", 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438391+0000", 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438418+0000", 2023-06-10T21:21:02.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6449999999999999e-05 2023-06-10T21:21:02.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438429+0000", 2023-06-10T21:21:02.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1637000000000001e-05 2023-06-10T21:21:02.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438438+0000", 2023-06-10T21:21:02.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5499999999999995e-06 2023-06-10T21:21:02.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438479+0000", 2023-06-10T21:21:02.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1526000000000002e-05 2023-06-10T21:21:02.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438597+0000", 2023-06-10T21:21:02.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0001172 2023-06-10T21:21:02.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:21:02.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.438773+0000", 2023-06-10T21:21:02.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000176715 2023-06-10T21:21:02.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440118+0000", 2023-06-10T21:21:02.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013448080000000001 2023-06-10T21:21:02.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440208+0000", 2023-06-10T21:21:02.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9641000000000005e-05 2023-06-10T21:21:02.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740804+0000", 2023-06-10T21:21:02.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30059585500000002 2023-06-10T21:21:02.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740861+0000", 2023-06-10T21:21:02.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7444000000000001e-05 2023-06-10T21:21:02.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740906+0000", 2023-06-10T21:21:02.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4586999999999998e-05 2023-06-10T21:21:02.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.790 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.791 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35400.0:17349 9.9 e131/127 ondisk, result = 0)", 2023-06-10T21:21:02.791 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:20:58.440465+0000", 2023-06-10T21:21:02.792 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.1545338340000004, 2023-06-10T21:21:02.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30044807099999998, 2023-06-10T21:21:02.792 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.792 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:21:02.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440465+0000", 2023-06-10T21:21:02.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440465+0000", 2023-06-10T21:21:02.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440464+0000", 2023-06-10T21:21:02.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:21:02.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440470+0000", 2023-06-10T21:21:02.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9440000000000002e-06 2023-06-10T21:21:02.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440481+0000", 2023-06-10T21:21:02.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0685e-05 2023-06-10T21:21:02.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740731+0000", 2023-06-10T21:21:02.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30024988800000002 2023-06-10T21:21:02.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740782+0000", 2023-06-10T21:21:02.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1632999999999998e-05 2023-06-10T21:21:02.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740796+0000", 2023-06-10T21:21:02.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4018e-05 2023-06-10T21:21:02.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740913+0000", 2023-06-10T21:21:02.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116665 2023-06-10T21:21:02.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.805 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.806 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:17355 9.8 e131/127 9:18310c43:::benchmark_data_smithi064_247471_object17354:head v 131'1143, mlcod=131'1141)", 2023-06-10T21:21:02.806 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:20:58.440507+0000", 2023-06-10T21:21:02.807 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.1544916609999998, 2023-06-10T21:21:02.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.301587671, 2023-06-10T21:21:02.807 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.807 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.808 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440507+0000", 2023-06-10T21:21:02.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440507+0000", 2023-06-10T21:21:02.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440507+0000", 2023-06-10T21:21:02.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:21:02.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.440532+0000", 2023-06-10T21:21:02.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5700000000000001e-05 2023-06-10T21:21:02.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740725+0000", 2023-06-10T21:21:02.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30019291199999998 2023-06-10T21:21:02.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740740+0000", 2023-06-10T21:21:02.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4878000000000001e-05 2023-06-10T21:21:02.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740812+0000", 2023-06-10T21:21:02.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2268999999999995e-05 2023-06-10T21:21:02.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.740834+0000", 2023-06-10T21:21:02.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2180000000000001e-05 2023-06-10T21:21:02.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.742034+0000", 2023-06-10T21:21:02.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011993100000000001 2023-06-10T21:21:02.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:20:58.742095+0000", 2023-06-10T21:21:02.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1073000000000003e-05 2023-06-10T21:21:02.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.828 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.829 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:22622 9.9 9:9b238eae:::benchmark_data_smithi064_247471_object22621:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-10T21:21:02.830 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:21:01.407595+0000", 2023-06-10T21:21:02.830 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.18740435, 2023-06-10T21:21:02.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.073160205000000006, 2023-06-10T21:21:02.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.831 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.831 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:21:02.832 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:21:02.832 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 22622 2023-06-10T21:21:02.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.832 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407595+0000", 2023-06-10T21:21:02.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407595+0000", 2023-06-10T21:21:02.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407593+0000", 2023-06-10T21:21:02.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:21:02.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407616+0000", 2023-06-10T21:21:02.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3011999999999999e-05 2023-06-10T21:21:02.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407629+0000", 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2921e-05 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.407637+0000", 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6159999999999996e-06 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408245+0000", 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060840099999999995 2023-06-10T21:21:02.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408396+0000", 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150486 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:21:02.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408532+0000", 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013616600000000001 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480606+0000", 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.072074685999999999 2023-06-10T21:21:02.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480645+0000", 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8967999999999997e-05 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480683+0000", 2023-06-10T21:21:02.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7558999999999997e-05 2023-06-10T21:21:02.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:02.845 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:02 smithi152 ceph-mon[156819]: pgmap v522: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480725+0000", 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1637999999999997e-05 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480755+0000", 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0202999999999999e-05 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.847 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:22627 9.5 9:ab1e840a:::benchmark_data_smithi064_247471_object22626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:21:01.408994+0000", 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.1860053479999999, 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071792455000000005, 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:21:02.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 22627 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408994+0000", 2023-06-10T21:21:02.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408994+0000", 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.408992+0000", 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409017+0000", 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4921000000000001e-05 2023-06-10T21:21:02.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409028+0000", 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1250000000000001e-05 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409037+0000", 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1669999999999999e-06 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409088+0000", 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1072000000000002e-05 2023-06-10T21:21:02.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409200+0000", 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112689 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.409335+0000", 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013439800000000001 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480597+0000", 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071262052000000006 2023-06-10T21:21:02.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480699+0000", 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102384 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:21:02.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480727+0000", 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8076999999999999e-05 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480763+0000", 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5482999999999999e-05 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480786+0000", 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3192000000000001e-05 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:22621 9.8 e132/127 9:111e92f4:::benchmark_data_smithi064_247471_object22620:head v 132'1453, mlcod=132'1449)", 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:21:01.410178+0000", 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.184821339, 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.071794627999999999, 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:21:02.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410178+0000", 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410178+0000", 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410177+0000", 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:21:02.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410197+0000", 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9352e-05 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:21:02.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410204+0000", 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4680000000000003e-06 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.410209+0000", 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2959999999999998e-06 2023-06-10T21:21:02.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480633+0000", 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.070423608999999998 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:21:02.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.480661+0000", 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.809e-05 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.481946+0000", 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001284859 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:21:02.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:21:01.481972+0000", 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6279999999999999e-05 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:21:02.866 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:21:02.867 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:21:02.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:02 smithi064 ceph-mon[186706]: pgmap v522: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-10T21:21:02.886 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:02 smithi064 ceph-mon[180297]: pgmap v522: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-10T21:21:02.894 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:02.979 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:02.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:03.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:03.133 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:03.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:02] "GET /metrics HTTP/1.1" 200 222967 "" "Prometheus/2.33.4" 2023-06-10T21:21:03.224 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:03.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:03.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:03.407 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:03.525 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 26905 26889 152.741 137.188 0.00698445 0.00654162 2023-06-10T21:21:03.526 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:03 smithi152 ceph-mon[156819]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:03.526 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:03.526 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[180297]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[186706]: from='client.45410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:03.931 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:21:04.525 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 28089 28073 146.18 74 0.00484791 0.00683361 2023-06-10T21:21:04.567 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":522,"stamp":"2023-06-10T21:21:01.416320+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1098913678,"num_objects":16783,"num_object_clones":0,"num_object_copies":50349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16783,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":17020,"num_write_kb":1075022,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16954,"ondisk_log_size":16954,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5315220,"kb_used_data":3210836,"kb_used_omap":20,"kb_used_meta":2104299,"kb_avail":744645996,"statfs":{"total":767960285184,"available":762517499904,"internally_reserved":0,"allocated":3287896064,"data_stored":3283871527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2154802830},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,1,1,0,0,0,0,0,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1098317824,"num_objects":16759,"num_object_clones":0,"num_object_copies":50277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16759,"num_write_kb":1072576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002923"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":114,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703683+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:20:47.703683+0000","last_peered":"2023-06-10T21:20:47.703683+0000","last_clean":"2023-06-10T21:20:47.703683+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:20:47.703683+0000","last_undegraded":"2023-06-10T21:20:47.703683+0000","last_fullsized":"2023-06-10T21:20:47.703683+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277747+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:20:47.277747+0000","last_peered":"2023-06-10T21:20:47.277747+0000","last_clean":"2023-06-10T21:20:47.277747+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:20:47.277747+0000","last_undegraded":"2023-06-10T21:20:47.277747+0000","last_fullsized":"2023-06-10T21:20:47.277747+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.740462+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:20:50.740462+0000","last_peered":"2023-06-10T21:20:50.740462+0000","last_clean":"2023-06-10T21:20:50.740462+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:20:50.740462+0000","last_undegraded":"2023-06-10T21:20:50.740462+0000","last_fullsized":"2023-06-10T21:20:50.740462+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702085+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:20:47.702085+0000","last_peered":"2023-06-10T21:20:47.702085+0000","last_clean":"2023-06-10T21:20:47.702085+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:20:47.702085+0000","last_undegraded":"2023-06-10T21:20:47.702085+0000","last_fullsized":"2023-06-10T21:20:47.702085+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277060+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:20:47.277060+0000","last_peered":"2023-06-10T21:20:47.277060+0000","last_clean":"2023-06-10T21:20:47.277060+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:20:47.277060+0000","last_undegraded":"2023-06-10T21:20:47.277060+0000","last_fullsized":"2023-06-10T21:20:47.277060+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.706125+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:20:47.706125+0000","last_peered":"2023-06-10T21:20:47.706125+0000","last_clean":"2023-06-10T21:20:47.706125+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:20:47.706125+0000","last_undegraded":"2023-06-10T21:20:47.706125+0000","last_fullsized":"2023-06-10T21:20:47.706125+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702455+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:20:47.702455+0000","last_peered":"2023-06-10T21:20:47.702455+0000","last_clean":"2023-06-10T21:20:47.702455+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:20:47.702455+0000","last_undegraded":"2023-06-10T21:20:47.702455+0000","last_fullsized":"2023-06-10T21:20:47.702455+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277056+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:20:47.277056+0000","last_peered":"2023-06-10T21:20:47.277056+0000","last_clean":"2023-06-10T21:20:47.277056+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:20:47.277056+0000","last_undegraded":"2023-06-10T21:20:47.277056+0000","last_fullsized":"2023-06-10T21:20:47.277056+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704740+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:20:47.704740+0000","last_peered":"2023-06-10T21:20:47.704740+0000","last_clean":"2023-06-10T21:20:47.704740+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:20:47.704740+0000","last_undegraded":"2023-06-10T21:20:47.704740+0000","last_fullsized":"2023-06-10T21:20:47.704740+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277353+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:20:47.277353+0000","last_peered":"2023-06-10T21:20:47.277353+0000","last_clean":"2023-06-10T21:20:47.277353+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:20:47.277353+0000","last_undegraded":"2023-06-10T21:20:47.277353+0000","last_fullsized":"2023-06-10T21:20:47.277353+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":101,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704913+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:20:47.704913+0000","last_peered":"2023-06-10T21:20:47.704913+0000","last_clean":"2023-06-10T21:20:47.704913+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:20:47.704913+0000","last_undegraded":"2023-06-10T21:20:47.704913+0000","last_fullsized":"2023-06-10T21:20:47.704913+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.740669+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:20:50.740669+0000","last_peered":"2023-06-10T21:20:50.740669+0000","last_clean":"2023-06-10T21:20:50.740669+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:20:50.740669+0000","last_undegraded":"2023-06-10T21:20:50.740669+0000","last_fullsized":"2023-06-10T21:20:50.740669+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705018+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:20:47.705018+0000","last_peered":"2023-06-10T21:20:47.705018+0000","last_clean":"2023-06-10T21:20:47.705018+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:20:47.705018+0000","last_undegraded":"2023-06-10T21:20:47.705018+0000","last_fullsized":"2023-06-10T21:20:47.705018+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703732+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:20:47.703732+0000","last_peered":"2023-06-10T21:20:47.703732+0000","last_clean":"2023-06-10T21:20:47.703732+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:20:47.703732+0000","last_undegraded":"2023-06-10T21:20:47.703732+0000","last_fullsized":"2023-06-10T21:20:47.703732+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704361+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:20:47.704361+0000","last_peered":"2023-06-10T21:20:47.704361+0000","last_clean":"2023-06-10T21:20:47.704361+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:20:47.704361+0000","last_undegraded":"2023-06-10T21:20:47.704361+0000","last_fullsized":"2023-06-10T21:20:47.704361+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277333+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:20:47.277333+0000","last_peered":"2023-06-10T21:20:47.277333+0000","last_clean":"2023-06-10T21:20:47.277333+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:20:47.277333+0000","last_undegraded":"2023-06-10T21:20:47.277333+0000","last_fullsized":"2023-06-10T21:20:47.277333+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.701956+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:20:47.701956+0000","last_peered":"2023-06-10T21:20:47.701956+0000","last_clean":"2023-06-10T21:20:47.701956+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:20:47.701956+0000","last_undegraded":"2023-06-10T21:20:47.701956+0000","last_fullsized":"2023-06-10T21:20:47.701956+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704887+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:20:47.704887+0000","last_peered":"2023-06-10T21:20:47.704887+0000","last_clean":"2023-06-10T21:20:47.704887+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:20:47.704887+0000","last_undegraded":"2023-06-10T21:20:47.704887+0000","last_fullsized":"2023-06-10T21:20:47.704887+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":83,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704321+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:20:47.704321+0000","last_peered":"2023-06-10T21:20:47.704321+0000","last_clean":"2023-06-10T21:20:47.704321+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:20:47.704321+0000","last_undegraded":"2023-06-10T21:20:47.704321+0000","last_fullsized":"2023-06-10T21:20:47.704321+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702222+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:20:47.702222+0000","last_peered":"2023-06-10T21:20:47.702222+0000","last_clean":"2023-06-10T21:20:47.702222+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:20:47.702222+0000","last_undegraded":"2023-06-10T21:20:47.702222+0000","last_fullsized":"2023-06-10T21:20:47.702222+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705576+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:20:47.705576+0000","last_peered":"2023-06-10T21:20:47.705576+0000","last_clean":"2023-06-10T21:20:47.705576+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:20:47.705576+0000","last_undegraded":"2023-06-10T21:20:47.705576+0000","last_fullsized":"2023-06-10T21:20:47.705576+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277505+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:20:47.277505+0000","last_peered":"2023-06-10T21:20:47.277505+0000","last_clean":"2023-06-10T21:20:47.277505+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:20:47.277505+0000","last_undegraded":"2023-06-10T21:20:47.277505+0000","last_fullsized":"2023-06-10T21:20:47.277505+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"131'956","reported_seq":969,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:57.683428+0000","last_change":"2023-06-10T21:20:45.268089+0000","last_active":"2023-06-10T21:20:57.683428+0000","last_peered":"2023-06-10T21:20:57.683428+0000","last_clean":"2023-06-10T21:20:57.683428+0000","last_became_active":"2023-06-10T21:20:45.267772+0000","last_became_peered":"2023-06-10T21:20:45.267772+0000","last_unstale":"2023-06-10T21:20:57.683428+0000","last_undegraded":"2023-06-10T21:20:57.683428+0000","last_fullsized":"2023-06-10T21:20:57.683428+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":956,"log_dups_size":0,"ondisk_log_size":956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:24:07.609015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":956,"num_write_kb":61184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":99,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704494+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:20:47.704494+0000","last_peered":"2023-06-10T21:20:47.704494+0000","last_clean":"2023-06-10T21:20:47.704494+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:20:47.704494+0000","last_undegraded":"2023-06-10T21:20:47.704494+0000","last_fullsized":"2023-06-10T21:20:47.704494+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705200+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:20:47.705200+0000","last_peered":"2023-06-10T21:20:47.705200+0000","last_clean":"2023-06-10T21:20:47.705200+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:20:47.705200+0000","last_undegraded":"2023-06-10T21:20:47.705200+0000","last_fullsized":"2023-06-10T21:20:47.705200+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"132'1305","reported_seq":1319,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-06-10T21:21:00.689367+0000","last_change":"2023-06-10T21:20:45.293144+0000","last_active":"2023-06-10T21:21:00.689367+0000","last_peered":"2023-06-10T21:21:00.689367+0000","last_clean":"2023-06-10T21:21:00.689367+0000","last_became_active":"2023-06-10T21:20:45.292848+0000","last_became_peered":"2023-06-10T21:20:45.292848+0000","last_unstale":"2023-06-10T21:21:00.689367+0000","last_undegraded":"2023-06-10T21:21:00.689367+0000","last_fullsized":"2023-06-10T21:21:00.689367+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1305,"log_dups_size":0,"ondisk_log_size":1305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:53:38.966995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85524480,"num_objects":1305,"num_object_clones":0,"num_object_copies":3915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1305,"num_write_kb":83520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703936+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:20:47.703936+0000","last_peered":"2023-06-10T21:20:47.703936+0000","last_clean":"2023-06-10T21:20:47.703936+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:20:47.703936+0000","last_undegraded":"2023-06-10T21:20:47.703936+0000","last_fullsized":"2023-06-10T21:20:47.703936+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704496+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:20:47.704496+0000","last_peered":"2023-06-10T21:20:47.704496+0000","last_clean":"2023-06-10T21:20:47.704496+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:20:47.704496+0000","last_undegraded":"2023-06-10T21:20:47.704496+0000","last_fullsized":"2023-06-10T21:20:47.704496+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"131'1197","reported_seq":1209,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.449396+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:20:59.449396+0000","last_peered":"2023-06-10T21:20:59.449396+0000","last_clean":"2023-06-10T21:20:59.449396+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:20:59.449396+0000","last_undegraded":"2023-06-10T21:20:59.449396+0000","last_fullsized":"2023-06-10T21:20:59.449396+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1197,"log_dups_size":0,"ondisk_log_size":1197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78446592,"num_objects":1197,"num_object_clones":0,"num_object_copies":3591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1197,"num_write_kb":76608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":84,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704743+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:20:47.704743+0000","last_peered":"2023-06-10T21:20:47.704743+0000","last_clean":"2023-06-10T21:20:47.704743+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:20:47.704743+0000","last_undegraded":"2023-06-10T21:20:47.704743+0000","last_fullsized":"2023-06-10T21:20:47.704743+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705799+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:20:47.705799+0000","last_peered":"2023-06-10T21:20:47.705799+0000","last_clean":"2023-06-10T21:20:47.705799+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:20:47.705799+0000","last_undegraded":"2023-06-10T21:20:47.705799+0000","last_fullsized":"2023-06-10T21:20:47.705799+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"131'1174","reported_seq":1185,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.434276+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:20:59.434276+0000","last_peered":"2023-06-10T21:20:59.434276+0000","last_clean":"2023-06-10T21:20:59.434276+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:20:59.434276+0000","last_undegraded":"2023-06-10T21:20:59.434276+0000","last_fullsized":"2023-06-10T21:20:59.434276+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1174,"log_dups_size":0,"ondisk_log_size":1174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76939264,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1174,"num_write_kb":75136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704982+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:20:47.704982+0000","last_peered":"2023-06-10T21:20:47.704982+0000","last_clean":"2023-06-10T21:20:47.704982+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:20:47.704982+0000","last_undegraded":"2023-06-10T21:20:47.704982+0000","last_fullsized":"2023-06-10T21:20:47.704982+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702173+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:20:47.702173+0000","last_peered":"2023-06-10T21:20:47.702173+0000","last_clean":"2023-06-10T21:20:47.702173+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:20:47.702173+0000","last_undegraded":"2023-06-10T21:20:47.702173+0000","last_fullsized":"2023-06-10T21:20:47.702173+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"131'727","reported_seq":740,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:56.498138+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:20:56.498138+0000","last_peered":"2023-06-10T21:20:56.498138+0000","last_clean":"2023-06-10T21:20:56.498138+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:20:56.498138+0000","last_undegraded":"2023-06-10T21:20:56.498138+0000","last_fullsized":"2023-06-10T21:20:56.498138+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":727,"log_dups_size":0,"ondisk_log_size":727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":727,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277304+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:20:47.277304+0000","last_peered":"2023-06-10T21:20:47.277304+0000","last_clean":"2023-06-10T21:20:47.277304+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:20:47.277304+0000","last_undegraded":"2023-06-10T21:20:47.277304+0000","last_fullsized":"2023-06-10T21:20:47.277304+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704917+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:20:47.704917+0000","last_peered":"2023-06-10T21:20:47.704917+0000","last_clean":"2023-06-10T21:20:47.704917+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:20:47.704917+0000","last_undegraded":"2023-06-10T21:20:47.704917+0000","last_fullsized":"2023-06-10T21:20:47.704917+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"132'1269","reported_seq":1283,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-06-10T21:21:00.695723+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:21:00.695723+0000","last_peered":"2023-06-10T21:21:00.695723+0000","last_clean":"2023-06-10T21:21:00.695723+0000","last_became_active":"2023-06-10T21:20:45.291689+0000","last_became_peered":"2023-06-10T21:20:45.291689+0000","last_unstale":"2023-06-10T21:21:00.695723+0000","last_undegraded":"2023-06-10T21:21:00.695723+0000","last_fullsized":"2023-06-10T21:21:00.695723+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1269,"log_dups_size":0,"ondisk_log_size":1269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:11:53.302103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1269,"num_write_kb":81216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705727+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:20:47.705727+0000","last_peered":"2023-06-10T21:20:47.705727+0000","last_clean":"2023-06-10T21:20:47.705727+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:20:47.705727+0000","last_undegraded":"2023-06-10T21:20:47.705727+0000","last_fullsized":"2023-06-10T21:20:47.705727+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704524+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:20:47.704524+0000","last_peered":"2023-06-10T21:20:47.704524+0000","last_clean":"2023-06-10T21:20:47.704524+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:20:47.704524+0000","last_undegraded":"2023-06-10T21:20:47.704524+0000","last_fullsized":"2023-06-10T21:20:47.704524+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"131'1166","reported_seq":1178,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.227228+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:20:59.227228+0000","last_peered":"2023-06-10T21:20:59.227228+0000","last_clean":"2023-06-10T21:20:59.227228+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:20:59.227228+0000","last_undegraded":"2023-06-10T21:20:59.227228+0000","last_fullsized":"2023-06-10T21:20:59.227228+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1166,"log_dups_size":0,"ondisk_log_size":1166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76414976,"num_objects":1166,"num_object_clones":0,"num_object_copies":3498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1166,"num_write_kb":74624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704278+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:20:47.704278+0000","last_peered":"2023-06-10T21:20:47.704278+0000","last_clean":"2023-06-10T21:20:47.704278+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:20:47.704278+0000","last_undegraded":"2023-06-10T21:20:47.704278+0000","last_fullsized":"2023-06-10T21:20:47.704278+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704046+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:20:47.704046+0000","last_peered":"2023-06-10T21:20:47.704046+0000","last_clean":"2023-06-10T21:20:47.704046+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:20:47.704046+0000","last_undegraded":"2023-06-10T21:20:47.704046+0000","last_fullsized":"2023-06-10T21:20:47.704046+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"131'752","reported_seq":765,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:56.524818+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:20:56.524818+0000","last_peered":"2023-06-10T21:20:56.524818+0000","last_clean":"2023-06-10T21:20:56.524818+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:20:56.524818+0000","last_undegraded":"2023-06-10T21:20:56.524818+0000","last_fullsized":"2023-06-10T21:20:56.524818+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":752,"log_dups_size":0,"ondisk_log_size":752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":752,"num_write_kb":48128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277168+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:20:47.277168+0000","last_peered":"2023-06-10T21:20:47.277168+0000","last_clean":"2023-06-10T21:20:47.277168+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:20:47.277168+0000","last_undegraded":"2023-06-10T21:20:47.277168+0000","last_fullsized":"2023-06-10T21:20:47.277168+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.741037+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:20:50.741037+0000","last_peered":"2023-06-10T21:20:50.741037+0000","last_clean":"2023-06-10T21:20:50.741037+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:20:50.741037+0000","last_undegraded":"2023-06-10T21:20:50.741037+0000","last_fullsized":"2023-06-10T21:20:50.741037+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"131'1165","reported_seq":1177,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.443877+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:20:59.443877+0000","last_peered":"2023-06-10T21:20:59.443877+0000","last_clean":"2023-06-10T21:20:59.443877+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:20:59.443877+0000","last_undegraded":"2023-06-10T21:20:59.443877+0000","last_fullsized":"2023-06-10T21:20:59.443877+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1165,"log_dups_size":0,"ondisk_log_size":1165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76349440,"num_objects":1165,"num_object_clones":0,"num_object_copies":3495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1165,"num_write_kb":74560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":100,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:55.711018+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:20:55.711018+0000","last_peered":"2023-06-10T21:20:55.711018+0000","last_clean":"2023-06-10T21:20:55.711018+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:20:55.711018+0000","last_undegraded":"2023-06-10T21:20:55.711018+0000","last_fullsized":"2023-06-10T21:20:55.711018+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705007+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:20:47.705007+0000","last_peered":"2023-06-10T21:20:47.705007+0000","last_clean":"2023-06-10T21:20:47.705007+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:20:47.705007+0000","last_undegraded":"2023-06-10T21:20:47.705007+0000","last_fullsized":"2023-06-10T21:20:47.705007+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702715+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:20:47.702715+0000","last_peered":"2023-06-10T21:20:47.702715+0000","last_clean":"2023-06-10T21:20:47.702715+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:20:47.702715+0000","last_undegraded":"2023-06-10T21:20:47.702715+0000","last_fullsized":"2023-06-10T21:20:47.702715+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"131'944","reported_seq":957,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:57.688263+0000","last_change":"2023-06-10T21:20:45.269372+0000","last_active":"2023-06-10T21:20:57.688263+0000","last_peered":"2023-06-10T21:20:57.688263+0000","last_clean":"2023-06-10T21:20:57.688263+0000","last_became_active":"2023-06-10T21:20:45.269098+0000","last_became_peered":"2023-06-10T21:20:45.269098+0000","last_unstale":"2023-06-10T21:20:57.688263+0000","last_undegraded":"2023-06-10T21:20:57.688263+0000","last_fullsized":"2023-06-10T21:20:57.688263+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":944,"log_dups_size":0,"ondisk_log_size":944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:23:51.802626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61865984,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":944,"num_write_kb":60416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704540+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:20:47.704540+0000","last_peered":"2023-06-10T21:20:47.704540+0000","last_clean":"2023-06-10T21:20:47.704540+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:20:47.704540+0000","last_undegraded":"2023-06-10T21:20:47.704540+0000","last_fullsized":"2023-06-10T21:20:47.704540+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703901+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:20:47.703901+0000","last_peered":"2023-06-10T21:20:47.703901+0000","last_clean":"2023-06-10T21:20:47.703901+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:20:47.703901+0000","last_undegraded":"2023-06-10T21:20:47.703901+0000","last_fullsized":"2023-06-10T21:20:47.703901+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":123,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:55.711036+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:20:55.711036+0000","last_peered":"2023-06-10T21:20:55.711036+0000","last_clean":"2023-06-10T21:20:55.711036+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:20:55.711036+0000","last_undegraded":"2023-06-10T21:20:55.711036+0000","last_fullsized":"2023-06-10T21:20:55.711036+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"131'1216","reported_seq":1228,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.438141+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:20:59.438141+0000","last_peered":"2023-06-10T21:20:59.438141+0000","last_clean":"2023-06-10T21:20:59.438141+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:20:59.438141+0000","last_undegraded":"2023-06-10T21:20:59.438141+0000","last_fullsized":"2023-06-10T21:20:59.438141+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1216,"log_dups_size":0,"ondisk_log_size":1216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79691776,"num_objects":1216,"num_object_clones":0,"num_object_copies":3648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1216,"num_write_kb":77824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702010+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:20:47.702010+0000","last_peered":"2023-06-10T21:20:47.702010+0000","last_clean":"2023-06-10T21:20:47.702010+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:20:47.702010+0000","last_undegraded":"2023-06-10T21:20:47.702010+0000","last_fullsized":"2023-06-10T21:20:47.702010+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.740462+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:20:50.740462+0000","last_peered":"2023-06-10T21:20:50.740462+0000","last_clean":"2023-06-10T21:20:50.740462+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:20:50.740462+0000","last_undegraded":"2023-06-10T21:20:50.740462+0000","last_fullsized":"2023-06-10T21:20:50.740462+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.741120+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:20:50.741120+0000","last_peered":"2023-06-10T21:20:50.741120+0000","last_clean":"2023-06-10T21:20:50.741120+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:20:50.741120+0000","last_undegraded":"2023-06-10T21:20:50.741120+0000","last_fullsized":"2023-06-10T21:20:50.741120+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"131'809","reported_seq":822,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:56.514263+0000","last_change":"2023-06-10T21:20:45.262983+0000","last_active":"2023-06-10T21:20:56.514263+0000","last_peered":"2023-06-10T21:20:56.514263+0000","last_clean":"2023-06-10T21:20:56.514263+0000","last_became_active":"2023-06-10T21:20:45.262641+0000","last_became_peered":"2023-06-10T21:20:45.262641+0000","last_unstale":"2023-06-10T21:20:56.514263+0000","last_undegraded":"2023-06-10T21:20:56.514263+0000","last_fullsized":"2023-06-10T21:20:56.514263+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":809,"log_dups_size":0,"ondisk_log_size":809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:27:45.171697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53018624,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":809,"num_write_kb":51776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705065+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:20:47.705065+0000","last_peered":"2023-06-10T21:20:47.705065+0000","last_clean":"2023-06-10T21:20:47.705065+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:20:47.705065+0000","last_undegraded":"2023-06-10T21:20:47.705065+0000","last_fullsized":"2023-06-10T21:20:47.705065+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":334,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703595+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:20:47.703595+0000","last_peered":"2023-06-10T21:20:47.703595+0000","last_clean":"2023-06-10T21:20:47.703595+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:20:47.703595+0000","last_undegraded":"2023-06-10T21:20:47.703595+0000","last_fullsized":"2023-06-10T21:20:47.703595+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277273+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:20:47.277273+0000","last_peered":"2023-06-10T21:20:47.277273+0000","last_clean":"2023-06-10T21:20:47.277273+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:20:47.277273+0000","last_undegraded":"2023-06-10T21:20:47.277273+0000","last_fullsized":"2023-06-10T21:20:47.277273+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.706056+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:20:47.706056+0000","last_peered":"2023-06-10T21:20:47.706056+0000","last_clean":"2023-06-10T21:20:47.706056+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:20:47.706056+0000","last_undegraded":"2023-06-10T21:20:47.706056+0000","last_fullsized":"2023-06-10T21:20:47.706056+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"131'1015","reported_seq":1028,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:57.688239+0000","last_change":"2023-06-10T21:20:45.259115+0000","last_active":"2023-06-10T21:20:57.688239+0000","last_peered":"2023-06-10T21:20:57.688239+0000","last_clean":"2023-06-10T21:20:57.688239+0000","last_became_active":"2023-06-10T21:20:45.258813+0000","last_became_peered":"2023-06-10T21:20:45.258813+0000","last_unstale":"2023-06-10T21:20:57.688239+0000","last_undegraded":"2023-06-10T21:20:57.688239+0000","last_fullsized":"2023-06-10T21:20:57.688239+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1015,"log_dups_size":0,"ondisk_log_size":1015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:29.551738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66519040,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1015,"num_write_kb":64960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":117,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703480+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:20:47.703480+0000","last_peered":"2023-06-10T21:20:47.703480+0000","last_clean":"2023-06-10T21:20:47.703480+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:20:47.703480+0000","last_undegraded":"2023-06-10T21:20:47.703480+0000","last_fullsized":"2023-06-10T21:20:47.703480+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.701793+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:20:47.701793+0000","last_peered":"2023-06-10T21:20:47.701793+0000","last_clean":"2023-06-10T21:20:47.701793+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:20:47.701793+0000","last_undegraded":"2023-06-10T21:20:47.701793+0000","last_fullsized":"2023-06-10T21:20:47.701793+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702168+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:20:47.702168+0000","last_peered":"2023-06-10T21:20:47.702168+0000","last_clean":"2023-06-10T21:20:47.702168+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:20:47.702168+0000","last_undegraded":"2023-06-10T21:20:47.702168+0000","last_fullsized":"2023-06-10T21:20:47.702168+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"131'1143","reported_seq":1155,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.225619+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:20:59.225619+0000","last_peered":"2023-06-10T21:20:59.225619+0000","last_clean":"2023-06-10T21:20:59.225619+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:20:59.225619+0000","last_undegraded":"2023-06-10T21:20:59.225619+0000","last_fullsized":"2023-06-10T21:20:59.225619+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1143,"log_dups_size":0,"ondisk_log_size":1143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74907648,"num_objects":1143,"num_object_clones":0,"num_object_copies":3429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1143,"num_write_kb":73152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704825+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:20:47.704825+0000","last_peered":"2023-06-10T21:20:47.704825+0000","last_clean":"2023-06-10T21:20:47.704825+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:20:47.704825+0000","last_undegraded":"2023-06-10T21:20:47.704825+0000","last_fullsized":"2023-06-10T21:20:47.704825+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:55.711054+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:20:55.711054+0000","last_peered":"2023-06-10T21:20:55.711054+0000","last_clean":"2023-06-10T21:20:55.711054+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:20:55.711054+0000","last_undegraded":"2023-06-10T21:20:55.711054+0000","last_fullsized":"2023-06-10T21:20:55.711054+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":70,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702047+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:20:47.702047+0000","last_peered":"2023-06-10T21:20:47.702047+0000","last_clean":"2023-06-10T21:20:47.702047+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:20:47.702047+0000","last_undegraded":"2023-06-10T21:20:47.702047+0000","last_fullsized":"2023-06-10T21:20:47.702047+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"131'1194","reported_seq":1206,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:59.220148+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:20:59.220148+0000","last_peered":"2023-06-10T21:20:59.220148+0000","last_clean":"2023-06-10T21:20:59.220148+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:20:59.220148+0000","last_undegraded":"2023-06-10T21:20:59.220148+0000","last_fullsized":"2023-06-10T21:20:59.220148+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":1194,"log_dups_size":0,"ondisk_log_size":1194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78249984,"num_objects":1194,"num_object_clones":0,"num_object_copies":3582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1194,"num_write_kb":76416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277549+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:20:47.277549+0000","last_peered":"2023-06-10T21:20:47.277549+0000","last_clean":"2023-06-10T21:20:47.277549+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:20:47.277549+0000","last_undegraded":"2023-06-10T21:20:47.277549+0000","last_fullsized":"2023-06-10T21:20:47.277549+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:50.740543+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:20:50.740543+0000","last_peered":"2023-06-10T21:20:50.740543+0000","last_clean":"2023-06-10T21:20:50.740543+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:20:50.740543+0000","last_undegraded":"2023-06-10T21:20:50.740543+0000","last_fullsized":"2023-06-10T21:20:50.740543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277152+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:20:47.277152+0000","last_peered":"2023-06-10T21:20:47.277152+0000","last_clean":"2023-06-10T21:20:47.277152+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:20:47.277152+0000","last_undegraded":"2023-06-10T21:20:47.277152+0000","last_fullsized":"2023-06-10T21:20:47.277152+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"131'727","reported_seq":740,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-06-10T21:20:56.520585+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:20:56.520585+0000","last_peered":"2023-06-10T21:20:56.520585+0000","last_clean":"2023-06-10T21:20:56.520585+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:20:56.520585+0000","last_undegraded":"2023-06-10T21:20:56.520585+0000","last_fullsized":"2023-06-10T21:20:56.520585+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":727,"log_dups_size":0,"ondisk_log_size":727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":727,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.706111+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:20:47.706111+0000","last_peered":"2023-06-10T21:20:47.706111+0000","last_clean":"2023-06-10T21:20:47.706111+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:20:47.706111+0000","last_undegraded":"2023-06-10T21:20:47.706111+0000","last_fullsized":"2023-06-10T21:20:47.706111+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.701983+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:20:47.701983+0000","last_peered":"2023-06-10T21:20:47.701983+0000","last_clean":"2023-06-10T21:20:47.701983+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:20:47.701983+0000","last_undegraded":"2023-06-10T21:20:47.701983+0000","last_fullsized":"2023-06-10T21:20:47.701983+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.704356+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:20:47.704356+0000","last_peered":"2023-06-10T21:20:47.704356+0000","last_clean":"2023-06-10T21:20:47.704356+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:20:47.704356+0000","last_undegraded":"2023-06-10T21:20:47.704356+0000","last_fullsized":"2023-06-10T21:20:47.704356+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702261+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:20:47.702261+0000","last_peered":"2023-06-10T21:20:47.702261+0000","last_clean":"2023-06-10T21:20:47.702261+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:20:47.702261+0000","last_undegraded":"2023-06-10T21:20:47.702261+0000","last_fullsized":"2023-06-10T21:20:47.702261+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702426+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:20:47.702426+0000","last_peered":"2023-06-10T21:20:47.702426+0000","last_clean":"2023-06-10T21:20:47.702426+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:20:47.702426+0000","last_undegraded":"2023-06-10T21:20:47.702426+0000","last_fullsized":"2023-06-10T21:20:47.702426+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705139+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:20:47.705139+0000","last_peered":"2023-06-10T21:20:47.705139+0000","last_clean":"2023-06-10T21:20:47.705139+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:20:47.705139+0000","last_undegraded":"2023-06-10T21:20:47.705139+0000","last_fullsized":"2023-06-10T21:20:47.705139+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703937+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:20:47.703937+0000","last_peered":"2023-06-10T21:20:47.703937+0000","last_clean":"2023-06-10T21:20:47.703937+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:20:47.703937+0000","last_undegraded":"2023-06-10T21:20:47.703937+0000","last_fullsized":"2023-06-10T21:20:47.703937+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.705131+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:20:47.705131+0000","last_peered":"2023-06-10T21:20:47.705131+0000","last_clean":"2023-06-10T21:20:47.705131+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:20:47.705131+0000","last_undegraded":"2023-06-10T21:20:47.705131+0000","last_fullsized":"2023-06-10T21:20:47.705131+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703841+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:20:47.703841+0000","last_peered":"2023-06-10T21:20:47.703841+0000","last_clean":"2023-06-10T21:20:47.703841+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:20:47.703841+0000","last_undegraded":"2023-06-10T21:20:47.703841+0000","last_fullsized":"2023-06-10T21:20:47.703841+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277459+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:20:47.277459+0000","last_peered":"2023-06-10T21:20:47.277459+0000","last_clean":"2023-06-10T21:20:47.277459+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:20:47.277459+0000","last_undegraded":"2023-06-10T21:20:47.277459+0000","last_fullsized":"2023-06-10T21:20:47.277459+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.277273+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:20:47.277273+0000","last_peered":"2023-06-10T21:20:47.277273+0000","last_clean":"2023-06-10T21:20:47.277273+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:20:47.277273+0000","last_undegraded":"2023-06-10T21:20:47.277273+0000","last_fullsized":"2023-06-10T21:20:47.277273+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.702364+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:20:47.702364+0000","last_peered":"2023-06-10T21:20:47.702364+0000","last_clean":"2023-06-10T21:20:47.702364+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:20:47.702364+0000","last_undegraded":"2023-06-10T21:20:47.702364+0000","last_fullsized":"2023-06-10T21:20:47.702364+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-06-10T21:20:47.703482+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:20:47.703482+0000","last_peered":"2023-06-10T21:20:47.703482+0000","last_clean":"2023-06-10T21:20:47.703482+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:20:47.703482+0000","last_undegraded":"2023-06-10T21:20:47.703482+0000","last_fullsized":"2023-06-10T21:20:47.703482+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1098317824,"num_objects":16759,"num_object_clones":0,"num_object_copies":50277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16759,"num_write_kb":1072576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3476226048,"data_stored":3476226048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16759,"ondisk_log_size":16759,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430487,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937340,"kb_used_data":626868,"kb_used_omap":0,"kb_used_meta":310464,"kb_avail":92807812,"statfs":{"total":95995035648,"available":95035199488,"internally_reserved":0,"allocated":641912832,"data_stored":641422705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317915136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.258,"15min":1.3480000000000001},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.22800000000000001},"max":{"1min":0.68899999999999995,"5min":10.914999999999999,"15min":10.914999999999999},"last":1.6020000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.96,"15min":1.95},"min":{"1min":0.41999999999999998,"5min":0.31900000000000001,"15min":0.245},"max":{"1min":0.75600000000000001,"5min":15.925000000000001,"15min":15.925000000000001},"last":1.363}]},{"osd":1,"last update":"Sat Jun 10 21:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.6899999999999999,"15min":1.482},"min":{"1min":0.436,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69899999999999995,"5min":10.083,"15min":10.083},"last":1.26},{"interface":"front","average":{"1min":0.498,"5min":1.3600000000000001,"15min":1.454},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":0.68700000000000006,"5min":6.4530000000000003,"15min":7},"last":0.88700000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":1.218},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.91700000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":1.1080000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.94699999999999995,"15min":1.9730000000000001},"min":{"1min":0.39800000000000002,"5min":0.28100000000000003,"15min":0.22800000000000001},"max":{"1min":0.66800000000000004,"5min":5.1319999999999997,"15min":219.17699999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.3140000000000001,"15min":2.1469999999999998},"min":{"1min":0.439,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":0.70899999999999996,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.77800000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.464,"15min":2.573},"min":{"1min":0.40600000000000003,"5min":0.29199999999999998,"15min":0.254},"max":{"1min":0.81899999999999995,"5min":37.759,"15min":311.202},"last":1.179},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.47,"15min":2.484},"min":{"1min":0.45700000000000002,"5min":0.33000000000000002,"15min":0.29699999999999999},"max":{"1min":0.79000000000000004,"5min":30.074999999999999,"15min":311.875},"last":0.48199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.1819999999999999,"15min":1.111},"min":{"1min":0.48799999999999999,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78300000000000003,"5min":6.8570000000000002,"15min":8.6630000000000003},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.97599999999999998,"15min":1.036},"min":{"1min":0.53300000000000003,"5min":0.34399999999999997,"15min":0.32400000000000001},"max":{"1min":0.83099999999999996,"5min":7.6900000000000004,"15min":8.5640000000000001},"last":0.69899999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626715,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484204,"kb_used_data":205668,"kb_used_omap":0,"kb_used_meta":278528,"kb_avail":93260948,"statfs":{"total":95995035648,"available":95499210752,"internally_reserved":0,"allocated":210604032,"data_stored":210129828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285212672},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":1.4059999999999999,"15min":2.1970000000000001},"min":{"1min":0.32100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":0.80200000000000005,"5min":9.7929999999999993,"15min":119.276},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.704,"15min":2.8410000000000002},"min":{"1min":0.378,"5min":0.252,"15min":0.252},"max":{"1min":0.73099999999999998,"5min":16.324999999999999,"15min":119.22},"last":0.33000000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:20:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.5309999999999999,"15min":1.7370000000000001},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":1.1599999999999999,"5min":14.897,"15min":14.897},"last":0.503},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.744,"15min":1.786},"min":{"1min":0.371,"5min":0.251,"15min":0.251},"max":{"1min":0.88300000000000001,"5min":10.766999999999999,"15min":10.766999999999999},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:20:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.95599999999999996,"15min":1.7909999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.66600000000000004,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.434},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.1890000000000001,"15min":1.994},"min":{"1min":0.379,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":0.71099999999999997,"5min":11.263999999999999,"15min":221.905},"last":0.38}]},{"osd":5,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.0149999999999999,"15min":2.2799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.72099999999999997,"5min":6.6680000000000001,"15min":313.661},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1080000000000001,"15min":2.3490000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.74399999999999999,"5min":9.75,"15min":314.38900000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:20:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.165,"15min":1.133},"min":{"1min":0.44,"5min":0.318,"15min":0.318},"max":{"1min":0.78100000000000003,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.1120000000000001,"15min":1.1200000000000001},"min":{"1min":0.441,"5min":0.308,"15min":0.308},"max":{"1min":0.76700000000000002,"5min":8.0980000000000008,"15min":8.8949999999999996},"last":0.497}]}]},{"osd":1,"up_from":105,"seq":450971566158,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914236,"kb_used_data":705652,"kb_used_omap":4,"kb_used_meta":208571,"kb_avail":92830916,"statfs":{"total":95995035648,"available":95058857984,"internally_reserved":0,"allocated":722587648,"data_stored":722040617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":213577614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.6539999999999999,"15min":1.7330000000000001},"min":{"1min":0.29499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58699999999999997,"5min":10.898,"15min":10.898},"last":12.380000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.052,"15min":2.2530000000000001},"min":{"1min":0.29599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61499999999999999,"5min":16.488,"15min":16.488},"last":12.432}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.895,"5min":1.895,"15min":1.895},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.907,"5min":1.907,"15min":1.907},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":1.208},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.81000000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.5640000000000001,"15min":1.8380000000000001},"min":{"1min":0.439,"5min":0.312,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":12.94,"15min":12.94},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.625,"5min":1.5229999999999999,"15min":2.5430000000000001},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93100000000000005,"5min":9.3870000000000005,"15min":12.603},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.899,"15min":2.0699999999999998},"min":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":0.97699999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":0.875},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.883,"15min":2.3700000000000001},"min":{"1min":0.45900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.96199999999999997,"5min":13.602,"15min":13.602},"last":0.88400000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.887,"15min":2.1070000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.91200000000000003,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.7949999999999999,"15min":2.3879999999999999},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82599999999999996,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.79000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.8080000000000001,"15min":3.0499999999999998},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.89200000000000002,"5min":13.025,"15min":13.025},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.8049999999999999,"15min":2.0590000000000002},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.85699999999999998,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.45400000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187580,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534292,"kb_used_data":227084,"kb_used_omap":5,"kb_used_meta":307194,"kb_avail":93210860,"statfs":{"total":95995035648,"available":95447920640,"internally_reserved":0,"allocated":232534016,"data_stored":231964635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":314567290},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":1.643,"15min":1.95},"min":{"1min":0.24399999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46600000000000003,"5min":16.780999999999999,"15min":16.780999999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.36299999999999999,"5min":2.7570000000000001,"15min":2.8620000000000001},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.65400000000000003,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.308}]},{"osd":2,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.441,"5min":0.441,"15min":0.441},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.27700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":163.56999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":164.239}]},{"osd":4,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.093,"15min":2.2639999999999998},"min":{"1min":0.437,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1609999999999996,"5min":10.948,"15min":10.948},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.5780000000000001,"15min":2.0179999999999998},"min":{"1min":0.42799999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":0.81799999999999995,"5min":8.8160000000000007,"15min":10.667},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.754,"15min":2.375},"min":{"1min":0.42699999999999999,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":0.86099999999999999,"5min":9.7119999999999997,"15min":10.693},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.5649999999999999,"15min":2.5840000000000001},"min":{"1min":0.442,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":10.964,"15min":17.460000000000001},"last":0.69999999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.873,"15min":2.4460000000000002},"min":{"1min":0.505,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":0.95799999999999996,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":2.6520000000000001,"15min":2.7029999999999998},"min":{"1min":0.503,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":0.94499999999999995,"5min":11.022,"15min":17.411000000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":2.5390000000000001,"15min":2.7029999999999998},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.36399999999999999},"max":{"1min":0.81299999999999994,"5min":11.224,"15min":17.486999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":2.3100000000000001,"15min":2.194},"min":{"1min":0.53000000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82699999999999996,"5min":9.3960000000000008,"15min":10.867000000000001},"last":0.64100000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075546,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358152,"kb_used_data":201472,"kb_used_omap":7,"kb_used_meta":156664,"kb_avail":93387000,"statfs":{"total":95995035648,"available":95628288000,"internally_reserved":0,"allocated":206307328,"data_stored":205806126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":160424017},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.378,"15min":0.378},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.253},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.71399999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":1.1739999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":1.4219999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:20:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":1.335},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":1.462}]}]},{"osd":3,"up_from":125,"seq":536870912020,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492484,"kb_used_data":378236,"kb_used_omap":2,"kb_used_meta":114237,"kb_avail":93252668,"statfs":{"total":95995035648,"available":95490732032,"internally_reserved":0,"allocated":387313664,"data_stored":386816462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":116978741},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"last":1.5820000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":1.282},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":1.8480000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":1.637},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.624}]},{"osd":4,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":1.748},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":1.879},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.499}]},{"osd":6,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":1.165},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.84799999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":1.254},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.81000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019171,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":537008,"kb_used_data":205480,"kb_used_omap":0,"kb_used_meta":331520,"kb_avail":93208144,"statfs":{"total":95995035648,"available":95445139456,"internally_reserved":0,"allocated":210411520,"data_stored":209932752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":339476480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.2869999999999999,"15min":2.0550000000000002},"min":{"1min":0.34599999999999997,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":0.66200000000000003,"5min":8.9269999999999996,"15min":10.01},"last":1.593},{"interface":"front","average":{"1min":0.48599999999999999,"5min":1.8520000000000001,"15min":2.298},"min":{"1min":0.33000000000000002,"5min":0.25,"15min":0.23100000000000001},"max":{"1min":0.71499999999999997,"5min":9.593,"15min":15.476000000000001},"last":1.373}]},{"osd":1,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.4159999999999999,"15min":1.728},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":0.86699999999999999,"5min":15.757,"15min":15.757},"last":1.6899999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.4550000000000001,"15min":2.2130000000000001},"min":{"1min":0.39200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66600000000000004,"5min":15.125,"15min":15.125},"last":1.625}]},{"osd":2,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":1.5129999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.6699999999999999,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":1.7450000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":1.5529999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:20:00 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.81899999999999995,"15min":1.2430000000000001},"min":{"1min":0.32000000000000001,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":0.70399999999999996,"5min":6.5010000000000003,"15min":92.653000000000006},"last":1.667},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.0649999999999999,"15min":1.532},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.253},"max":{"1min":0.83199999999999996,"5min":6.9539999999999997,"15min":93.281999999999996},"last":1.4810000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.92200000000000004,"15min":0.97299999999999998},"min":{"1min":0.378,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.69199999999999995,"5min":5.867,"15min":6.4340000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.998,"15min":0.88700000000000001},"min":{"1min":0.49199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81200000000000006,"5min":8.8420000000000005,"15min":8.8420000000000005},"last":1.73}]},{"osd":7,"last update":"Sat Jun 10 21:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.1579999999999999,"15min":1.278},"min":{"1min":0.42499999999999999,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":0.85099999999999998,"5min":8.4459999999999997,"15min":19.989000000000001},"last":1.71},{"interface":"front","average":{"1min":0.626,"5min":0.92400000000000004,"15min":1.083},"min":{"1min":0.435,"5min":0.314,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":4.7709999999999999,"15min":19.783000000000001},"last":0.52600000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822943,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057504,"kb_used_data":660376,"kb_used_omap":0,"kb_used_meta":397120,"kb_avail":92687648,"statfs":{"total":95995035648,"available":94912151552,"internally_reserved":0,"allocated":676225024,"data_stored":675758402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":406650880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:06 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":1.3220000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":0.67800000000000005,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.0139999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.85599999999999998,"5min":11.807,"15min":11.807},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.76,"15min":1.907},"min":{"1min":0.27800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.66100000000000003,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":1.385},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.9370000000000001,"15min":2.492},"min":{"1min":0.45200000000000001,"5min":0.28299999999999997,"15min":0.26800000000000002},"max":{"1min":0.81100000000000005,"5min":15.962999999999999,"15min":15.962999999999999},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:20:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":2.774}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":1.6100000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":2.819}]},{"osd":4,"last update":"Sat Jun 10 21:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.0649999999999999,"15min":1.0069999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":0.68000000000000005,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.228,"15min":1.1439999999999999},"min":{"1min":0.35699999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.80500000000000005,"5min":9.218,"15min":12.901},"last":1.0740000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:20:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.107,"15min":1.1080000000000001},"min":{"1min":0.38500000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.72399999999999998,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0700000000000001,"15min":1.2430000000000001},"min":{"1min":0.44,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":0.84399999999999997,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.98999999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.2849999999999999,"15min":1.2989999999999999},"min":{"1min":0.44,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.872,"5min":8.5109999999999992,"15min":8.5109999999999992},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.25,"15min":1.111},"min":{"1min":0.437,"5min":0.318,"15min":0.318},"max":{"1min":0.753,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":258015232,"data_stored":258015232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":778567680,"data_stored":778567680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":210108416,"data_stored":210108416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":392822784,"data_stored":392822784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":223674368,"data_stored":223674368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":707264512,"data_stored":707264512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":225837056,"data_stored":225837056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":679936000,"data_stored":679936000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:21:04.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_4 pgp_num 16 2023-06-10T21:21:04.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:21:04.711 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:04.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:04.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:04.820 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:04 smithi064 ceph-mon[186706]: pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-06-10T21:21:04.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:04 smithi064 ceph-mon[180297]: pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-06-10T21:21:04.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:04.941 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:04 smithi152 ceph-mon[156819]: pgmap v523: 89 pgs: 89 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-06-10T21:21:05.161 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:05.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:05.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:05.314 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:05.406 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:05.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:05.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:05.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:05.525 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 29814 29798 143.228 107.812 0.00189384 0.0069588 2023-06-10T21:21:05.542 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:05.583 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:05.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:05.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:05.720 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:05.778 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:05.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:05.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:05.960 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:06.197 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:06.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:06.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:06.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:06.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:06.532 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 31027 31011 138.344 75.8125 0.00740186 0.00722195 2023-06-10T21:21:06.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:06.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:06.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:06.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:06.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:06.777 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:06.780 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:06.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:06 smithi064 ceph-mon[186706]: pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:21:06.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:06 smithi064 ceph-mon[180297]: pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:21:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:06 smithi152 ceph-mon[156819]: pgmap v524: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2023-06-10T21:21:07.039 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:07.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:07.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:07.057 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:21:07.097 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:21:07.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:21:07.147 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:07.173 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:21:07.208 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:21:07.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:07.514 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:21:07.515 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:21:07.515 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:21:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.516 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:21:07.532 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 32297 32281 134.414 79.375 0.00307646 0.00730095 2023-06-10T21:21:07.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1342323792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:07.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1342323792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1342323792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:08.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:08.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:08.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:08.322 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:08.532 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 34100 34084 133.056 112.688 0.00184485 0.00748499 2023-06-10T21:21:08.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:08 smithi064 ceph-mon[186706]: pgmap v525: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-06-10T21:21:08.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:08 smithi064 ceph-mon[180297]: pgmap v525: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-06-10T21:21:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:08 smithi152 ceph-mon[156819]: pgmap v525: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.17k op/s 2023-06-10T21:21:09.043 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:09.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:09.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:09.204 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:09.280 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:09.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:09.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:09.424 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:09.501 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:09.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:09.532 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 35821 35805 131.556 107.562 0.0113571 0.0075952 2023-06-10T21:21:09.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:09.695 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:10.535 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 37484 37468 130.014 103.938 0.00608845 0.00768639 2023-06-10T21:21:10.566 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:21:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[186706]: pgmap v526: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-10T21:21:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2698214065' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:10.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:10.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[180297]: pgmap v526: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-10T21:21:10.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2698214065' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:10.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:10.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:10 smithi152 ceph-mon[156819]: pgmap v526: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-10T21:21:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2698214065' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:11.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:21:11.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 38528 38512 126.607 65.25 0.00616076 0.00789387 2023-06-10T21:21:11.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:11.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:11.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: osdmap e133: 8 total, 8 up, 8 in 2023-06-10T21:21:11.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[180297]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:11.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:11.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: osdmap e133: 8 total, 8 up, 8 in 2023-06-10T21:21:11.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:11 smithi064 ceph-mon[186706]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: osdmap e133: 8 total, 8 up, 8 in 2023-06-10T21:21:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: failed to encode map e133 with expected crc 2023-06-10T21:21:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:11 smithi152 ceph-mon[156819]: failed to encode map e133 with expected crc 2023-06-10T21:21:12.083 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:12.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:12.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:12.227 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:12.311 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:12.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:12.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:12.476 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:12.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:21:12.531912+0000 min lat: 0.00114967 max lat: 0.351579 avg lat: 0.00799246 2023-06-10T21:21:12.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:21:12.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40052 40036 125.039 95.25 0.0150535 0.00799246 2023-06-10T21:21:12.559 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:12.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:12 smithi064 ceph-mon[186706]: pgmap v528: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:21:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:12 smithi064 ceph-mon[180297]: pgmap v528: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:21:12.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:12.761 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:12.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:12 smithi152 ceph-mon[156819]: pgmap v528: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:21:13.009 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:13.016 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:12] "GET /metrics HTTP/1.1" 200 223344 "" "Prometheus/2.33.4" 2023-06-10T21:21:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:13.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:13.073 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:13.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:13.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:13.464 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:13.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:13.526 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:21:13.526 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:21:13.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 41542 41526 123.52 93.125 0.00261398 0.00808916 2023-06-10T21:21:13.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:13.613 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:13.696 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:13.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:13.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:13.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:13.894 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:14.012 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:14.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:14.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:14.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:14.157 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:14.254 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:14.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:14.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:14.402 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:14.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 43035 43019 122.147 93.3125 0.0019431 0.00815641 2023-06-10T21:21:14.598 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 9 pgp_num to 16 2023-06-10T21:21:14.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[186706]: pgmap v529: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:21:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1326259510' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[180297]: pgmap v529: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:21:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1326259510' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:21:14.857 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:14.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:14.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:14 smithi152 ceph-mon[156819]: pgmap v529: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-06-10T21:21:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1326259510' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:21:14.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:14.990 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:15.077 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:15.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:15.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:15.240 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:15.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 44130 44114 119.812 68.4375 0.00568334 0.00834226 2023-06-10T21:21:15.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[180297]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[180297]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:21:15.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[180297]: osdmap e134: 8 total, 8 up, 8 in 2023-06-10T21:21:15.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[186706]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[186706]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:21:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:15 smithi064 ceph-mon[186706]: osdmap e134: 8 total, 8 up, 8 in 2023-06-10T21:21:15.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:15 smithi152 ceph-mon[156819]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:15 smithi152 ceph-mon[156819]: failed to encode map e134 with expected crc 2023-06-10T21:21:15.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:15 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:21:15.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:15 smithi152 ceph-mon[156819]: osdmap e134: 8 total, 8 up, 8 in 2023-06-10T21:21:16.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:21:16.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:16.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:16.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:16.493 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:16.540 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 45521 45505 118.443 86.9375 0.0173333 0.00843586 2023-06-10T21:21:16.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:16.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:16.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:16.789 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:16.800 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:16.883 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:16.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:16.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:16 smithi064 ceph-mon[180297]: pgmap v531: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-06-10T21:21:16.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:16 smithi064 ceph-mon[186706]: pgmap v531: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-06-10T21:21:16.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:16 smithi152 ceph-mon[156819]: pgmap v531: 89 pgs: 89 active+clean; 2.5 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-06-10T21:21:16.944 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:21:16.987 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:21:16.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:17.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:21:17.030 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:17.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:21:17.087 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:17.116 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:21:17.153 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:21:17.187 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:21:17.210 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:17.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:17.225 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:21:17.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.239 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.240 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.241 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:21:17.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:17.462 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:17.534 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 46916 46900 117.193 87.1875 0.00765943 0.00852778 2023-06-10T21:21:17.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:17.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:17.643 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2245620743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:17.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:17.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2245620743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:17.726 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:17.831 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:17.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2245620743' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:17.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:18.009 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:18.129 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:18.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:18.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:18.294 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:18.402 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:18.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:18.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:18.535 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 48475 48459 116.433 97.4375 0.00357248 0.00857971 2023-06-10T21:21:18.546 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:18.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:18 smithi064 ceph-mon[186706]: pgmap v532: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-10T21:21:18.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:18 smithi064 ceph-mon[180297]: pgmap v532: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-10T21:21:18.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:18 smithi152 ceph-mon[156819]: pgmap v532: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-10T21:21:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:19.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:19.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:19.536 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 49822 49806 115.231 84.1875 0.0122953 0.00867307 2023-06-10T21:21:19.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:19.613 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:19.655 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:19.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:19.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:19.811 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:19.890 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:19.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:19.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:19.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:20.061 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:20.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:20.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:20.303 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:20.376 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:20.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:20.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:20.538 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 51036 51020 113.826 75.875 0.0305977 0.00878045 2023-06-10T21:21:20.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:20.657 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:20 smithi152 ceph-mon[156819]: pgmap v533: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-06-10T21:21:20.688 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:20.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:20.797 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:21:20.798 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:21:20.798 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:21:20.798 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:21:20.798 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:21:20.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:20.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:20 smithi064 ceph-mon[186706]: pgmap v533: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-06-10T21:21:20.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:20 smithi064 ceph-mon[180297]: pgmap v533: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-06-10T21:21:20.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:21.023 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:21.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:21.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:21.179 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:21.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:21.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:21.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:21.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:21.436 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:21:21.436 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:21:21.437 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-06-10T21:21:21.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:21:21.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:21.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:21.536 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 52216 52200 112.445 73.75 0.00194418 0.00884001 2023-06-10T21:21:21.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:21.610 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:21.661 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:21.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:21.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:21.696 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:21:21.731 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2287056934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:21.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:21.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2287056934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:21.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:21.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:21.834 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:21.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:21:21.918 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:21.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:21.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2287056934' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:21.929 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:22.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:22.105 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:22.537 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 53151 53135 110.642 58.4375 0.00476945 0.00903347 2023-06-10T21:21:22.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:21:22.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:22.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:22.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:22] "GET /metrics HTTP/1.1" 200 223416 "" "Prometheus/2.33.4" 2023-06-10T21:21:22.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: pgmap v534: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:21:22.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:21:22.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:21:22.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:22.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: osdmap e135: 8 total, 8 up, 8 in 2023-06-10T21:21:22.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[180297]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: pgmap v534: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: osdmap e135: 8 total, 8 up, 8 in 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:22 smithi064 ceph-mon[186706]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: pgmap v534: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-06-10T21:21:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:21:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: osdmap e135: 8 total, 8 up, 8 in 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:22 smithi152 ceph-mon[156819]: failed to encode map e135 with expected crc 2023-06-10T21:21:22.981 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:23.043 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:23.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:23.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:23.201 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:23.209 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:23.262 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:23.273 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:23.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:23.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:23.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:23.402 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:23.445 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:23.537 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 54447 54431 109.685 81 0.00914424 0.00911305 2023-06-10T21:21:23.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:23.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:23.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:24.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:24.105 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:24.507 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:24.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:24.538 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 55699 55683 108.703 78.25 0.00927271 0.00919518 2023-06-10T21:21:24.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:24.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:24 smithi064 ceph-mon[186706]: pgmap v536: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:21:24.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:24 smithi064 ceph-mon[180297]: pgmap v536: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:21:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:24 smithi152 ceph-mon[156819]: pgmap v536: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:21:25.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:21:25.214 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:25.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:25.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:25.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:25.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:25.538 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 56816 56800 107.525 69.8125 0.00844465 0.00929446 2023-06-10T21:21:25.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:25.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:25.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:26.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:26.144 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:26.232 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:26.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:26.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:26.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:26.367 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:26.408 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:26.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:26.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:26.538 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 57795 57779 106.162 61.1875 0.00984881 0.00941465 2023-06-10T21:21:26.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:26.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:26 smithi152 ceph-mon[156819]: pgmap v537: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:21:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:26 smithi064 ceph-mon[186706]: pgmap v537: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:21:26.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:26 smithi064 ceph-mon[180297]: pgmap v537: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:21:27.037 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:27.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:27.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:27.247 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:27.476 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:27.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:27.509 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:27.541 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 59125 59109 105.494 83.125 0.00465887 0.00947376 2023-06-10T21:21:27.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:27.603 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:21:27.630 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:27.655 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:21:27.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:21:27.705 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:27.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:27.739 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1322479709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:27.740 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1322479709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:27.751 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:21:27.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:21:27.815 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:27.836 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:21:27.875 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:21:27.920 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:21:27.923 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:27.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1322479709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.946 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.947 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:21:27.947 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:21:28.488 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:21:28.488 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:21:28.496 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:28.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:28.541 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 60294 60278 104.593 73.0625 0.0144304 0.00955553 2023-06-10T21:21:28.553 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:21:28.555 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:21:28.605 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:28.776 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:28.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:28.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:28 smithi064 ceph-mon[186706]: pgmap v538: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:21:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:28 smithi064 ceph-mon[180297]: pgmap v538: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:21:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:28 smithi152 ceph-mon[156819]: pgmap v538: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:21:29.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:29.543 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 61575 61559 103.93 80.0625 0.00607889 0.00961715 2023-06-10T21:21:29.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:29 smithi064 ceph-mon[186706]: from='client.55306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:29 smithi064 ceph-mon[180297]: from='client.55306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:30.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:29 smithi152 ceph-mon[156819]: from='client.55306 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:21:30.541 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 62813 62797 103.232 77.375 0.0147119 0.00968187 2023-06-10T21:21:30.690 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:30.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:30.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:30.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:30.828 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:31.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:30 smithi064 ceph-mon[186706]: pgmap v539: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:21:31.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:30 smithi064 ceph-mon[180297]: pgmap v539: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:21:31.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:30 smithi152 ceph-mon[156819]: pgmap v539: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:21:31.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:21:31.542 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 63807 63791 102.178 62.125 0.00399121 0.00977583 2023-06-10T21:21:31.747 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:31.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:31.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:31.904 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:31.997 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:32.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:32.060 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":538,"stamp":"2023-06-10T21:21:27.424854+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3659077518,"num_objects":55848,"num_object_clones":0,"num_object_copies":167544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55848,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":56085,"num_write_kb":3575182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":56019,"ondisk_log_size":56019,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12817008,"kb_used_data":10602544,"kb_used_omap":20,"kb_used_meta":2214379,"kb_avail":737144208,"statfs":{"total":767960285184,"available":754835668992,"internally_reserved":0,"allocated":10857005056,"data_stored":10852940020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2267524750},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,10,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":828112896,"num_objects":12636,"num_object_clones":0,"num_object_copies":37908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12636,"num_write_kb":808704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002763"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593241+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:21:14.593241+0000","last_peered":"2023-06-10T21:21:14.593241+0000","last_clean":"2023-06-10T21:21:14.593241+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:21:14.593241+0000","last_undegraded":"2023-06-10T21:21:14.593241+0000","last_fullsized":"2023-06-10T21:21:14.593241+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676128+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:21:14.676128+0000","last_peered":"2023-06-10T21:21:14.676128+0000","last_clean":"2023-06-10T21:21:14.676128+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:21:14.676128+0000","last_undegraded":"2023-06-10T21:21:14.676128+0000","last_fullsized":"2023-06-10T21:21:14.676128+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.673808+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:21:14.673808+0000","last_peered":"2023-06-10T21:21:14.673808+0000","last_clean":"2023-06-10T21:21:14.673808+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:21:14.673808+0000","last_undegraded":"2023-06-10T21:21:14.673808+0000","last_fullsized":"2023-06-10T21:21:14.673808+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":90,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678515+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:21:14.678515+0000","last_peered":"2023-06-10T21:21:14.678515+0000","last_clean":"2023-06-10T21:21:14.678515+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:21:14.678515+0000","last_undegraded":"2023-06-10T21:21:14.678515+0000","last_fullsized":"2023-06-10T21:21:14.678515+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.722483+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:21:14.722483+0000","last_peered":"2023-06-10T21:21:14.722483+0000","last_clean":"2023-06-10T21:21:14.722483+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:21:14.722483+0000","last_undegraded":"2023-06-10T21:21:14.722483+0000","last_fullsized":"2023-06-10T21:21:14.722483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593365+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:21:14.593365+0000","last_peered":"2023-06-10T21:21:14.593365+0000","last_clean":"2023-06-10T21:21:14.593365+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:21:14.593365+0000","last_undegraded":"2023-06-10T21:21:14.593365+0000","last_fullsized":"2023-06-10T21:21:14.593365+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":89,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678875+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:21:14.678875+0000","last_peered":"2023-06-10T21:21:14.678875+0000","last_clean":"2023-06-10T21:21:14.678875+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:21:14.678875+0000","last_undegraded":"2023-06-10T21:21:14.678875+0000","last_fullsized":"2023-06-10T21:21:14.678875+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.723373+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:21:14.723373+0000","last_peered":"2023-06-10T21:21:14.723373+0000","last_clean":"2023-06-10T21:21:14.723373+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:21:14.723373+0000","last_undegraded":"2023-06-10T21:21:14.723373+0000","last_fullsized":"2023-06-10T21:21:14.723373+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675329+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:21:14.675329+0000","last_peered":"2023-06-10T21:21:14.675329+0000","last_clean":"2023-06-10T21:21:14.675329+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:21:14.675329+0000","last_undegraded":"2023-06-10T21:21:14.675329+0000","last_fullsized":"2023-06-10T21:21:14.675329+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":73,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676178+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:21:14.676178+0000","last_peered":"2023-06-10T21:21:14.676178+0000","last_clean":"2023-06-10T21:21:14.676178+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:21:14.676178+0000","last_undegraded":"2023-06-10T21:21:14.676178+0000","last_fullsized":"2023-06-10T21:21:14.676178+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":105,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676764+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:21:14.676764+0000","last_peered":"2023-06-10T21:21:14.676764+0000","last_clean":"2023-06-10T21:21:14.676764+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:21:14.676764+0000","last_undegraded":"2023-06-10T21:21:14.676764+0000","last_fullsized":"2023-06-10T21:21:14.676764+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675058+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:21:14.675058+0000","last_peered":"2023-06-10T21:21:14.675058+0000","last_clean":"2023-06-10T21:21:14.675058+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:21:14.675058+0000","last_undegraded":"2023-06-10T21:21:14.675058+0000","last_fullsized":"2023-06-10T21:21:14.675058+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593688+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:21:14.593688+0000","last_peered":"2023-06-10T21:21:14.593688+0000","last_clean":"2023-06-10T21:21:14.593688+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:21:14.593688+0000","last_undegraded":"2023-06-10T21:21:14.593688+0000","last_fullsized":"2023-06-10T21:21:14.593688+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593348+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:21:14.593348+0000","last_peered":"2023-06-10T21:21:14.593348+0000","last_clean":"2023-06-10T21:21:14.593348+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:21:14.593348+0000","last_undegraded":"2023-06-10T21:21:14.593348+0000","last_fullsized":"2023-06-10T21:21:14.593348+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675150+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:21:14.675150+0000","last_peered":"2023-06-10T21:21:14.675150+0000","last_clean":"2023-06-10T21:21:14.675150+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:21:14.675150+0000","last_undegraded":"2023-06-10T21:21:14.675150+0000","last_fullsized":"2023-06-10T21:21:14.675150+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.722376+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:21:14.722376+0000","last_peered":"2023-06-10T21:21:14.722376+0000","last_clean":"2023-06-10T21:21:14.722376+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:21:14.722376+0000","last_undegraded":"2023-06-10T21:21:14.722376+0000","last_fullsized":"2023-06-10T21:21:14.722376+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":89,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678432+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:21:14.678432+0000","last_peered":"2023-06-10T21:21:14.678432+0000","last_clean":"2023-06-10T21:21:14.678432+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:21:14.678432+0000","last_undegraded":"2023-06-10T21:21:14.678432+0000","last_fullsized":"2023-06-10T21:21:14.678432+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675605+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:21:14.675605+0000","last_peered":"2023-06-10T21:21:14.675605+0000","last_clean":"2023-06-10T21:21:14.675605+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:21:14.675605+0000","last_undegraded":"2023-06-10T21:21:14.675605+0000","last_fullsized":"2023-06-10T21:21:14.675605+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675590+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:21:14.675590+0000","last_peered":"2023-06-10T21:21:14.675590+0000","last_clean":"2023-06-10T21:21:14.675590+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:21:14.675590+0000","last_undegraded":"2023-06-10T21:21:14.675590+0000","last_fullsized":"2023-06-10T21:21:14.675590+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678593+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:21:14.678593+0000","last_peered":"2023-06-10T21:21:14.678593+0000","last_clean":"2023-06-10T21:21:14.678593+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:21:14.678593+0000","last_undegraded":"2023-06-10T21:21:14.678593+0000","last_fullsized":"2023-06-10T21:21:14.678593+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.592798+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:21:14.592798+0000","last_peered":"2023-06-10T21:21:14.592798+0000","last_clean":"2023-06-10T21:21:14.592798+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:21:14.592798+0000","last_undegraded":"2023-06-10T21:21:14.592798+0000","last_fullsized":"2023-06-10T21:21:14.592798+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.722374+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:21:14.722374+0000","last_peered":"2023-06-10T21:21:14.722374+0000","last_clean":"2023-06-10T21:21:14.722374+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:21:14.722374+0000","last_undegraded":"2023-06-10T21:21:14.722374+0000","last_fullsized":"2023-06-10T21:21:14.722374+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"135'3360","reported_seq":3376,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:22.704678+0000","last_change":"2023-06-10T21:20:45.268089+0000","last_active":"2023-06-10T21:21:22.704678+0000","last_peered":"2023-06-10T21:21:22.704678+0000","last_clean":"2023-06-10T21:21:22.704678+0000","last_became_active":"2023-06-10T21:20:45.267772+0000","last_became_peered":"2023-06-10T21:20:45.267772+0000","last_unstale":"2023-06-10T21:21:22.704678+0000","last_undegraded":"2023-06-10T21:21:22.704678+0000","last_fullsized":"2023-06-10T21:21:22.704678+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3360,"log_dups_size":0,"ondisk_log_size":3360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:24:07.609015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220200960,"num_objects":3360,"num_object_clones":0,"num_object_copies":10080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3360,"num_write_kb":215040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":103,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676783+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:21:14.676783+0000","last_peered":"2023-06-10T21:21:14.676783+0000","last_clean":"2023-06-10T21:21:14.676783+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:21:14.676783+0000","last_undegraded":"2023-06-10T21:21:14.676783+0000","last_fullsized":"2023-06-10T21:21:14.676783+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":86,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675532+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:21:14.675532+0000","last_peered":"2023-06-10T21:21:14.675532+0000","last_clean":"2023-06-10T21:21:14.675532+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:21:14.675532+0000","last_undegraded":"2023-06-10T21:21:14.675532+0000","last_fullsized":"2023-06-10T21:21:14.675532+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"135'3553","reported_seq":3570,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:25.550112+0000","last_change":"2023-06-10T21:20:45.293144+0000","last_active":"2023-06-10T21:21:25.550112+0000","last_peered":"2023-06-10T21:21:25.550112+0000","last_clean":"2023-06-10T21:21:25.550112+0000","last_became_active":"2023-06-10T21:20:45.292848+0000","last_became_peered":"2023-06-10T21:20:45.292848+0000","last_unstale":"2023-06-10T21:21:25.550112+0000","last_undegraded":"2023-06-10T21:21:25.550112+0000","last_fullsized":"2023-06-10T21:21:25.550112+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3553,"log_dups_size":0,"ondisk_log_size":3553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:53:38.966995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232849408,"num_objects":3553,"num_object_clones":0,"num_object_copies":10659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3553,"num_write_kb":227392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.677574+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:21:14.677574+0000","last_peered":"2023-06-10T21:21:14.677574+0000","last_clean":"2023-06-10T21:21:14.677574+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:21:14.677574+0000","last_undegraded":"2023-06-10T21:21:14.677574+0000","last_fullsized":"2023-06-10T21:21:14.677574+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593396+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:21:14.593396+0000","last_peered":"2023-06-10T21:21:14.593396+0000","last_clean":"2023-06-10T21:21:14.593396+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:21:14.593396+0000","last_undegraded":"2023-06-10T21:21:14.593396+0000","last_fullsized":"2023-06-10T21:21:14.593396+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"135'3446","reported_seq":3462,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.463591+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:21:24.463591+0000","last_peered":"2023-06-10T21:21:24.463591+0000","last_clean":"2023-06-10T21:21:24.463591+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:21:24.463591+0000","last_undegraded":"2023-06-10T21:21:24.463591+0000","last_fullsized":"2023-06-10T21:21:24.463591+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3446,"log_dups_size":0,"ondisk_log_size":3446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225837056,"num_objects":3446,"num_object_clones":0,"num_object_copies":10338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3446,"num_write_kb":220544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":88,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676615+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:21:14.676615+0000","last_peered":"2023-06-10T21:21:14.676615+0000","last_clean":"2023-06-10T21:21:14.676615+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:21:14.676615+0000","last_undegraded":"2023-06-10T21:21:14.676615+0000","last_fullsized":"2023-06-10T21:21:14.676615+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593006+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:21:14.593006+0000","last_peered":"2023-06-10T21:21:14.593006+0000","last_clean":"2023-06-10T21:21:14.593006+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:21:14.593006+0000","last_undegraded":"2023-06-10T21:21:14.593006+0000","last_fullsized":"2023-06-10T21:21:14.593006+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"135'3396","reported_seq":3412,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.457188+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:21:24.457188+0000","last_peered":"2023-06-10T21:21:24.457188+0000","last_clean":"2023-06-10T21:21:24.457188+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:21:24.457188+0000","last_undegraded":"2023-06-10T21:21:24.457188+0000","last_fullsized":"2023-06-10T21:21:24.457188+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3396,"log_dups_size":0,"ondisk_log_size":3396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222560256,"num_objects":3396,"num_object_clones":0,"num_object_copies":10188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3396,"num_write_kb":217344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675392+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:21:14.675392+0000","last_peered":"2023-06-10T21:21:14.675392+0000","last_clean":"2023-06-10T21:21:14.675392+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:21:14.675392+0000","last_undegraded":"2023-06-10T21:21:14.675392+0000","last_fullsized":"2023-06-10T21:21:14.675392+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.679114+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:21:14.679114+0000","last_peered":"2023-06-10T21:21:14.679114+0000","last_clean":"2023-06-10T21:21:14.679114+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:21:14.679114+0000","last_undegraded":"2023-06-10T21:21:14.679114+0000","last_fullsized":"2023-06-10T21:21:14.679114+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"135'3593","reported_seq":3610,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:26.523840+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:21:26.523840+0000","last_peered":"2023-06-10T21:21:26.523840+0000","last_clean":"2023-06-10T21:21:26.523840+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:21:26.523840+0000","last_undegraded":"2023-06-10T21:21:26.523840+0000","last_fullsized":"2023-06-10T21:21:26.523840+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3593,"log_dups_size":0,"ondisk_log_size":3593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235470848,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3593,"num_write_kb":229952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":78,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676096+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:21:14.676096+0000","last_peered":"2023-06-10T21:21:14.676096+0000","last_clean":"2023-06-10T21:21:14.676096+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:21:14.676096+0000","last_undegraded":"2023-06-10T21:21:14.676096+0000","last_fullsized":"2023-06-10T21:21:14.676096+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593617+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:21:14.593617+0000","last_peered":"2023-06-10T21:21:14.593617+0000","last_clean":"2023-06-10T21:21:14.593617+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:21:14.593617+0000","last_undegraded":"2023-06-10T21:21:14.593617+0000","last_fullsized":"2023-06-10T21:21:14.593617+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"135'3482","reported_seq":3499,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:25.556463+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:21:25.556463+0000","last_peered":"2023-06-10T21:21:25.556463+0000","last_clean":"2023-06-10T21:21:25.556463+0000","last_became_active":"2023-06-10T21:20:45.291689+0000","last_became_peered":"2023-06-10T21:20:45.291689+0000","last_unstale":"2023-06-10T21:21:25.556463+0000","last_undegraded":"2023-06-10T21:21:25.556463+0000","last_fullsized":"2023-06-10T21:21:25.556463+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3482,"log_dups_size":0,"ondisk_log_size":3482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:11:53.302103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228196352,"num_objects":3482,"num_object_clones":0,"num_object_copies":10446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3482,"num_write_kb":222848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.592914+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:21:14.592914+0000","last_peered":"2023-06-10T21:21:14.592914+0000","last_clean":"2023-06-10T21:21:14.592914+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:21:14.592914+0000","last_undegraded":"2023-06-10T21:21:14.592914+0000","last_fullsized":"2023-06-10T21:21:14.592914+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593543+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:21:14.593543+0000","last_peered":"2023-06-10T21:21:14.593543+0000","last_clean":"2023-06-10T21:21:14.593543+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:21:14.593543+0000","last_undegraded":"2023-06-10T21:21:14.593543+0000","last_fullsized":"2023-06-10T21:21:14.593543+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"135'3488","reported_seq":3504,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.231322+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:21:24.231322+0000","last_peered":"2023-06-10T21:21:24.231322+0000","last_clean":"2023-06-10T21:21:24.231322+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:21:24.231322+0000","last_undegraded":"2023-06-10T21:21:24.231322+0000","last_fullsized":"2023-06-10T21:21:24.231322+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3488,"log_dups_size":0,"ondisk_log_size":3488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589568,"num_objects":3488,"num_object_clones":0,"num_object_copies":10464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3488,"num_write_kb":223232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593170+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:21:14.593170+0000","last_peered":"2023-06-10T21:21:14.593170+0000","last_clean":"2023-06-10T21:21:14.593170+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:21:14.593170+0000","last_undegraded":"2023-06-10T21:21:14.593170+0000","last_fullsized":"2023-06-10T21:21:14.593170+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593731+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:21:14.593731+0000","last_peered":"2023-06-10T21:21:14.593731+0000","last_clean":"2023-06-10T21:21:14.593731+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:21:14.593731+0000","last_undegraded":"2023-06-10T21:21:14.593731+0000","last_fullsized":"2023-06-10T21:21:14.593731+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"135'3619","reported_seq":3636,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:26.575056+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:21:26.575056+0000","last_peered":"2023-06-10T21:21:26.575056+0000","last_clean":"2023-06-10T21:21:26.575056+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:21:26.575056+0000","last_undegraded":"2023-06-10T21:21:26.575056+0000","last_fullsized":"2023-06-10T21:21:26.575056+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3619,"log_dups_size":0,"ondisk_log_size":3619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237174784,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3619,"num_write_kb":231616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":95,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.722707+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:21:14.722707+0000","last_peered":"2023-06-10T21:21:14.722707+0000","last_clean":"2023-06-10T21:21:14.722707+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:21:14.722707+0000","last_undegraded":"2023-06-10T21:21:14.722707+0000","last_fullsized":"2023-06-10T21:21:14.722707+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:21.687883+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:21:21.687883+0000","last_peered":"2023-06-10T21:21:21.687883+0000","last_clean":"2023-06-10T21:21:21.687883+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:21:21.687883+0000","last_undegraded":"2023-06-10T21:21:21.687883+0000","last_fullsized":"2023-06-10T21:21:21.687883+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"135'3431","reported_seq":3447,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.443173+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:21:24.443173+0000","last_peered":"2023-06-10T21:21:24.443173+0000","last_clean":"2023-06-10T21:21:24.443173+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:21:24.443173+0000","last_undegraded":"2023-06-10T21:21:24.443173+0000","last_fullsized":"2023-06-10T21:21:24.443173+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3431,"log_dups_size":0,"ondisk_log_size":3431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224854016,"num_objects":3431,"num_object_clones":0,"num_object_copies":10293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3431,"num_write_kb":219584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":103,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674566+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:21:14.674566+0000","last_peered":"2023-06-10T21:21:14.674566+0000","last_clean":"2023-06-10T21:21:14.674566+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:21:14.674566+0000","last_undegraded":"2023-06-10T21:21:14.674566+0000","last_fullsized":"2023-06-10T21:21:14.674566+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675684+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:21:14.675684+0000","last_peered":"2023-06-10T21:21:14.675684+0000","last_clean":"2023-06-10T21:21:14.675684+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:21:14.675684+0000","last_undegraded":"2023-06-10T21:21:14.675684+0000","last_fullsized":"2023-06-10T21:21:14.675684+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.677616+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:21:14.677616+0000","last_peered":"2023-06-10T21:21:14.677616+0000","last_clean":"2023-06-10T21:21:14.677616+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:21:14.677616+0000","last_undegraded":"2023-06-10T21:21:14.677616+0000","last_fullsized":"2023-06-10T21:21:14.677616+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"135'3325","reported_seq":3342,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:22.693994+0000","last_change":"2023-06-10T21:20:45.269372+0000","last_active":"2023-06-10T21:21:22.693994+0000","last_peered":"2023-06-10T21:21:22.693994+0000","last_clean":"2023-06-10T21:21:22.693994+0000","last_became_active":"2023-06-10T21:20:45.269098+0000","last_became_peered":"2023-06-10T21:20:45.269098+0000","last_unstale":"2023-06-10T21:21:22.693994+0000","last_undegraded":"2023-06-10T21:21:22.693994+0000","last_fullsized":"2023-06-10T21:21:22.693994+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3325,"log_dups_size":0,"ondisk_log_size":3325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:23:51.802626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907200,"num_objects":3325,"num_object_clones":0,"num_object_copies":9975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3325,"num_write_kb":212800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676091+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:21:14.676091+0000","last_peered":"2023-06-10T21:21:14.676091+0000","last_clean":"2023-06-10T21:21:14.676091+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:21:14.676091+0000","last_undegraded":"2023-06-10T21:21:14.676091+0000","last_fullsized":"2023-06-10T21:21:14.676091+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593443+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:21:14.593443+0000","last_peered":"2023-06-10T21:21:14.593443+0000","last_clean":"2023-06-10T21:21:14.593443+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:21:14.593443+0000","last_undegraded":"2023-06-10T21:21:14.593443+0000","last_fullsized":"2023-06-10T21:21:14.593443+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":126,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674928+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:21:14.674928+0000","last_peered":"2023-06-10T21:21:14.674928+0000","last_clean":"2023-06-10T21:21:14.674928+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:21:14.674928+0000","last_undegraded":"2023-06-10T21:21:14.674928+0000","last_fullsized":"2023-06-10T21:21:14.674928+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"135'3535","reported_seq":3551,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.408590+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:21:24.408590+0000","last_peered":"2023-06-10T21:21:24.408590+0000","last_clean":"2023-06-10T21:21:24.408590+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:21:24.408590+0000","last_undegraded":"2023-06-10T21:21:24.408590+0000","last_fullsized":"2023-06-10T21:21:24.408590+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3535,"log_dups_size":0,"ondisk_log_size":3535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231669760,"num_objects":3535,"num_object_clones":0,"num_object_copies":10605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3535,"num_write_kb":226240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":101,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678033+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:21:14.678033+0000","last_peered":"2023-06-10T21:21:14.678033+0000","last_clean":"2023-06-10T21:21:14.678033+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:21:14.678033+0000","last_undegraded":"2023-06-10T21:21:14.678033+0000","last_fullsized":"2023-06-10T21:21:14.678033+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.673631+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:21:14.673631+0000","last_peered":"2023-06-10T21:21:14.673631+0000","last_clean":"2023-06-10T21:21:14.673631+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:21:14.673631+0000","last_undegraded":"2023-06-10T21:21:14.673631+0000","last_fullsized":"2023-06-10T21:21:14.673631+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":98,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674465+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:21:14.674465+0000","last_peered":"2023-06-10T21:21:14.674465+0000","last_clean":"2023-06-10T21:21:14.674465+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:21:14.674465+0000","last_undegraded":"2023-06-10T21:21:14.674465+0000","last_fullsized":"2023-06-10T21:21:14.674465+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"135'3601","reported_seq":3618,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:26.560595+0000","last_change":"2023-06-10T21:20:45.262983+0000","last_active":"2023-06-10T21:21:26.560595+0000","last_peered":"2023-06-10T21:21:26.560595+0000","last_clean":"2023-06-10T21:21:26.560595+0000","last_became_active":"2023-06-10T21:20:45.262641+0000","last_became_peered":"2023-06-10T21:20:45.262641+0000","last_unstale":"2023-06-10T21:21:26.560595+0000","last_undegraded":"2023-06-10T21:21:26.560595+0000","last_fullsized":"2023-06-10T21:21:26.560595+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3601,"log_dups_size":0,"ondisk_log_size":3601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:27:45.171697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235995136,"num_objects":3601,"num_object_clones":0,"num_object_copies":10803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3601,"num_write_kb":230464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675316+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:21:14.675316+0000","last_peered":"2023-06-10T21:21:14.675316+0000","last_clean":"2023-06-10T21:21:14.675316+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:21:14.675316+0000","last_undegraded":"2023-06-10T21:21:14.675316+0000","last_fullsized":"2023-06-10T21:21:14.675316+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":338,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674718+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:21:14.674718+0000","last_peered":"2023-06-10T21:21:14.674718+0000","last_clean":"2023-06-10T21:21:14.674718+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:21:14.674718+0000","last_undegraded":"2023-06-10T21:21:14.674718+0000","last_fullsized":"2023-06-10T21:21:14.674718+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":73,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675788+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:21:14.675788+0000","last_peered":"2023-06-10T21:21:14.675788+0000","last_clean":"2023-06-10T21:21:14.675788+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:21:14.675788+0000","last_undegraded":"2023-06-10T21:21:14.675788+0000","last_fullsized":"2023-06-10T21:21:14.675788+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593146+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:21:14.593146+0000","last_peered":"2023-06-10T21:21:14.593146+0000","last_clean":"2023-06-10T21:21:14.593146+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:21:14.593146+0000","last_undegraded":"2023-06-10T21:21:14.593146+0000","last_fullsized":"2023-06-10T21:21:14.593146+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"135'3372","reported_seq":3389,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:22.703548+0000","last_change":"2023-06-10T21:20:45.259115+0000","last_active":"2023-06-10T21:21:22.703548+0000","last_peered":"2023-06-10T21:21:22.703548+0000","last_clean":"2023-06-10T21:21:22.703548+0000","last_became_active":"2023-06-10T21:20:45.258813+0000","last_became_peered":"2023-06-10T21:20:45.258813+0000","last_unstale":"2023-06-10T21:21:22.703548+0000","last_undegraded":"2023-06-10T21:21:22.703548+0000","last_fullsized":"2023-06-10T21:21:22.703548+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3372,"log_dups_size":0,"ondisk_log_size":3372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:29.551738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220987392,"num_objects":3372,"num_object_clones":0,"num_object_copies":10116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3372,"num_write_kb":215808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":121,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674673+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:21:14.674673+0000","last_peered":"2023-06-10T21:21:14.674673+0000","last_clean":"2023-06-10T21:21:14.674673+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:21:14.674673+0000","last_undegraded":"2023-06-10T21:21:14.674673+0000","last_fullsized":"2023-06-10T21:21:14.674673+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":90,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678584+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:21:14.678584+0000","last_peered":"2023-06-10T21:21:14.678584+0000","last_clean":"2023-06-10T21:21:14.678584+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:21:14.678584+0000","last_undegraded":"2023-06-10T21:21:14.678584+0000","last_fullsized":"2023-06-10T21:21:14.678584+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678573+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:21:14.678573+0000","last_peered":"2023-06-10T21:21:14.678573+0000","last_clean":"2023-06-10T21:21:14.678573+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:21:14.678573+0000","last_undegraded":"2023-06-10T21:21:14.678573+0000","last_fullsized":"2023-06-10T21:21:14.678573+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"135'3561","reported_seq":3576,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.240395+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:21:24.240395+0000","last_peered":"2023-06-10T21:21:24.240395+0000","last_clean":"2023-06-10T21:21:24.240395+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:21:24.240395+0000","last_undegraded":"2023-06-10T21:21:24.240395+0000","last_fullsized":"2023-06-10T21:21:24.240395+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3561,"log_dups_size":0,"ondisk_log_size":3561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233373696,"num_objects":3561,"num_object_clones":0,"num_object_copies":10683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3561,"num_write_kb":227904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676686+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:21:14.676686+0000","last_peered":"2023-06-10T21:21:14.676686+0000","last_clean":"2023-06-10T21:21:14.676686+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:21:14.676686+0000","last_undegraded":"2023-06-10T21:21:14.676686+0000","last_fullsized":"2023-06-10T21:21:14.676686+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":111,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674906+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:21:14.674906+0000","last_peered":"2023-06-10T21:21:14.674906+0000","last_clean":"2023-06-10T21:21:14.674906+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:21:14.674906+0000","last_undegraded":"2023-06-10T21:21:14.674906+0000","last_fullsized":"2023-06-10T21:21:14.674906+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678926+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:21:14.678926+0000","last_peered":"2023-06-10T21:21:14.678926+0000","last_clean":"2023-06-10T21:21:14.678926+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:21:14.678926+0000","last_undegraded":"2023-06-10T21:21:14.678926+0000","last_fullsized":"2023-06-10T21:21:14.678926+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"135'3483","reported_seq":3499,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:24.197778+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:21:24.197778+0000","last_peered":"2023-06-10T21:21:24.197778+0000","last_clean":"2023-06-10T21:21:24.197778+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:21:24.197778+0000","last_undegraded":"2023-06-10T21:21:24.197778+0000","last_fullsized":"2023-06-10T21:21:24.197778+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3483,"log_dups_size":0,"ondisk_log_size":3483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228261888,"num_objects":3483,"num_object_clones":0,"num_object_copies":10449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3483,"num_write_kb":222912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":78,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.722483+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:21:14.722483+0000","last_peered":"2023-06-10T21:21:14.722483+0000","last_clean":"2023-06-10T21:21:14.722483+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:21:14.722483+0000","last_undegraded":"2023-06-10T21:21:14.722483+0000","last_fullsized":"2023-06-10T21:21:14.722483+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":96,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.674686+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:21:14.674686+0000","last_peered":"2023-06-10T21:21:14.674686+0000","last_clean":"2023-06-10T21:21:14.674686+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:21:14.674686+0000","last_undegraded":"2023-06-10T21:21:14.674686+0000","last_fullsized":"2023-06-10T21:21:14.674686+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:21.684878+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:21:21.684878+0000","last_peered":"2023-06-10T21:21:21.684878+0000","last_clean":"2023-06-10T21:21:21.684878+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:21:21.684878+0000","last_undegraded":"2023-06-10T21:21:21.684878+0000","last_fullsized":"2023-06-10T21:21:21.684878+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"135'3579","reported_seq":3596,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:26.517397+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:21:26.517397+0000","last_peered":"2023-06-10T21:21:26.517397+0000","last_clean":"2023-06-10T21:21:26.517397+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:21:26.517397+0000","last_undegraded":"2023-06-10T21:21:26.517397+0000","last_fullsized":"2023-06-10T21:21:26.517397+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":3579,"log_dups_size":0,"ondisk_log_size":3579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234553344,"num_objects":3579,"num_object_clones":0,"num_object_copies":10737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3579,"num_write_kb":229056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593293+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:21:14.593293+0000","last_peered":"2023-06-10T21:21:14.593293+0000","last_clean":"2023-06-10T21:21:14.593293+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:21:14.593293+0000","last_undegraded":"2023-06-10T21:21:14.593293+0000","last_fullsized":"2023-06-10T21:21:14.593293+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":93,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678478+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:21:14.678478+0000","last_peered":"2023-06-10T21:21:14.678478+0000","last_clean":"2023-06-10T21:21:14.678478+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:21:14.678478+0000","last_undegraded":"2023-06-10T21:21:14.678478+0000","last_fullsized":"2023-06-10T21:21:14.678478+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593263+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:21:14.593263+0000","last_peered":"2023-06-10T21:21:14.593263+0000","last_clean":"2023-06-10T21:21:14.593263+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:21:14.593263+0000","last_undegraded":"2023-06-10T21:21:14.593263+0000","last_fullsized":"2023-06-10T21:21:14.593263+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":78,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.679203+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:21:14.679203+0000","last_peered":"2023-06-10T21:21:14.679203+0000","last_clean":"2023-06-10T21:21:14.679203+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:21:14.679203+0000","last_undegraded":"2023-06-10T21:21:14.679203+0000","last_fullsized":"2023-06-10T21:21:14.679203+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":106,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.677727+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:21:14.677727+0000","last_peered":"2023-06-10T21:21:14.677727+0000","last_clean":"2023-06-10T21:21:14.677727+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:21:14.677727+0000","last_undegraded":"2023-06-10T21:21:14.677727+0000","last_fullsized":"2023-06-10T21:21:14.677727+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":104,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675543+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:21:14.675543+0000","last_peered":"2023-06-10T21:21:14.675543+0000","last_clean":"2023-06-10T21:21:14.675543+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:21:14.675543+0000","last_undegraded":"2023-06-10T21:21:14.675543+0000","last_fullsized":"2023-06-10T21:21:14.675543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593452+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:21:14.593452+0000","last_peered":"2023-06-10T21:21:14.593452+0000","last_clean":"2023-06-10T21:21:14.593452+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:21:14.593452+0000","last_undegraded":"2023-06-10T21:21:14.593452+0000","last_fullsized":"2023-06-10T21:21:14.593452+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":89,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.675771+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:21:14.675771+0000","last_peered":"2023-06-10T21:21:14.675771+0000","last_clean":"2023-06-10T21:21:14.675771+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:21:14.675771+0000","last_undegraded":"2023-06-10T21:21:14.675771+0000","last_fullsized":"2023-06-10T21:21:14.675771+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593329+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:21:14.593329+0000","last_peered":"2023-06-10T21:21:14.593329+0000","last_clean":"2023-06-10T21:21:14.593329+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:21:14.593329+0000","last_undegraded":"2023-06-10T21:21:14.593329+0000","last_fullsized":"2023-06-10T21:21:14.593329+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-06-10T21:21:21.684285+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:21:21.684285+0000","last_peered":"2023-06-10T21:21:21.684285+0000","last_clean":"2023-06-10T21:21:21.684285+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:21:21.684285+0000","last_undegraded":"2023-06-10T21:21:21.684285+0000","last_fullsized":"2023-06-10T21:21:21.684285+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":123,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.676082+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:21:14.676082+0000","last_peered":"2023-06-10T21:21:14.676082+0000","last_clean":"2023-06-10T21:21:14.676082+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:21:14.676082+0000","last_undegraded":"2023-06-10T21:21:14.676082+0000","last_fullsized":"2023-06-10T21:21:14.676082+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":94,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.678790+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:21:14.678790+0000","last_peered":"2023-06-10T21:21:14.678790+0000","last_clean":"2023-06-10T21:21:14.678790+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:21:14.678790+0000","last_undegraded":"2023-06-10T21:21:14.678790+0000","last_fullsized":"2023-06-10T21:21:14.678790+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":134,"state":"active+clean","last_fresh":"2023-06-10T21:21:14.593151+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:21:14.593151+0000","last_peered":"2023-06-10T21:21:14.593151+0000","last_clean":"2023-06-10T21:21:14.593151+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:21:14.593151+0000","last_undegraded":"2023-06-10T21:21:14.593151+0000","last_fullsized":"2023-06-10T21:21:14.593151+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3658481664,"num_objects":55824,"num_object_clones":0,"num_object_copies":167472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":55824,"num_write_kb":3572736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10988093440,"data_stored":10988093440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55824,"ondisk_log_size":55824,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430492,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303772,"kb_used_data":1966996,"kb_used_omap":0,"kb_used_meta":336768,"kb_avail":91441380,"statfs":{"total":95995035648,"available":93635973120,"internally_reserved":0,"allocated":2014203904,"data_stored":2013712009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":344850432},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:21:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":1.0109999999999999,"15min":1.3160000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.22800000000000001},"max":{"1min":1.673,"5min":10.914999999999999,"15min":10.914999999999999},"last":2.3290000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":1.6339999999999999,"15min":1.8520000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":1.5609999999999999,"5min":15.925000000000001,"15min":15.925000000000001},"last":2.375}]},{"osd":1,"last update":"Sat Jun 10 21:20:49 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.6899999999999999,"15min":1.482},"min":{"1min":0.436,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.69899999999999995,"5min":10.083,"15min":10.083},"last":2.5259999999999998},{"interface":"front","average":{"1min":0.498,"5min":1.3600000000000001,"15min":1.454},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":0.68700000000000006,"5min":6.4530000000000003,"15min":7},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.52900000000000003,"15min":0.52100000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":2.452},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.54800000000000004,"15min":0.53800000000000003},"min":{"1min":0.40000000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":2.4910000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":2.403}]},{"osd":4,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.94699999999999995,"15min":1.9730000000000001},"min":{"1min":0.39800000000000002,"5min":0.28100000000000003,"15min":0.22800000000000001},"max":{"1min":0.66800000000000004,"5min":5.1319999999999997,"15min":219.17699999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":1.3140000000000001,"15min":2.1469999999999998},"min":{"1min":0.439,"5min":0.29299999999999998,"15min":0.25700000000000001},"max":{"1min":0.70899999999999996,"5min":7.1929999999999996,"15min":219.13999999999999},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.464,"15min":2.573},"min":{"1min":0.40600000000000003,"5min":0.29199999999999998,"15min":0.254},"max":{"1min":0.81899999999999995,"5min":37.759,"15min":311.202},"last":2.2919999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":1.47,"15min":2.484},"min":{"1min":0.45700000000000002,"5min":0.33000000000000002,"15min":0.29699999999999999},"max":{"1min":0.79000000000000004,"5min":30.074999999999999,"15min":311.875},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.1819999999999999,"15min":1.111},"min":{"1min":0.48799999999999999,"5min":0.33300000000000002,"15min":0.28999999999999998},"max":{"1min":0.78300000000000003,"5min":6.8570000000000002,"15min":8.6630000000000003},"last":2.3490000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.97599999999999998,"15min":1.036},"min":{"1min":0.53300000000000003,"5min":0.34399999999999997,"15min":0.32400000000000001},"max":{"1min":0.83099999999999996,"5min":7.6900000000000004,"15min":8.5640000000000001},"last":0.56200000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626720,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":941516,"kb_used_data":647428,"kb_used_omap":0,"kb_used_meta":294080,"kb_avail":92803636,"statfs":{"total":95995035648,"available":95030923264,"internally_reserved":0,"allocated":662966272,"data_stored":662490300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":301137920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.1890000000000001,"15min":2.1110000000000002},"min":{"1min":0.28599999999999998,"5min":0.254,"15min":0.23599999999999999},"max":{"1min":0.94399999999999995,"5min":9.7929999999999993,"15min":119.276},"last":0.871},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.268,"15min":2.6840000000000002},"min":{"1min":0.33000000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.69699999999999995,"5min":10.574999999999999,"15min":119.22},"last":0.92900000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:20:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.5309999999999999,"15min":1.7370000000000001},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.28299999999999997},"max":{"1min":1.1599999999999999,"5min":14.897,"15min":14.897},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.744,"15min":1.786},"min":{"1min":0.371,"5min":0.251,"15min":0.251},"max":{"1min":0.88300000000000001,"5min":10.766999999999999,"15min":10.766999999999999},"last":5.1120000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.59399999999999997,"15min":0.57199999999999995},"min":{"1min":0.41199999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.0089999999999999,"5min":3.0089999999999999,"15min":3.0089999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:20:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.629},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":1.0660000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.95599999999999996,"15min":1.7909999999999999},"min":{"1min":0.34699999999999998,"5min":0.25600000000000001,"15min":0.254},"max":{"1min":0.66600000000000004,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.1890000000000001,"15min":1.994},"min":{"1min":0.379,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":0.71099999999999997,"5min":11.263999999999999,"15min":221.905},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:20:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.0149999999999999,"15min":2.2799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28199999999999997,"15min":0.25700000000000001},"max":{"1min":0.72099999999999997,"5min":6.6680000000000001,"15min":313.661},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.1080000000000001,"15min":2.3490000000000002},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":0.74399999999999999,"5min":9.75,"15min":314.38900000000001},"last":0.748}]},{"osd":7,"last update":"Sat Jun 10 21:20:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":1.165,"15min":1.133},"min":{"1min":0.44,"5min":0.318,"15min":0.318},"max":{"1min":0.78100000000000003,"5min":8.6460000000000008,"15min":9.9239999999999995},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.1120000000000001,"15min":1.1200000000000001},"min":{"1min":0.441,"5min":0.308,"15min":0.308},"max":{"1min":0.76700000000000002,"5min":8.0980000000000008,"15min":8.8949999999999996},"last":0.71699999999999997}]}]},{"osd":1,"up_from":105,"seq":450971566163,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2268000,"kb_used_data":2027416,"kb_used_omap":4,"kb_used_meta":240571,"kb_avail":91477152,"statfs":{"total":95995035648,"available":93672603648,"internally_reserved":0,"allocated":2076073984,"data_stored":2075521420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":246345614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.6539999999999999,"15min":1.7330000000000001},"min":{"1min":0.29499999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58699999999999997,"5min":10.898,"15min":10.898},"last":0.23999999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.052,"15min":2.2530000000000001},"min":{"1min":0.29599999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61499999999999999,"5min":16.488,"15min":16.488},"last":4.2750000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.53500000000000003,"15min":0.50900000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.1560000000000001,"5min":2.1560000000000001,"15min":2.1560000000000001},"last":9.6080000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.48599999999999999,"15min":0.47499999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":3.7559999999999998,"5min":3.7559999999999998,"15min":3.7559999999999998},"last":1.038}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":2.0099999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":9.6579999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.5640000000000001,"15min":1.8380000000000001},"min":{"1min":0.439,"5min":0.312,"15min":0.312},"max":{"1min":0.80500000000000005,"5min":12.94,"15min":12.94},"last":3.4180000000000001},{"interface":"front","average":{"1min":0.625,"5min":1.5229999999999999,"15min":2.5430000000000001},"min":{"1min":0.35599999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93100000000000005,"5min":9.3870000000000005,"15min":12.603},"last":0.66900000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.899,"15min":2.0699999999999998},"min":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.31900000000000001},"max":{"1min":0.97699999999999998,"5min":11.007999999999999,"15min":11.007999999999999},"last":4.3460000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.883,"15min":2.3700000000000001},"min":{"1min":0.45900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.96199999999999997,"5min":13.602,"15min":13.602},"last":1.802}]},{"osd":6,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.887,"15min":2.1070000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.91200000000000003,"5min":12.861000000000001,"15min":12.861000000000001},"last":4.4969999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.7949999999999999,"15min":2.3879999999999999},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.82599999999999996,"5min":12.970000000000001,"15min":12.970000000000001},"last":0.996}]},{"osd":7,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.8080000000000001,"15min":3.0499999999999998},"min":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.89200000000000002,"5min":13.025,"15min":13.025},"last":0.95699999999999996},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.8049999999999999,"15min":2.0590000000000002},"min":{"1min":0.44800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.85699999999999998,"5min":19.286999999999999,"15min":19.286999999999999},"last":0.70299999999999996}]}]},{"osd":0,"up_from":86,"seq":369367187585,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155644,"kb_used_data":848436,"kb_used_omap":5,"kb_used_meta":307194,"kb_avail":92589508,"statfs":{"total":95995035648,"available":94811656192,"internally_reserved":0,"allocated":868798464,"data_stored":868219785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":314567290},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":1.643,"15min":1.95},"min":{"1min":0.24399999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46600000000000003,"5min":16.780999999999999,"15min":16.780999999999999},"last":1.853},{"interface":"front","average":{"1min":0.36299999999999999,"5min":2.7570000000000001,"15min":2.8620000000000001},"min":{"1min":0.249,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.65400000000000003,"5min":21.998000000000001,"15min":21.998000000000001},"last":0.311}]},{"osd":2,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.376,"15min":0.35099999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.4470000000000001,"5min":3.4470000000000001,"15min":3.4470000000000001},"last":1.9530000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.40799999999999997,"15min":0.374},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":4.8239999999999998},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":1.9950000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.38100000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.611,"15min":2.1389999999999998},"min":{"1min":0.40600000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":2.73,"5min":10.948,"15min":10.948},"last":2.044},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.296,"15min":1.905},"min":{"1min":0.45300000000000001,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":3.512,"5min":8.8160000000000007,"15min":10.667},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":1.327,"15min":2.2149999999999999},"min":{"1min":0.44800000000000001,"5min":0.33900000000000002,"15min":0.32500000000000001},"max":{"1min":1.601,"5min":8.1539999999999999,"15min":10.693},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.76900000000000002,"5min":2.04,"15min":2.4529999999999998},"min":{"1min":0.41499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.9870000000000001,"5min":9.8390000000000004,"15min":17.460000000000001},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.4370000000000001,"15min":2.2879999999999998},"min":{"1min":0.501,"5min":0.378,"15min":0.35899999999999999},"max":{"1min":1.6379999999999999,"5min":11.848000000000001,"15min":11.848000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.093,"15min":2.5600000000000001},"min":{"1min":0.47899999999999998,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":2.1219999999999999,"5min":10.561,"15min":17.411000000000001},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.181,"15min":2.5659999999999998},"min":{"1min":0.47999999999999998,"5min":0.39200000000000002,"15min":0.36399999999999999},"max":{"1min":4.7869999999999999,"5min":11.224,"15min":17.486999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.865,"15min":2.0950000000000002},"min":{"1min":0.48899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.1080000000000001,"5min":9.3960000000000008,"15min":10.867000000000001},"last":8.0779999999999994}]}]},{"osd":2,"up_from":120,"seq":515396075551,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790960,"kb_used_data":642472,"kb_used_omap":7,"kb_used_meta":148472,"kb_avail":92954192,"statfs":{"total":95995035648,"available":95185092608,"internally_reserved":0,"allocated":657891328,"data_stored":657380828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":152035409},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.44900000000000001,"15min":0.40100000000000002},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.3879999999999999,"5min":7.3879999999999999,"15min":7.3879999999999999},"last":31.873000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.38200000000000001,"15min":0.37},"min":{"1min":0.26200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":31.282}]},{"osd":1,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.61199999999999999,"5min":0.622,"15min":0.622},"last":29.486000000000001},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.48599999999999999,"15min":0.437},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":7.4180000000000001,"5min":7.4180000000000001,"15min":7.4180000000000001},"last":28.013999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.247},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.26800000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.66500000000000004,"15min":0.65900000000000003},"min":{"1min":0.38900000000000001,"5min":0.371,"15min":0.371},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.65300000000000002,"15min":0.66100000000000003},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.86799999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.73599999999999999,"15min":0.70199999999999996},"min":{"1min":0.40999999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67400000000000004,"15min":0.67300000000000004},"min":{"1min":0.46899999999999997,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.499}]},{"osd":6,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.72099999999999997,"15min":0.70699999999999996},"min":{"1min":0.47999999999999998,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":2.0750000000000002,"5min":2.0750000000000002,"15min":2.0750000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.72399999999999998,"15min":0.72099999999999997},"min":{"1min":0.44400000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":2.0859999999999999,"5min":2.0859999999999999,"15min":2.0859999999999999},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:21:05 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.79700000000000004,"15min":0.755},"min":{"1min":0.47199999999999998,"5min":0.438,"15min":0.438},"max":{"1min":7.3550000000000004,"5min":7.3550000000000004,"15min":7.3550000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.74199999999999999,"15min":0.73199999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.137,"5min":2.137,"15min":2.137},"last":0.53700000000000003}]}]},{"osd":3,"up_from":125,"seq":536870912026,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1951532,"kb_used_data":1815140,"kb_used_omap":2,"kb_used_meta":136381,"kb_avail":91793620,"statfs":{"total":95995035648,"available":93996666880,"internally_reserved":0,"allocated":1858703360,"data_stored":1858196860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":139654197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":3.2970000000000002},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"last":6.1139999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":5.9409999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":6.4989999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":6.2050000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":3.5529999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":6.1580000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":3.609}]},{"osd":5,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":6.2460000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":3.5819999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":3.637},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":3.3370000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:20:29 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":4.024},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":3.484}]}]},{"osd":4,"up_from":30,"seq":128849019176,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994000,"kb_used_data":659016,"kb_used_omap":0,"kb_used_meta":334976,"kb_avail":92751152,"statfs":{"total":95995035648,"available":94977179648,"internally_reserved":0,"allocated":674832384,"data_stored":674351848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":343015424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.091,"15min":1.9119999999999999},"min":{"1min":0.33200000000000002,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":1.593,"5min":8.9269999999999996,"15min":10.01},"last":0.373},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.4970000000000001,"15min":2.1469999999999998},"min":{"1min":0.28000000000000003,"5min":0.253,"15min":0.23100000000000001},"max":{"1min":1.373,"5min":9.593,"15min":15.476000000000001},"last":1.0669999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:20:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.4159999999999999,"15min":1.728},"min":{"1min":0.371,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":0.86699999999999999,"5min":15.757,"15min":15.757},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.4550000000000001,"15min":2.2130000000000001},"min":{"1min":0.39200000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66600000000000004,"5min":15.125,"15min":15.125},"last":0.497}]},{"osd":2,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":2.6480000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.626,"15min":0.627},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.7450000000000001,"5min":2.6699999999999999,"15min":2.6699999999999999},"last":0.439}]},{"osd":3,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":1.4510000000000001},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":1.464}]},{"osd":5,"last update":"Sat Jun 10 21:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.75900000000000001,"15min":1.161},"min":{"1min":0.313,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":1.667,"5min":6.5010000000000003,"15min":92.653000000000006},"last":1.8120000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.0069999999999999,"15min":1.4890000000000001},"min":{"1min":0.34499999999999997,"5min":0.28699999999999998,"15min":0.253},"max":{"1min":1.4810000000000001,"5min":6.9539999999999997,"15min":93.281999999999996},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:20:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.92200000000000004,"15min":0.97299999999999998},"min":{"1min":0.378,"5min":0.28199999999999997,"15min":0.26700000000000002},"max":{"1min":0.69199999999999995,"5min":5.867,"15min":6.4340000000000002},"last":1.042},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.998,"15min":0.88700000000000001},"min":{"1min":0.49199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.81200000000000006,"5min":8.8420000000000005,"15min":8.8420000000000005},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:20:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.1579999999999999,"15min":1.278},"min":{"1min":0.42499999999999999,"5min":0.31,"15min":0.30099999999999999},"max":{"1min":0.85099999999999998,"5min":8.4459999999999997,"15min":19.989000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.92400000000000004,"15min":1.083},"min":{"1min":0.435,"5min":0.314,"15min":0.312},"max":{"1min":0.78000000000000003,"5min":4.7709999999999999,"15min":19.783000000000001},"last":1.0109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822948,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2411584,"kb_used_data":1995640,"kb_used_omap":0,"kb_used_meta":415936,"kb_avail":91333568,"statfs":{"total":95995035648,"available":93525573632,"internally_reserved":0,"allocated":2043535360,"data_stored":2043066970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":425918464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":1.167,"15min":1.093},"min":{"1min":0.33100000000000002,"5min":0.23799999999999999,"15min":0.23300000000000001},"max":{"1min":1.9810000000000001,"5min":8.0670000000000002,"15min":8.0670000000000002},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.647,"15min":1.9259999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":2.52,"5min":11.807,"15min":11.807},"last":1.5469999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.76,"15min":1.907},"min":{"1min":0.27800000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.66100000000000003,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":1.9850000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.9370000000000001,"15min":2.492},"min":{"1min":0.45200000000000001,"5min":0.28299999999999997,"15min":0.26800000000000002},"max":{"1min":0.81100000000000005,"5min":15.962999999999999,"15min":15.962999999999999},"last":1.8100000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:21:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.61699999999999999,"15min":0.60899999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.4969999999999999,"5min":2.4969999999999999,"15min":2.4969999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.625,"15min":0.61299999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.774,"5min":2.774,"15min":2.774},"last":0.48599999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":3.7170000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66000000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:20:53 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.0649999999999999,"15min":1.0069999999999999},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":0.68000000000000005,"5min":7.9939999999999998,"15min":7.9939999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.228,"15min":1.1439999999999999},"min":{"1min":0.35699999999999998,"5min":0.28000000000000003,"15min":0.255},"max":{"1min":0.80500000000000005,"5min":9.218,"15min":12.901},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:21:12 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.0720000000000001,"15min":1.0960000000000001},"min":{"1min":0.38200000000000001,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":1.831,"5min":8.1199999999999992,"15min":8.1199999999999992},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.99399999999999999,"15min":1.232},"min":{"1min":0.40200000000000002,"5min":0.31,"15min":0.31},"max":{"1min":2.4830000000000001,"5min":7.9370000000000003,"15min":8.1310000000000002},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:20:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.2849999999999999,"15min":1.2989999999999999},"min":{"1min":0.44,"5min":0.32400000000000001,"15min":0.29699999999999999},"max":{"1min":0.872,"5min":8.5109999999999992,"15min":8.5109999999999992},"last":0.442},{"interface":"front","average":{"1min":0.63600000000000001,"5min":1.25,"15min":1.111},"min":{"1min":0.437,"5min":0.318,"15min":0.318},"max":{"1min":0.753,"5min":8.0950000000000006,"15min":8.1809999999999992},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":877723648,"data_stored":877723648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2097872896,"data_stored":2097872896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":657653760,"data_stored":657653760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1889730560,"data_stored":1889730560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":686096384,"data_stored":686096384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2048524288,"data_stored":2048524288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":679411712,"data_stored":679411712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2051080192,"data_stored":2051080192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:21:32.067 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:21:32.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set device_health_metrics pg_num 11 2023-06-10T21:21:32.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:32.178 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:32.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:21:32.441 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:32.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:32.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:32.542 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:21:32.539872+0000 min lat: 0.00114967 max lat: 0.351579 avg lat: 0.00985823 2023-06-10T21:21:32.543 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:21:32.543 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 64936 64920 101.388 70.5625 0.00781952 0.00985823 2023-06-10T21:21:32.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:32.619 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:32.741 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:32 smithi064 ceph-mon[180297]: pgmap v540: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:32.741 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:32 smithi064 ceph-mon[186706]: pgmap v540: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:32.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:32.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:32.840 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:32.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:32.928 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:32.998 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:32] "GET /metrics HTTP/1.1" 200 223416 "" "Prometheus/2.33.4" 2023-06-10T21:21:33.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:32 smithi152 ceph-mon[156819]: pgmap v540: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-10T21:21:33.234 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:33.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:33.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:33.413 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:33.529 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:33.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:33.542 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 65930 65914 100.43 62.125 0.0181685 0.00995191 2023-06-10T21:21:33.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:33.727 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:33.886 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:33.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:33.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:34.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:34.179 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:21:34.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:34.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:34.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:34.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:34.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:34.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/372942521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.400 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/372942521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/372942521' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:21:34.499 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:34.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:34.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 67185 67169 99.9001 78.4375 0.0139704 0.010006 2023-06-10T21:21:34.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:34.635 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: pgmap v541: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:21:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:21:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[180297]: osdmap e136: 8 total, 8 up, 8 in 2023-06-10T21:21:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: pgmap v541: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:21:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:21:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:35 smithi064 ceph-mon[186706]: osdmap e136: 8 total, 8 up, 8 in 2023-06-10T21:21:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: pgmap v541: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:21:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: failed to encode map e136 with expected crc 2023-06-10T21:21:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:21:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:35 smithi152 ceph-mon[156819]: osdmap e136: 8 total, 8 up, 8 in 2023-06-10T21:21:35.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 67943 67927 98.679 47.375 0.0132943 0.0101287 2023-06-10T21:21:36.077 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:36.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:36.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:36.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:36.344 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:36.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:36.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:36.542 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:36.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 68947 68931 97.8627 62.75 0.00805641 0.0102133 2023-06-10T21:21:36.591 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:36.739 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:36.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:36.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:36.898 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:37.166 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:37 smithi064 ceph-mon[180297]: pgmap v543: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:21:37.229 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:21:37.229 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:21:37.256 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:37.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:37.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:37.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-10T21:21:37.517 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:37 smithi152 ceph-mon[156819]: pgmap v543: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:21:37.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 70074 70058 97.2533 70.4375 0.0144109 0.0102775 2023-06-10T21:21:37.561 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:37.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:37 smithi064 ceph-mon[186706]: pgmap v543: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:21:37.673 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:37.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:37.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:37.830 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:38.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:21:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1322892455' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1322892455' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.456 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1322892455' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.457 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-10T21:21:38.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 71083 71067 96.5102 63.0625 0.0161728 0.0103559 2023-06-10T21:21:39.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:39 smithi064 ceph-mon[186706]: pgmap v544: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-06-10T21:21:39.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:39 smithi064 ceph-mon[180297]: pgmap v544: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-06-10T21:21:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:39 smithi152 ceph-mon[156819]: pgmap v544: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-06-10T21:21:39.545 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 71950 71934 95.61 54.1875 0.0169949 0.0104545 2023-06-10T21:21:39.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:39.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:39.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:39.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:21:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-10T21:21:40.003 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:40.127 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:40.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:40.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:40.266 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:40.267 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:40.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:40.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:40.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:21:40.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:40.467 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:21:40.507 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:21:40.537 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:40.547 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 73017 73001 95.0048 66.6875 0.0182214 0.0105206 2023-06-10T21:21:40.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:21:40.592 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:21:40.631 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:21:40.632 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:40.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:40.671 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:21:40.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:21:40.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.718 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:21:40.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:40.807 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:41 smithi152 ceph-mon[156819]: pgmap v545: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:21:41.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/282953816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:41.547 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 74089 74073 94.4333 67 0.00477102 0.0105696 2023-06-10T21:21:41.633 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:41.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:41 smithi064 ceph-mon[186706]: pgmap v545: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:21:41.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/282953816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:41.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:41 smithi064 ceph-mon[180297]: pgmap v545: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:21:41.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/282953816' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:41.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:41.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:41.806 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:41.899 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:41.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:42.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:42.073 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:42.176 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:42.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:42.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:42.313 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:42.355 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:42.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:42.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:42.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:42.547 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 74897 74881 93.5549 50.5 0.00956764 0.0106841 2023-06-10T21:21:42.725 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:42.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:42.742 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:42.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:42.886 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:42.983 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:42.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:43.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:43.138 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:42] "GET /metrics HTTP/1.1" 200 223487 "" "Prometheus/2.33.4" 2023-06-10T21:21:43.142 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:43.223 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:43.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:43.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:43.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:43.384 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:43.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:43 smithi152 ceph-mon[156819]: pgmap v546: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-10T21:21:43.476 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:43.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:43.548 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 76031 76015 93.1097 70.875 0.0110313 0.0107357 2023-06-10T21:21:43.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:43.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:43 smithi064 ceph-mon[180297]: pgmap v546: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-10T21:21:43.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:43 smithi064 ceph-mon[186706]: pgmap v546: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-10T21:21:43.636 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:43.715 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:43.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:43.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:43.869 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:43.947 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:43.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:44.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:44.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:44.148 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:44.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:44.234 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:21:44.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:44.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:44.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3982149157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3982149157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.547 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 76969 76953 92.4467 58.625 0.0158519 0.0108119 2023-06-10T21:21:44.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:21:44.662 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:44.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3982149157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:21:44.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:44.803 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:44.882 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:21:44.882 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:21:44.883 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-06-10T21:21:44.883 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:21:44.883 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:21:44.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-06-10T21:21:45.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: pgmap v547: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:21:45.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:45.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:45.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: osdmap e137: 8 total, 8 up, 8 in 2023-06-10T21:21:45.216 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.217 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.218 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[186706]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:45.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:45.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:45.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: pgmap v547: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:21:45.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:45.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:45.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: osdmap e137: 8 total, 8 up, 8 in 2023-06-10T21:21:45.527 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:45 smithi064 ceph-mon[180297]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:45.549 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 77892 77876 91.788 57.6875 0.0101208 0.01089 2023-06-10T21:21:45.691 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:45.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: pgmap v547: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: osdmap e137: 8 total, 8 up, 8 in 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:45 smithi152 ceph-mon[156819]: failed to encode map e137 with expected crc 2023-06-10T21:21:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:45.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:45.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:45.841 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:45.899 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:45.906 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:45.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:46.003 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:21:46.003 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:21:46.003 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:21:46.003 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:21:46.004 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:21:46.005 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:46.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:46.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:46.549 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 78939 78923 91.3001 65.4375 0.0032428 0.010942 2023-06-10T21:21:47.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:47.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:47.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:47.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:47.316 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:47.401 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:47.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:47.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:47.421 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:47 smithi064 ceph-mon[180297]: pgmap v549: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-06-10T21:21:47.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:21:47.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:47 smithi064 ceph-mon[186706]: pgmap v549: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-06-10T21:21:47.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:21:47.510 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:47.540 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:47 smithi152 ceph-mon[156819]: pgmap v549: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2023-06-10T21:21:47.542 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:47.549 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 79844 79828 90.6687 56.5625 0.0181133 0.011024 2023-06-10T21:21:47.649 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:47.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:47.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:47.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:48.124 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:48.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:48.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:48.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:48.318 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:48.402 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:48.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:48.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:48.549 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 15 80665 80650 89.9672 51.375 0.00952307 0.0111103 2023-06-10T21:21:48.592 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:49.096 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:49.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:49.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:49.305 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:49.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[180297]: pgmap v550: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-06-10T21:21:49.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:49.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:49.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[186706]: pgmap v550: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-06-10T21:21:49.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:49.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:49.406 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:49.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:49.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:49 smithi152 ceph-mon[156819]: pgmap v550: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s 2023-06-10T21:21:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:21:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:21:49.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:49.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 81552 81536 89.36 55.375 0.00748464 0.0111862 2023-06-10T21:21:49.566 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:49.599 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:49.603 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:49.674 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:49.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:49.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:49.747 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:21:49.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:21:49.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:21:49.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:21:49.834 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:49.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:21:49.900 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:21:49.911 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:21:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:49.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:21:49.944 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:21:49.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:21:50.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:50.031 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:21:50.039 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:21:50.122 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:50.146 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/233465093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:50.160 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:21:50.160 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:21:50.160 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:21:50.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/233465093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:50.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/233465093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:21:50.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 82471 82455 88.8097 57.4375 0.0135124 0.0112551 2023-06-10T21:21:51.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:51 smithi064 ceph-mon[180297]: pgmap v551: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-06-10T21:21:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:51 smithi064 ceph-mon[186706]: pgmap v551: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-06-10T21:21:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:51 smithi152 ceph-mon[156819]: pgmap v551: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-06-10T21:21:51.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 83229 83213 88.1076 47.375 0.00622217 0.0113314 2023-06-10T21:21:52.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:52.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:52.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:52.443 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:52.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:21:52.548281+0000 min lat: 0.00114967 max lat: 0.351579 avg lat: 0.0113921 2023-06-10T21:21:52.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:21:52.551 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 84289 84273 87.7433 66.25 0.0151001 0.0113921 2023-06-10T21:21:52.580 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:52.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:52.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:52.799 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:21:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:21:52] "GET /metrics HTTP/1.1" 200 223494 "" "Prometheus/2.33.4" 2023-06-10T21:21:53.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:53.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:53.432 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:53 smithi152 ceph-mon[156819]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-06-10T21:21:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:21:53.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:53 smithi064 ceph-mon[180297]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-06-10T21:21:53.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:21:53.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:53 smithi064 ceph-mon[186706]: pgmap v552: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2023-06-10T21:21:53.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:21:53.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:53.550 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 85152 85136 87.1889 53.9375 0.0221534 0.0114636 2023-06-10T21:21:53.621 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:53.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:53.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:53.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:21:53.820 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:53.973 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:53.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:54.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:54.120 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:54.184 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:54.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:54.235 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:21:54.235 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:21:54.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:54.384 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:54.487 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:21:54.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:54.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:21:54.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 86012 85996 86.6439 53.75 0.016812 0.0115365 2023-06-10T21:21:54.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:54.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:54.734 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:54.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:54.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:21:54.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:55.211 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:21:55.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:21:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:55.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:21:55.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:55.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[180297]: pgmap v553: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:21:55.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2159923643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:55.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:55.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[186706]: pgmap v553: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:21:55.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2159923643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:55.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:55.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:21:55.522 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:55.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 86787 86771 86.0375 48.4375 0.0290433 0.0116176 2023-06-10T21:21:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:55 smithi152 ceph-mon[156819]: pgmap v553: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:21:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2159923643' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:21:56.091 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:56.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:56.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:56.295 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:56.477 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:56.511 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:56.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:56.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[180297]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:56.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[180297]: osdmap e138: 8 total, 8 up, 8 in 2023-06-10T21:21:56.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[180297]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:56.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[186706]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:56.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[186706]: osdmap e138: 8 total, 8 up, 8 in 2023-06-10T21:21:56.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:56 smithi064 ceph-mon[186706]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:56.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:56.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 87623 87607 85.5097 52.25 0.0146468 0.0116899 2023-06-10T21:21:56.586 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:56.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:56.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:56 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:21:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:56 smithi152 ceph-mon[156819]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:21:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:56 smithi152 ceph-mon[156819]: osdmap e138: 8 total, 8 up, 8 in 2023-06-10T21:21:56.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:56 smithi152 ceph-mon[156819]: failed to encode map e138 with expected crc 2023-06-10T21:21:56.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:56.827 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:56.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:56.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:56.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:21:57.021 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:57.190 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:21:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:57.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:21:57.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:57.369 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:57.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:57 smithi152 ceph-mon[156819]: pgmap v555: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-06-10T21:21:57.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 88487 88471 85.025 54 0.00989875 0.0117447 2023-06-10T21:21:57.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:57 smithi064 ceph-mon[180297]: pgmap v555: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-06-10T21:21:57.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:57 smithi064 ceph-mon[186706]: pgmap v555: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-06-10T21:21:57.801 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:57.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:57.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:21:57.945 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:58.242 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:21:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:58.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:21:58.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:21:58.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:21:58.367 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:21:58.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 0 1 2023-06-10T21:21:58.444 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:58.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 89255 89239 84.4642 48 0.0153764 0.0118351 2023-06-10T21:21:59.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:21:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:59.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:21:59.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:21:59.312 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:59.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:21:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:59.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:21:59.446 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:21:59 smithi064 ceph-mon[186706]: pgmap v556: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-06-10T21:21:59.446 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:21:59 smithi064 ceph-mon[180297]: pgmap v556: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-06-10T21:21:59.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:21:59.561 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 90023 90007 83.9152 48 0.0225094 0.0119125 2023-06-10T21:21:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:21:59.674 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:59.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:21:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:21:59 smithi152 ceph-mon[156819]: pgmap v556: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 895 op/s 2023-06-10T21:21:59.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:21:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:21:59.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:21:59.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:21:59.963 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:00.105 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:00.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:00.182 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:00.192 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:00.193 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:00.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:00.226 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:00.242 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:00.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:00.288 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:00.301 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:00.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:00.332 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:00.375 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:00.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:00.424 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:00.459 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:00.461 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.462 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:22:00.465 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:00.550 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:00.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:00.560 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 90776 90760 83.3734 47.0625 0.0268392 0.0119895 2023-06-10T21:22:00.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:00.746 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:00.901 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[186706]: pgmap v557: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-06-10T21:22:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/616038161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[180297]: pgmap v557: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-06-10T21:22:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/616038161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:00.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:00 smithi152 ceph-mon[156819]: pgmap v557: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 882 op/s 2023-06-10T21:22:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/616038161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:01.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:01.072 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:01.561 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 91573 91557 82.8852 49.8125 0.106247 0.0120532 2023-06-10T21:22:01.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:01.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:02.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:02.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:22:02.076 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:02.177 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:02.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:02.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:02.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:02.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:02.562 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 92493 92477 82.5226 57.5 0.00750534 0.0121132 2023-06-10T21:22:02.842 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:02.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:02 smithi064 ceph-mon[186706]: pgmap v558: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:22:02.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:02 smithi064 ceph-mon[180297]: pgmap v558: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:22:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:02 smithi152 ceph-mon[156819]: pgmap v558: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:22:02.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:02.988 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:03.090 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:03.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:02] "GET /metrics HTTP/1.1" 200 223494 "" "Prometheus/2.33.4" 2023-06-10T21:22:03.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:03.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:03.561 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 93280 93264 82.0533 49.1875 0.0149621 0.0121826 2023-06-10T21:22:03.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:03.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:03.959 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:03.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:04.024 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:04.090 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:04.154 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:04.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:04.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:04.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:04.396 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:04.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:04.564 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 94154 94138 81.6697 54.625 0.0303069 0.0122383 2023-06-10T21:22:04.711 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:04.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:04.725 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-06-10T21:22:04.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:04.871 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:04.945 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:04 smithi152 ceph-mon[156819]: pgmap v559: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-10T21:22:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:22:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:04.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:04.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:22:05.057 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:04 smithi064 ceph-mon[186706]: pgmap v559: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-10T21:22:05.057 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:22:05.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:04 smithi064 ceph-mon[180297]: pgmap v559: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-10T21:22:05.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:22:05.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:05.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:05.165 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:05.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:05.276 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:05.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:05.565 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 95015 94999 81.2874 53.8125 0.0181092 0.0122973 2023-06-10T21:22:05.732 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:22:06.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[186706]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:22:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[186706]: osdmap e139: 8 total, 8 up, 8 in 2023-06-10T21:22:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[186706]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[180297]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:22:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[180297]: osdmap e139: 8 total, 8 up, 8 in 2023-06-10T21:22:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[180297]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:05 smithi152 ceph-mon[156819]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4294790155' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:22:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:05 smithi152 ceph-mon[156819]: osdmap e139: 8 total, 8 up, 8 in 2023-06-10T21:22:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:05 smithi152 ceph-mon[156819]: failed to encode map e139 with expected crc 2023-06-10T21:22:06.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:06.565 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 95779 95763 80.8343 47.75 0.00312415 0.0123535 2023-06-10T21:22:07.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: pgmap v561: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-10T21:22:07.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:22:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[186706]: osdmap e140: 8 total, 8 up, 8 in 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: pgmap v561: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:07.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:06 smithi064 ceph-mon[180297]: osdmap e140: 8 total, 8 up, 8 in 2023-06-10T21:22:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: pgmap v561: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-10T21:22:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:22:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: failed to encode map e140 with expected crc 2023-06-10T21:22:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1138335468' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:06 smithi152 ceph-mon[156819]: osdmap e140: 8 total, 8 up, 8 in 2023-06-10T21:22:07.515 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:07.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:07.565 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 96607 96591 80.4464 51.75 0.011962 0.0124262 2023-06-10T21:22:07.625 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:07.672 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:07.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 1 1 2023-06-10T21:22:07.765 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:07.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:07.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:08.037 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:08.078 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:08.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:08.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:08.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:08.368 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:08.446 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:08.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:08.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:08.571 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 97386 97370 80.0267 48.6875 0.023996 0.0124909 2023-06-10T21:22:08.588 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:22:08.588 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:22:08.589 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:22:08.589 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:22:08.589 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:22:08.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:08.684 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:08.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[186706]: pgmap v563: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-06-10T21:22:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:08.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[180297]: pgmap v563: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-06-10T21:22:08.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:08.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:08.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:09.023 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:08 smithi152 ceph-mon[156819]: pgmap v563: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-06-10T21:22:09.023 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:09.024 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:09.032 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:09.137 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:09.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:09.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:09.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:09.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:09.567 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 98247 98231 79.6863 53.8125 0.00514978 0.012533 2023-06-10T21:22:09.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:22:10.453 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:10.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:10.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:22:10.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:10.573 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 99028 99012 79.2884 48.8125 0.0251587 0.0126068 2023-06-10T21:22:10.585 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:10.666 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:10.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:10.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:11.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:11.038 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[186706]: pgmap v564: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-06-10T21:22:11.039 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:22:11.039 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:22:11.039 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:11.042 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[180297]: pgmap v564: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-06-10T21:22:11.042 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:22:11.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:22:11.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:11.067 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:10 smithi152 ceph-mon[156819]: pgmap v564: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-06-10T21:22:11.067 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:22:11.067 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:22:11.067 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:11.076 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:11.168 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:11.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:11.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:11.182 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:11.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:11.328 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:11.356 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:11.394 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:11.415 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:11.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:11.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:11.434 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:11.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:11.511 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:11.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:11.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:11.570 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 99827 99811 78.917 49.9375 0.0180287 0.0126667 2023-06-10T21:22:11.597 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:11.604 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:11.627 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:11.652 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:22:11.681 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:11.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:11.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:11.866 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2747892850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:11.869 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:11.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2747892850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:11.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2747892850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:11.974 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:11.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:11.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:12.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:12.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:12.571 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:22:12.569413+0000 min lat: 0.00114967 max lat: 0.351579 avg lat: 0.0127276 2023-06-10T21:22:12.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:22:12.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 100611 100595 78.5415 49 0.00922001 0.0127276 2023-06-10T21:22:12.857 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:12.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:12.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:13.025 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:12 smithi064 ceph-mon[180297]: pgmap v565: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-06-10T21:22:13.025 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:12] "GET /metrics HTTP/1.1" 200 223501 "" "Prometheus/2.33.4" 2023-06-10T21:22:13.025 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:12 smithi064 ceph-mon[186706]: pgmap v565: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-06-10T21:22:13.030 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:12 smithi152 ceph-mon[156819]: pgmap v565: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s 2023-06-10T21:22:13.516 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:13.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:13.571 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 101462 101446 78.2286 53.1875 0.0139732 0.0127782 2023-06-10T21:22:13.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:13.716 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:14.547 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:14.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 102185 102169 77.8258 45.1875 0.0151939 0.0128441 2023-06-10T21:22:14.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:14.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:14.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:14 smithi064 ceph-mon[186706]: pgmap v566: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-06-10T21:22:14.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:14 smithi064 ceph-mon[180297]: pgmap v566: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-06-10T21:22:14.835 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:14.919 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:15.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:14 smithi152 ceph-mon[156819]: pgmap v566: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-06-10T21:22:15.257 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:15.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:15.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:15.469 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:15.571 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 103029 103013 77.5238 52.75 0.0164244 0.0128938 2023-06-10T21:22:15.718 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:15.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:15.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:15.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:15.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:16.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:16.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:16.314 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:16.368 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:16.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[180297]: pgmap v567: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-06-10T21:22:16.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2008404811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3996033122' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:16.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:16.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[186706]: pgmap v567: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-06-10T21:22:16.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2008404811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3996033122' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:16.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:16.564 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-06-10T21:22:16.566 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:22:16.573 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 103842 103826 77.2059 50.8125 0.0410888 0.0129463 2023-06-10T21:22:16.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:22:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:16 smithi152 ceph-mon[156819]: pgmap v567: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2023-06-10T21:22:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2008404811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:22:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3996033122' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:17.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:17.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:17.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:17.153 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:17.225 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:17.568 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:22:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:17.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[180297]: osdmap e141: 8 total, 8 up, 8 in 2023-06-10T21:22:17.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 104742 104726 76.9594 56.25 0.0136195 0.0129891 2023-06-10T21:22:17.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:17.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:22:17.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:17.893 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:17 smithi064 ceph-mon[186706]: osdmap e141: 8 total, 8 up, 8 in 2023-06-10T21:22:17.924 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:17.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: failed to encode map e141 with expected crc 2023-06-10T21:22:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:22:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:17 smithi152 ceph-mon[156819]: osdmap e141: 8 total, 8 up, 8 in 2023-06-10T21:22:18.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:18.106 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:18.196 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:18.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:18.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:22:18.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:18.350 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:18.432 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:18.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:18.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:18.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 105638 105622 76.7154 56 0.0360421 0.0130297 2023-06-10T21:22:18.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:18 smithi064 ceph-mon[180297]: pgmap v569: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:22:18.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:18.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:18 smithi064 ceph-mon[186706]: pgmap v569: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:22:18.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:18.603 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:18 smithi152 ceph-mon[156819]: pgmap v569: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:22:18.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:18.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:18.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:18.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:18.904 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:18.943 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:19.063 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:19.063 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:19.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:19.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:19.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:19.234 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:19.352 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:19.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:19.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:19.523 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:19.572 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 106466 106450 76.4286 51.75 0.0167019 0.0130798 2023-06-10T21:22:20.252 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:20.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:20.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:20.322 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:20.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:22:20.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:20.462 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:20.573 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 107379 107363 76.2085 57.0625 0.013036 0.0131166 2023-06-10T21:22:20.615 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:20.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:20.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:22:20.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:20 smithi152 ceph-mon[156819]: pgmap v570: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-06-10T21:22:20.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:20.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:20.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:20.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:20 smithi064 ceph-mon[180297]: pgmap v570: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-06-10T21:22:20.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:20 smithi064 ceph-mon[186706]: pgmap v570: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-06-10T21:22:20.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:20.921 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:20.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:21.067 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:21.229 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:21.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:21.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:21.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:21.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:21.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:21.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:21.575 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 108191 108175 75.9203 50.75 0.0167472 0.0131672 2023-06-10T21:22:21.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:21.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:22.005 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:22.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:22.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:22.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:22.207 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:22.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:22.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:22.340 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:22.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:22.344 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:22.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:22.415 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:22.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:22.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:22.490 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:22.506 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:22.517 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.518 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.519 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:22:22.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:22.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:22.578 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0085 2023-06-10T21:22:22.579 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 108985 2023-06-10T21:22:22.579 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:22:22.579 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:22:22.579 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 75.6768 2023-06-10T21:22:22.579 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 35.1841 2023-06-10T21:22:22.580 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 205.625 2023-06-10T21:22:22.580 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 45.1875 2023-06-10T21:22:22.580 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1210 2023-06-10T21:22:22.580 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 562.946 2023-06-10T21:22:22.580 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3290 2023-06-10T21:22:22.581 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 723 2023-06-10T21:22:22.583 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0132103 2023-06-10T21:22:22.583 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.017746 2023-06-10T21:22:22.584 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.351579 2023-06-10T21:22:22.584 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00114967 2023-06-10T21:22:22.584 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:22:22.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:22.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:22 smithi064 ceph-mon[186706]: pgmap v571: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:22:22.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3703392015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:22.885 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:22] "GET /metrics HTTP/1.1" 200 223515 "" "Prometheus/2.33.4" 2023-06-10T21:22:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:22 smithi064 ceph-mon[180297]: pgmap v571: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:22:22.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3703392015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:22 smithi152 ceph-mon[156819]: pgmap v571: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:22:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3703392015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:23.694 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:23.700 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:23.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:23.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:23.877 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:24.019 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:24.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:24.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:24.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:24.237 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:24.260 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:22:24.261 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:22:24.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:110208 9.e e141/127 9:7f400bf9:::benchmark_data_smithi064_247471_object1220:head v 141'6942, mlcod=141'6940)", 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.251996+0000", 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0040996629999999999, 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041211479999999998, 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:22:24.262 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:24.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:24.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.251996+0000", 2023-06-10T21:22:24.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:24.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.251996+0000", 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.251996+0000", 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3939999999999996e-06 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.252004+0000", 2023-06-10T21:22:24.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6415000000000001e-05 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.252020+0000", 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7980000000000002e-06 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.252026+0000", 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0262000000000001e-05 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.252076+0000", 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.372e-05 2023-06-10T21:22:24.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.252100+0000", 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103699 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:24.269 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:24.270 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:24.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.270 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:110211 9.4 e141/127 9:210db3ce:::benchmark_data_smithi064_247471_object1223:head v 141'6971, mlcod=141'6968)", 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.253150+0000", 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0029459389999999998, 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003053095, 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:24.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253150+0000", 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:24.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253150+0000", 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253149+0000", 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2099999999999998e-06 2023-06-10T21:22:24.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253156+0000", 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1411e-05 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:24.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253167+0000", 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3240000000000002e-06 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253172+0000", 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4388999999999998e-05 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:24.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253217+0000", 2023-06-10T21:22:24.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3008999999999999e-05 2023-06-10T21:22:24.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:24.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.253240+0000", 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9662999999999999e-05 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:24.277 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:24.278 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:22:24.278 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 2 2023-06-10T21:22:24.278 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:22:24.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:24.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:24.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:24.616 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:22:24.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:24.692 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:24.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:24.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:24.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:24.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:24 smithi064 ceph-mon[186706]: pgmap v572: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:22:24.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:22:24.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:24 smithi064 ceph-mon[180297]: pgmap v572: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:22:24.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:22:24.870 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:24.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:24.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:24.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:24 smithi152 ceph-mon[156819]: pgmap v572: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:22:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:22:25.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:25.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:25.169 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:22:25.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:22:25.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:22:25.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[180297]: osdmap e142: 8 total, 8 up, 8 in 2023-06-10T21:22:25.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[180297]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[180297]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[180297]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:22:25.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[186706]: osdmap e142: 8 total, 8 up, 8 in 2023-06-10T21:22:25.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[186706]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.787 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[186706]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:25 smithi064 ceph-mon[186706]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.807 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/133729630' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:22:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:25 smithi152 ceph-mon[156819]: osdmap e142: 8 total, 8 up, 8 in 2023-06-10T21:22:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:25 smithi152 ceph-mon[156819]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:25 smithi152 ceph-mon[156819]: failed to encode map e142 with expected crc 2023-06-10T21:22:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:25 smithi152 ceph-mon[156819]: failed to encode map e142 with expected crc 2023-06-10T21:22:26.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:26.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:26.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:26.115 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:26.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:22:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:26.344 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:26.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:26.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:26.496 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:26.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:26 smithi064 ceph-mon[180297]: pgmap v574: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 114 B/s rd, 50 MiB/s wr, 878 op/s 2023-06-10T21:22:26.672 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:26.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:26 smithi152 ceph-mon[156819]: pgmap v574: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 114 B/s rd, 50 MiB/s wr, 878 op/s 2023-06-10T21:22:26.881 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:26.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:26 smithi064 ceph-mon[186706]: pgmap v574: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 114 B/s rd, 50 MiB/s wr, 878 op/s 2023-06-10T21:22:26.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:27.056 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:27.236 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:27.335 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:27.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:27.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:27.512 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:27.657 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:22:27.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:27.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:27.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:27.965 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:22:27.965 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:22:27.965 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:22:27.965 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:22:27.966 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:22:28.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:28.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:28.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:28.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:28.542 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:28.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: pgmap v575: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 819 op/s 2023-06-10T21:22:28.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:28.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: osdmap e143: 8 total, 8 up, 8 in 2023-06-10T21:22:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[180297]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:22:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: pgmap v575: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 819 op/s 2023-06-10T21:22:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: osdmap e143: 8 total, 8 up, 8 in 2023-06-10T21:22:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:28 smithi064 ceph-mon[186706]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: pgmap v575: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 819 op/s 2023-06-10T21:22:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2806280807' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: osdmap e143: 8 total, 8 up, 8 in 2023-06-10T21:22:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: failed to encode map e143 with expected crc 2023-06-10T21:22:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:28 smithi152 ceph-mon[156819]: failed to encode map e143 with expected crc 2023-06-10T21:22:29.426 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:29.804 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:29.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:29.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:29.977 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:30.225 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:30.263 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:30.279 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:30.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:30.282 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:30.314 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:30.342 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:30.381 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:30.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:30.418 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:30.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:30.470 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.471 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.472 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.473 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:22:30.476 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:30.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:30 smithi064 ceph-mon[186706]: pgmap v577: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 812 op/s 2023-06-10T21:22:30.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3320206441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:30 smithi064 ceph-mon[180297]: pgmap v577: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 812 op/s 2023-06-10T21:22:30.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3320206441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:30 smithi152 ceph-mon[156819]: pgmap v577: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 812 op/s 2023-06-10T21:22:30.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3320206441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:31.385 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:31.386 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:31.452 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:31.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:31.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:31.639 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:31.673 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:22:31.706 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:31.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:31.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:31.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1986136569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:31.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:31 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:31.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1986136569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:31.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:31 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:31.839 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:31.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:22:31.925 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:31.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1986136569' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:31 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:22:32.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:32.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:32.109 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:32.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:32.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:32.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:22:32.272 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:32.586 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:32.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:32.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:32.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 4 1 2023-06-10T21:22:32.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:32 smithi152 ceph-mon[156819]: pgmap v578: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 12 MiB/s wr, 745 op/s 2023-06-10T21:22:32.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:32 smithi152 ceph-mon[156819]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:32 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:22:32.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:32 smithi152 ceph-mon[156819]: osdmap e144: 8 total, 8 up, 8 in 2023-06-10T21:22:32.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:32 smithi152 ceph-mon[156819]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.717 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:32.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:32.865 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:32.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[186706]: pgmap v578: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 12 MiB/s wr, 745 op/s 2023-06-10T21:22:32.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[186706]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:22:32.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[186706]: osdmap e144: 8 total, 8 up, 8 in 2023-06-10T21:22:32.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[186706]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.888 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:32] "GET /metrics HTTP/1.1" 200 223515 "" "Prometheus/2.33.4" 2023-06-10T21:22:32.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[180297]: pgmap v578: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 127 B/s rd, 12 MiB/s wr, 745 op/s 2023-06-10T21:22:32.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[180297]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:22:32.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[180297]: osdmap e144: 8 total, 8 up, 8 in 2023-06-10T21:22:32.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:32 smithi064 ceph-mon[180297]: failed to encode map e144 with expected crc 2023-06-10T21:22:32.899 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:32.938 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:32.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:32.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:33.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:33.132 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:33.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:33.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:33.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:33.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:33.451 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:33.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:22:33.776 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:33.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:33.813 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:33.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:33.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:33.914 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:33.963 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:34.228 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:34.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:34.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:34.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:34.392 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:34.511 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:34.562 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:34.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:34.622 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:34.668 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:34.735 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:34.839 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:34.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:34.852 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:34 smithi152 ceph-mon[156819]: pgmap v580: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.5 MiB/s wr, 716 op/s 2023-06-10T21:22:34.858 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:34.858 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:34.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:34.969 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:34 smithi064 ceph-mon[180297]: pgmap v580: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.5 MiB/s wr, 716 op/s 2023-06-10T21:22:34.970 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:34 smithi064 ceph-mon[186706]: pgmap v580: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 2.5 MiB/s wr, 716 op/s 2023-06-10T21:22:35.026 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:35.486 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:35.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:35.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:35.682 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:35.726 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:22:35.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:22:35.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3270544793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3270544793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.757 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.814 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:35.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:35.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3270544793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:22:35.960 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:36.077 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:36.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:36.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:36.225 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:36.352 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:36.355 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:36.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:36.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:36.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:36.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: pgmap v581: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-06-10T21:22:36.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:36.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:22:36.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: osdmap e145: 8 total, 8 up, 8 in 2023-06-10T21:22:36.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[180297]: failed to encode map e145 with expected crc 2023-06-10T21:22:36.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:36.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:37.024 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: pgmap v581: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-06-10T21:22:37.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:37.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:22:37.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: osdmap e145: 8 total, 8 up, 8 in 2023-06-10T21:22:37.025 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.026 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:36 smithi152 ceph-mon[156819]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.034 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: pgmap v581: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2023-06-10T21:22:37.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:37.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:22:37.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: osdmap e145: 8 total, 8 up, 8 in 2023-06-10T21:22:37.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.035 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.036 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.036 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.036 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.036 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.036 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:36 smithi064 ceph-mon[186706]: failed to encode map e145 with expected crc 2023-06-10T21:22:37.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:37.111 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:37.192 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:37.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:37.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:37.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:37.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:37.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:37.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:37.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:37.893 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:38.234 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:38.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:38.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:38.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:38.399 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:38.400 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:38.406 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:38.604 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:38.649 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:38.682 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:38.726 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-06-10T21:22:38.730 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:38.767 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:38.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:38.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:38.875 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:38.897 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.898 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:22:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:38.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:39.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:39.068 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[180297]: pgmap v583: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-06-10T21:22:39.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3661963697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:39.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1975154057' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.069 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[186706]: pgmap v583: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-06-10T21:22:39.069 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3661963697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:39.070 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1975154057' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.070 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:39.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:38 smithi152 ceph-mon[156819]: pgmap v583: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-06-10T21:22:39.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3661963697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:39.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1975154057' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:22:39.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:22:39.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:22:39.961 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:40.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:40.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:40.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:40.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:22:40.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: osdmap e146: 8 total, 8 up, 8 in 2023-06-10T21:22:40.073 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.074 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[186706]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:22:40.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: osdmap e146: 8 total, 8 up, 8 in 2023-06-10T21:22:40.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:39 smithi064 ceph-mon[180297]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:40.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:22:40.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: osdmap e146: 8 total, 8 up, 8 in 2023-06-10T21:22:40.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:39 smithi152 ceph-mon[156819]: failed to encode map e146 with expected crc 2023-06-10T21:22:40.201 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:40.283 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:40.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:40.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 5 1 2023-06-10T21:22:40.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:40.454 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:40.640 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:40.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:40.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:40.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:40.755 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:40 smithi152 ceph-mon[156819]: pgmap v585: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-06-10T21:22:40.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:40.794 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:40.838 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:40.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:40.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:40 smithi064 ceph-mon[180297]: pgmap v585: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-06-10T21:22:40.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:40 smithi064 ceph-mon[186706]: pgmap v585: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-06-10T21:22:40.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:40.974 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:41.116 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:41.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:41.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:41.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:41.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:41.311 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:41.471 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:41.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:41.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:41.687 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:41.903 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:41.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:42.006 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:22:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:22:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:22:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:22:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.44528.0:41977 8.3 8:cdb983ae:::benchmark_data_smithi064_208298_object41976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e102)", 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:14:18.435512+0000", 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 503.56523495800002, 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0127060510000001, 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:22:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.44528", 2023-06-10T21:22:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:22:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 41977 2023-06-10T21:22:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.435512+0000", 2023-06-10T21:22:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.435512+0000", 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.435510+0000", 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6636e-05 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.435547+0000", 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043790629999999999 2023-06-10T21:22:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.439926+0000", 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1235e-05 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.439937+0000", 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016086900000000001 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.440098+0000", 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8899999999999996e-06 2023-06-10T21:22:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:18.440107+0000", 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.007098584 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:19.447206+0000", 2023-06-10T21:22:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0010125659999999999 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:19.448218+0000", 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0127060510000001 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.44528.0:75209 8.3 8:ca70350a:::benchmark_data_smithi064_208298_object75208:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e106)", 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:14:42.421932+0000", 2023-06-10T21:22:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 479.57881544600002, 2023-06-10T21:22:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.158139037, 2023-06-10T21:22:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:22:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.44528", 2023-06-10T21:22:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:22:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75209 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.421932+0000", 2023-06-10T21:22:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.421932+0000", 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.421930+0000", 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000169827 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422100+0000", 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5118e-05 2023-06-10T21:22:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422115+0000", 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8399999999999995e-06 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422123+0000", 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026540599999999999 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422388+0000", 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3864e-05 2023-06-10T21:22:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422402+0000", 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1451015069999999 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.567503+0000", 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00082202600000000001 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.568325+0000", 2023-06-10T21:22:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011613653 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.579939+0000", 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131824 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.580071+0000", 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.158139037 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.44528.0:75221 8.3 8:c720e183:::benchmark_data_smithi064_208298_object75220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e106)", 2023-06-10T21:22:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:14:42.422139+0000", 2023-06-10T21:22:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 479.578607815, 2023-06-10T21:22:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.158602715, 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.44528", 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:22:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 75221 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422139+0000", 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422139+0000", 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422138+0000", 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7309999999999998e-05 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422196+0000", 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0699e-05 2023-06-10T21:22:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422206+0000", 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.789e-06 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422212+0000", 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041596999999999999 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422628+0000", 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4639999999999995e-06 2023-06-10T21:22:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:42.422638+0000", 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1457819579999999 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.568420+0000", 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000229729 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.568649+0000", 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000189495 2023-06-10T21:22:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,6", 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.568839+0000", 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0073502619999999998 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.576189+0000", 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00117103 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.577360+0000", 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002745749 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.580106+0000", 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2552000000000002e-05 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.580148+0000", 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000593751 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:14:43.580742+0000", 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.158602715 2023-06-10T21:22:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20063 9.f 9:f8ae3157:::benchmark_data_smithi064_247471_object20062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.981864+0000", 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.01888280599999, 2023-06-10T21:22:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29579981500000002, 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20063 2023-06-10T21:22:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981864+0000", 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981864+0000", 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981864+0000", 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9080999999999996e-05 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981953+0000", 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1130000000000007e-06 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981962+0000", 2023-06-10T21:22:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1139999999999999e-06 2023-06-10T21:22:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.981966+0000", 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9276000000000001e-05 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.982015+0000", 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013593599999999999 2023-06-10T21:22:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.982151+0000", 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111737 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:22:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.982263+0000", 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00099993900000000004 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983263+0000", 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021570949999999999 2023-06-10T21:22:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985420+0000", 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29207980900000002 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.277500+0000", 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011233199999999999 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.277612+0000", 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2191999999999998e-05 2023-06-10T21:22:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.277664+0000", 2023-06-10T21:22:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29579981500000002 2023-06-10T21:22:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20067 9.6 9:6b301878:::benchmark_data_smithi064_247471_object20066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.983938+0000", 2023-06-10T21:22:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.016808712, 2023-06-10T21:22:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29448421400000002, 2023-06-10T21:22:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20067 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983938+0000", 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983938+0000", 2023-06-10T21:22:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983937+0000", 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024132900000000001 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984179+0000", 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2130000000000001e-05 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984191+0000", 2023-06-10T21:22:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8919999999999999e-06 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984196+0000", 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3816999999999997e-05 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984240+0000", 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000194902 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984435+0000", 2023-06-10T21:22:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015248900000000001 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984587+0000", 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00091722600000000004 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985504+0000", 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00070814900000000004 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986212+0000", 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29211395800000001 2023-06-10T21:22:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278326+0000", 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5092e-05 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278381+0000", 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1254000000000002e-05 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278423+0000", 2023-06-10T21:22:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29448421400000002 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20068 9.f 9:f18e7a99:::benchmark_data_smithi064_247471_object20067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984211+0000", 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.01653646600001, 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29404230999999997, 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20068 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984211+0000", 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984211+0000", 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984210+0000", 2023-06-10T21:22:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9431000000000006e-05 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984289+0000", 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8469999999999996e-06 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984297+0000", 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6909999999999999e-06 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984301+0000", 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7617999999999997e-05 2023-06-10T21:22:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984349+0000", 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130798 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984479+0000", 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015517499999999999 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984635+0000", 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023679550000000001 2023-06-10T21:22:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987003+0000", 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8660000000000006e-06 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987010+0000", 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29116160600000002 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278172+0000", 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3909999999999998e-05 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278216+0000", 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7092999999999999e-05 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278253+0000", 2023-06-10T21:22:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29404230999999997 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20070 9.a 9:5e46163a:::benchmark_data_smithi064_247471_object20069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984310+0000", 2023-06-10T21:22:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.016436906, 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29453610099999999, 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20070 2023-06-10T21:22:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984310+0000", 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984310+0000", 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984310+0000", 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1272999999999997e-05 2023-06-10T21:22:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984341+0000", 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7959999999999996e-06 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984349+0000", 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8779999999999997e-06 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984354+0000", 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5567000000000004e-05 2023-06-10T21:22:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984419+0000", 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000128138 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984547+0000", 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016702899999999999 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984714+0000", 2023-06-10T21:22:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001787184 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986501+0000", 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019394309999999999 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988441+0000", 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29030747099999998 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278748+0000", 2023-06-10T21:22:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2793000000000001e-05 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278801+0000", 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5207999999999998e-05 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278846+0000", 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29453610099999999 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20073 9.a 9:5fdfb7f6:::benchmark_data_smithi064_247471_object20072:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984366+0000", 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.01638092500001, 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29467828699999998, 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20073 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984366+0000", 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984366+0000", 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984366+0000", 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9609000000000001e-05 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984395+0000", 2023-06-10T21:22:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8399999999999997e-06 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984402+0000", 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8099999999999997e-06 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984407+0000", 2023-06-10T21:22:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001142866 2023-06-10T21:22:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985550+0000", 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101313 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985651+0000", 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000150371 2023-06-10T21:22:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985801+0000", 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020402689999999999 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987842+0000", 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062850600000000003 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988470+0000", 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29047314299999999 2023-06-10T21:22:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278943+0000", 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9447000000000001e-05 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279013+0000", 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1717999999999997e-05 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279045+0000", 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29467828699999998 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20075 9.a 9:5bf8bcae:::benchmark_data_smithi064_247471_object20074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984412+0000", 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.016334888, 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29488785499999998, 2023-06-10T21:22:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20075 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984412+0000", 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984412+0000", 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984412+0000", 2023-06-10T21:22:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020284599999999999 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984615+0000", 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3159999999999997e-06 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984623+0000", 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3029999999999999e-06 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984628+0000", 2023-06-10T21:22:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018941209999999999 2023-06-10T21:22:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986522+0000", 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011772 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986640+0000", 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148572 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986789+0000", 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001059957 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987849+0000", 2023-06-10T21:22:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00067728099999999996 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988526+0000", 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29064099999999998 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279167+0000", 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0836999999999998e-05 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279258+0000", 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2488999999999999e-05 2023-06-10T21:22:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279300+0000", 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29488785499999998 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109295 9.9 e141/127 9:9b3a09da:::benchmark_data_smithi064_247471_object307:head v 141'6783, mlcod=141'6781)", 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.101948264000001, 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998, 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898798+0000", 2023-06-10T21:22:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2339999999999997e-06 2023-06-10T21:22:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898805+0000", 2023-06-10T21:22:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5519999999999993e-06 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898815+0000", 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2270000000000001e-06 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898820+0000", 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4044e-05 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898864+0000", 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9573e-05 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898884+0000", 2023-06-10T21:22:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35487087899999997 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253755+0000", 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3262000000000001e-05 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253788+0000", 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109297 9.5 e141/127 9:a2279825:::benchmark_data_smithi064_247471_object309:head v 141'6825, mlcod=141'6823)", 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.101531443999999, 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998, 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899215+0000", 2023-06-10T21:22:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8650000000000003e-06 2023-06-10T21:22:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899222+0000", 2023-06-10T21:22:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2023-06-10T21:22:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899232+0000", 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6350000000000003e-06 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899237+0000", 2023-06-10T21:22:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0815000000000001e-05 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899278+0000", 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1216000000000002e-05 2023-06-10T21:22:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899299+0000", 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.349632361 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248932+0000", 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100882 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249032+0000", 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109299 9.7 9:e2e5e039:::benchmark_data_smithi064_247471_object311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.098617488999999, 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998, 2023-06-10T21:22:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109299 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902128+0000", 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2023-06-10T21:22:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902135+0000", 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0420000000000008e-06 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902144+0000", 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6470000000000003e-06 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902150+0000", 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6499000000000003e-05 2023-06-10T21:22:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902197+0000", 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011322100000000001 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012462399999999999 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902434+0000", 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046768010000000004 2023-06-10T21:22:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907111+0000", 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6680000000000004e-06 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907118+0000", 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34198863299999999 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249107+0000", 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102996 2023-06-10T21:22:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249210+0000", 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4842000000000002e-05 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249254+0000", 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109302 9.a 9:593e78fe:::benchmark_data_smithi064_247471_object314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.098436038999999, 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999, 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.151 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109302 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.152 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:22:42.153 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5360000000000003e-06 2023-06-10T21:22:42.154 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902315+0000", 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.669e-06 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.155 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902322+0000", 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9060000000000004e-06 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.156 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902325+0000", 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074886199999999999 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903074+0000", 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010393800000000001 2023-06-10T21:22:42.157 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903178+0000", 2023-06-10T21:22:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8347000000000001e-05 2023-06-10T21:22:42.158 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903247+0000", 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.345750055 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248997+0000", 2023-06-10T21:22:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000337282 2023-06-10T21:22:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249334+0000", 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6821e-05 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249361+0000", 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7002000000000002e-05 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249408+0000", 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8299e-05 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249446+0000", 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999 2023-06-10T21:22:42.163 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.164 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109305 9.6 9:68c28d89:::benchmark_data_smithi064_247471_object317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.097694023999999, 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997, 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.165 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109305 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:22:42.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903052+0000", 2023-06-10T21:22:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:22:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903058+0000", 2023-06-10T21:22:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0450000000000006e-06 2023-06-10T21:22:42.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903067+0000", 2023-06-10T21:22:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8790000000000002e-06 2023-06-10T21:22:42.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903071+0000", 2023-06-10T21:22:42.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8942e-05 2023-06-10T21:22:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903120+0000", 2023-06-10T21:22:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100547 2023-06-10T21:22:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903221+0000", 2023-06-10T21:22:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109324 2023-06-10T21:22:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:22:42.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903330+0000", 2023-06-10T21:22:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34556609100000002 2023-06-10T21:22:42.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248896+0000", 2023-06-10T21:22:42.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042370840000000003 2023-06-10T21:22:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253133+0000", 2023-06-10T21:22:42.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3943000000000006e-05 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253217+0000", 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3278999999999997e-05 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253261+0000", 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3470000000000001e-05 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253314+0000", 2023-06-10T21:22:42.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109296 9.9 e141/127 9:9d3096da:::benchmark_data_smithi064_247471_object308:head v 141'6784, mlcod=141'6781)", 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:22:42.181 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.095983745000002, 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001, 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3180000000000003e-06 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904769+0000", 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0399999999999993e-06 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904777+0000", 2023-06-10T21:22:42.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6299999999999997e-06 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904782+0000", 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115028 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904897+0000", 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8039000000000001e-05 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904915+0000", 2023-06-10T21:22:42.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34887546699999999 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253790+0000", 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5639999999999999e-05 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253806+0000", 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109303 9.b e141/127 9:d16859ff:::benchmark_data_smithi064_247471_object315:head v 141'6884, mlcod=141'6882)", 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.095953864999998, 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885, 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:22:42.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.456e-06 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904797+0000", 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9980000000000003e-06 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904802+0000", 2023-06-10T21:22:42.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0400000000000001e-06 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904805+0000", 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34422613600000002 2023-06-10T21:22:42.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249031+0000", 2023-06-10T21:22:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7886000000000001e-05 2023-06-10T21:22:42.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249089+0000", 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078164589999999996 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256905+0000", 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2179999999999999e-05 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256947+0000", 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885 2023-06-10T21:22:42.197 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109304 9.5 e141/127 9:a38538a9:::benchmark_data_smithi064_247471_object316:head v 141'6826, mlcod=141'6823)", 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.095932564999998, 2023-06-10T21:22:42.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202, 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904814+0000", 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8880000000000002e-06 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904820+0000", 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2023-06-10T21:22:42.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904825+0000", 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.965e-06 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904828+0000", 2023-06-10T21:22:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002189528 2023-06-10T21:22:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907018+0000", 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3083e-05 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907031+0000", 2023-06-10T21:22:42.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34204052800000001 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249071+0000", 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7423000000000003e-05 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249119+0000", 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.206 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109300 9.9 e141/127 9:94ae4f7e:::benchmark_data_smithi064_247471_object312:head v 141'6785, mlcod=141'6781)", 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.092943531, 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001, 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.207 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:22:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907803+0000", 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.772e-06 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907812+0000", 2023-06-10T21:22:42.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34109469599999998 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248906+0000", 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1025999999999997e-05 2023-06-10T21:22:42.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248937+0000", 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016277199999999999 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249100+0000", 2023-06-10T21:22:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4561999999999997e-05 2023-06-10T21:22:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249155+0000", 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083045289999999997 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257459+0000", 2023-06-10T21:22:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8495000000000002e-05 2023-06-10T21:22:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257508+0000", 2023-06-10T21:22:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001 2023-06-10T21:22:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.217 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109307 9.f 9:f866cb96:::benchmark_data_smithi064_247471_object319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.092357320000001, 2023-06-10T21:22:42.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092, 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109307 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.219 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:22:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908388+0000", 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4800000000000004e-06 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908396+0000", 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34054057399999998 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248936+0000", 2023-06-10T21:22:42.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1486000000000001e-05 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248958+0000", 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032944700000000002 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249287+0000", 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101111 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249388+0000", 2023-06-10T21:22:42.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123366 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249512+0000", 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077532640000000002 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257265+0000", 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045560400000000002 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257721+0000", 2023-06-10T21:22:42.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003373905 2023-06-10T21:22:42.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:22:42.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261094+0000", 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6616000000000002e-05 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261141+0000", 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6924000000000001e-05 2023-06-10T21:22:42.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261178+0000", 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.231 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:109299 9.7 e141/127 ondisk, result = 0)", 2023-06-10T21:22:42.231 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:22:42.231 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 19.089393463, 2023-06-10T21:22:42.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999, 2023-06-10T21:22:42.232 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:22:42.232 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:22:42.232 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:22:42.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:22:42.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:22:42.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:22:42.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:22:42.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:22:42.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:22:42.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911353+0000", 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7229999999999998e-06 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:22:42.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911359+0000", 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33752210999999999 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248882+0000", 2023-06-10T21:22:42.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6460000000000001e-05 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248908+0000", 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015431399999999999 2023-06-10T21:22:42.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249062+0000", 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2586999999999999e-05 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249085+0000", 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017446900000000001 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:22:42.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:22:42.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249259+0000", 2023-06-10T21:22:42.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999 2023-06-10T21:22:42.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.240 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.241 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:22:42.241 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:22:42.241 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:22:42.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:42.324 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:42.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:42.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:42.543 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:42.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:22:42.586 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:42.779 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:42 smithi064 ceph-mon[180297]: pgmap v586: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-06-10T21:22:42.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:42 smithi152 ceph-mon[156819]: pgmap v586: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-06-10T21:22:42.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:22:42.880 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:42.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:42.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:42.993 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:43.043 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:42] "GET /metrics HTTP/1.1" 200 223513 "" "Prometheus/2.33.4" 2023-06-10T21:22:43.044 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:42 smithi064 ceph-mon[186706]: pgmap v586: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2023-06-10T21:22:43.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:43.409 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:43.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:43.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:43.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:43.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:43.833 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:22:43.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/380692000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:43.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:43 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:43.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/380692000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:43.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:43 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:43.877 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:43.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:43.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:43.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/380692000' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:43 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:22:44.018 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:44.069 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:44.169 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:44.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:44.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:44.310 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:44.822 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-06-10T21:22:44.829 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:44.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:44.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:45.003 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: pgmap v587: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:22:45.004 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:45.004 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:22:45.005 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.006 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:45.006 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: osdmap e147: 8 total, 8 up, 8 in 2023-06-10T21:22:45.006 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.006 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:22:45.009 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:45.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: pgmap v587: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:22:45.118 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:45.119 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:22:45.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:45.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: osdmap e147: 8 total, 8 up, 8 in 2023-06-10T21:22:45.120 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.121 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:22:45.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: pgmap v587: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-06-10T21:22:45.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:22:45.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:22:45.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:22:45.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: osdmap e147: 8 total, 8 up, 8 in 2023-06-10T21:22:45.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: failed to encode map e147 with expected crc 2023-06-10T21:22:45.122 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:22:45.142 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:45.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:45.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:45.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:45.335 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:45.380 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:22:45.382 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:22:45.383 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:22:45.383 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:22:45.401 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:45.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:45.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:45.570 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:45.702 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:45.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:45.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:45.819 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:45.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 6 1 2023-06-10T21:22:45.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: failed to encode map e148 with expected crc 2023-06-10T21:22:45.884 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:22:45.884 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: osdmap e148: 8 total, 8 up, 8 in 2023-06-10T21:22:45.885 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: failed to encode map e148 with expected crc 2023-06-10T21:22:45.885 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: failed to encode map e148 with expected crc 2023-06-10T21:22:45.885 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: failed to encode map e148 with expected crc 2023-06-10T21:22:45.885 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:45 smithi152 ceph-mon[156819]: failed to encode map e148 with expected crc 2023-06-10T21:22:45.891 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: osdmap e148: 8 total, 8 up, 8 in 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[186706]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3166899968' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: osdmap e148: 8 total, 8 up, 8 in 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:45 smithi064 ceph-mon[180297]: failed to encode map e148 with expected crc 2023-06-10T21:22:46.189 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:46.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:46.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:22:46.293 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:46.410 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:46.536 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:46.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:46.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:46.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:46.712 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:46.880 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:46.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:46.902 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:47.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:47.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:22:47.028 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:47.081 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:46 smithi064 ceph-mon[186706]: pgmap v590: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-06-10T21:22:47.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:46 smithi064 ceph-mon[180297]: pgmap v590: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-06-10T21:22:47.098 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:47.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:46 smithi152 ceph-mon[156819]: pgmap v590: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 752 op/s 2023-06-10T21:22:47.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:47.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:47.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:47.478 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:47.657 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:47.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:47.736 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:47.758 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:47.775 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:47.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:47.842 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1156399101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:47.888 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:47.939 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:47.964 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.965 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:22:47.966 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:22:48.034 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:48.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:48.140 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:48.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1156399101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:48.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1156399101' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:48.188 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:48.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:48.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:48.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:48.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:48.515 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:48.935 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:48.935 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:49.337 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[186706]: pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-10T21:22:49.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:49.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:49.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[180297]: pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-10T21:22:49.338 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:49.339 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:49 smithi152 ceph-mon[156819]: pgmap v591: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-10T21:22:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:22:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:22:49.576 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:49.600 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:49.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:49.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:49.775 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:49.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:22:49.879 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:49.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:49.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:50.048 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:50.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:50.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:50.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:50.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:50.410 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:50.437 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:50.642 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:50.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:50.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:50.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:50.851 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:51.170 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-06-10T21:22:51.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:51 smithi152 ceph-mon[156819]: pgmap v592: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-06-10T21:22:51.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2850338901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.465 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:51.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:51.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[186706]: pgmap v592: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-06-10T21:22:51.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2850338901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[180297]: pgmap v592: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-06-10T21:22:51.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2850338901' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:22:51.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:51.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:22:51.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:52.086 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:52.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:52.198 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:52.267 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:52.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:22:52.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[180297]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:22:52.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[180297]: osdmap e149: 8 total, 8 up, 8 in 2023-06-10T21:22:52.393 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[180297]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[186706]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:22:52.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[186706]: osdmap e149: 8 total, 8 up, 8 in 2023-06-10T21:22:52.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:52 smithi064 ceph-mon[186706]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.412 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:52.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:52.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:52.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:52.520 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:52.520 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:52.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:52 smithi152 ceph-mon[156819]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:22:52.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:52 smithi152 ceph-mon[156819]: osdmap e149: 8 total, 8 up, 8 in 2023-06-10T21:22:52.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:52 smithi152 ceph-mon[156819]: failed to encode map e149 with expected crc 2023-06-10T21:22:52.571 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:52.724 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:52.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:52.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:52.785 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:52.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:52.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:52.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:52.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:53.032 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:53.050 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:53.140 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:22:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:22:52] "GET /metrics HTTP/1.1" 200 223528 "" "Prometheus/2.33.4" 2023-06-10T21:22:53.157 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:53.245 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:22:53.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:22:53.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:53 smithi064 ceph-mon[180297]: pgmap v594: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-06-10T21:22:53.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:53.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:53 smithi064 ceph-mon[186706]: pgmap v594: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-06-10T21:22:53.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:53.444 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:53.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:53.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:53.652 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:53 smithi152 ceph-mon[156819]: pgmap v594: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2023-06-10T21:22:53.653 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:22:53.656 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:53.774 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:53.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:53.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:53.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:53.961 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:54.057 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:54.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:54.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:54.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:54.216 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:54.363 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:54.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:54.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:54.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:54.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: osdmap e150: 8 total, 8 up, 8 in 2023-06-10T21:22:54.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[186706]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:54.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:54.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: osdmap e150: 8 total, 8 up, 8 in 2023-06-10T21:22:54.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:54 smithi064 ceph-mon[180297]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:54.506 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/859905186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: osdmap e150: 8 total, 8 up, 8 in 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: failed to encode map e150 with expected crc 2023-06-10T21:22:54.546 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:54 smithi152 ceph-mon[156819]: failed to encode map e150 with expected crc 2023-06-10T21:22:55.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:55.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:55.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:55.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:55.292 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:55.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:55 smithi064 ceph-mon[180297]: pgmap v596: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-06-10T21:22:55.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:55 smithi064 ceph-mon[186706]: pgmap v596: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-06-10T21:22:55.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:55 smithi152 ceph-mon[156819]: pgmap v596: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2023-06-10T21:22:55.556 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:55.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:55.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:55.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:55.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:55.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:55.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:56.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:56.128 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:56.204 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:22:56.205 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:22:56.243 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:56.246 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:22:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:56.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:56.287 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:22:56.318 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:22:56.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:22:56.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:22:56.413 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:22:56.427 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:56.469 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:22:56.499 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:22:56.569 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:22:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:56.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:22:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:56 smithi064 ceph-mon[186706]: pgmap v597: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-06-10T21:22:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2650248492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:56 smithi064 ceph-mon[180297]: pgmap v597: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-06-10T21:22:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2650248492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:56.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:56.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:22:56.708 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:56.710 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.710 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.710 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.711 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:22:56.780 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:56.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:56.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:22:56.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:56.941 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:56.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:56 smithi152 ceph-mon[156819]: pgmap v597: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-06-10T21:22:56.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2650248492' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:22:57.124 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:57.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:57.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:22:57.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:22:57.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:57.322 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:57.479 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:57.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:57.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4011876620' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:57 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.589 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:22:57.593 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:22:57.644 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:57.745 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:22:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:57.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:22:57.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:22:57.892 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:22:57.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4011876620' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:57 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.915 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4011876620' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:57 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:22:57.922 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:58.027 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:22:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:58.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:22:58.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:22:58.192 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:58.296 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:22:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:58.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:22:58.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:22:58.465 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:58.556 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:22:58.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:22:58.612 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:22:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:58.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:22:58.654 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:22:58.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:22:58.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: pgmap v598: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: osdmap e151: 8 total, 8 up, 8 in 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[180297]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.829 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:58.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: pgmap v598: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-06-10T21:22:58.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:22:58.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: osdmap e151: 8 total, 8 up, 8 in 2023-06-10T21:22:58.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:22:58 smithi064 ceph-mon[186706]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: pgmap v598: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-06-10T21:22:58.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:22:58.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: osdmap e151: 8 total, 8 up, 8 in 2023-06-10T21:22:58.958 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: failed to encode map e151 with expected crc 2023-06-10T21:22:58.959 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:22:58 smithi152 ceph-mon[156819]: failed to encode map e151 with expected crc 2023-06-10T21:22:59.006 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:22:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:59.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:22:59.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:59.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:59.443 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:59.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:59.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:22:59.602 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:59.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:22:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:59.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:22:59.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:22:59.865 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:22:59.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:22:59.979 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:22:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:22:59.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:00.069 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:23:00.069 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:23:00.069 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:23:00.069 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:23:00.070 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:23:00.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:00.142 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:00.253 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:00.548 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:00.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:00.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:00.654 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:00.745 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:00.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:00 smithi064 ceph-mon[180297]: pgmap v600: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-06-10T21:23:00.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:00 smithi064 ceph-mon[186706]: pgmap v600: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-06-10T21:23:00.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:00 smithi152 ceph-mon[156819]: pgmap v600: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-06-10T21:23:00.990 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:23:01.287 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:01.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:01.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:01.489 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:01.608 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:23:01.615 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:23:01.646 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:01.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:01.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:01.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:01.816 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:01.817 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:01.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:01.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:01.948 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:01.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:01.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:02.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:02.151 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:02.376 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:02.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:02.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:02.520 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:02.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:02.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:02.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:02.612 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:02.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: pgmap v601: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:23:02.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:02.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: osdmap e152: 8 total, 8 up, 8 in 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[180297]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: pgmap v601: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:02.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: osdmap e152: 8 total, 8 up, 8 in 2023-06-10T21:23:02.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:02 smithi064 ceph-mon[186706]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:02.698 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: pgmap v601: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:23:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2310954155' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: osdmap e152: 8 total, 8 up, 8 in 2023-06-10T21:23:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:02 smithi152 ceph-mon[156819]: failed to encode map e152 with expected crc 2023-06-10T21:23:02.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:02.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:03.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:03.140 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:03.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:02] "GET /metrics HTTP/1.1" 200 223528 "" "Prometheus/2.33.4" 2023-06-10T21:23:03.224 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:03.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:03.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:03.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:03.398 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:03.473 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:23:03.492 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:23:03.614 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:03.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:03.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:03.724 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:23:03.724 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:17364 9.3 9:c62284ab:::benchmark_data_smithi064_247471_object17363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:23:03.725 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 125.276784217, 2023-06-10T21:23:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28749911299999997, 2023-06-10T21:23:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.726 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17364 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:23:03.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443108+0000", 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0887999999999997e-05 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443168+0000", 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3409e-05 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443182+0000", 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0399999999999998e-06 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443188+0000", 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2876e-05 2023-06-10T21:23:03.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443241+0000", 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016888800000000001 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443410+0000", 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017182300000000001 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443582+0000", 2023-06-10T21:23:03.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096170299999999995 2023-06-10T21:23:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.444543+0000", 2023-06-10T21:23:03.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000265596 2023-06-10T21:23:03.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.444809+0000", 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28572154900000002 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730530+0000", 2023-06-10T21:23:03.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4171e-05 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730585+0000", 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3592e-05 2023-06-10T21:23:03.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730608+0000", 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28749911299999997 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.739 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:17367 9.d 9:bff3de3b:::benchmark_data_smithi064_247471_object17366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 125.276691821, 2023-06-10T21:23:03.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28726967199999998, 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17367 2023-06-10T21:23:03.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:23:03.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9278999999999999e-05 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443230+0000", 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4099999999999997e-06 2023-06-10T21:23:03.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443239+0000", 2023-06-10T21:23:03.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2370000000000003e-06 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443244+0000", 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3118999999999997e-05 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443297+0000", 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012421000000000001 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443421+0000", 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015397699999999999 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443575+0000", 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015038860000000001 2023-06-10T21:23:03.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.445079+0000", 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00056494599999999998 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.445644+0000", 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28475432899999997 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730398+0000", 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8642e-05 2023-06-10T21:23:03.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730437+0000", 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4273000000000002e-05 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730471+0000", 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28726967199999998 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20066 9.d 9:b754690b:::benchmark_data_smithi064_247471_object20065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.754 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.736017586, 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999, 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20066 2023-06-10T21:23:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:23:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:23:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983875+0000", 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019127799999999999 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984066+0000", 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2920000000000001e-06 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984075+0000", 2023-06-10T21:23:03.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9660000000000001e-06 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984081+0000", 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8265999999999999e-05 2023-06-10T21:23:03.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984159+0000", 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121333 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984281+0000", 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144637 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984425+0000", 2023-06-10T21:23:03.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014625199999999999 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985888+0000", 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024362569999999998 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988324+0000", 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29050814800000002 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278832+0000", 2023-06-10T21:23:03.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5896999999999998e-05 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278878+0000", 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7633999999999999e-05 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278916+0000", 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.771 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20069 9.3 9:c33031bf:::benchmark_data_smithi064_247471_object20068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.73554993499999, 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999, 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20069 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984342+0000", 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012432649999999999 2023-06-10T21:23:03.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985585+0000", 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6175e-05 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985602+0000", 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8390000000000001e-06 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985609+0000", 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1567999999999998e-05 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985660+0000", 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013685800000000001 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985797+0000", 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016983999999999999 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985967+0000", 2023-06-10T21:23:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026175999999999999 2023-06-10T21:23:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988584+0000", 2023-06-10T21:23:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012783530000000001 2023-06-10T21:23:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989863+0000", 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28834436000000002 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278207+0000", 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278244+0000", 2023-06-10T21:23:03.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8973e-05 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278293+0000", 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.784 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20072 9.3 9:ce7b60fb:::benchmark_data_smithi064_247471_object20071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.734269559, 2023-06-10T21:23:03.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998, 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20072 2023-06-10T21:23:03.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985623+0000", 2023-06-10T21:23:03.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7206e-05 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985650+0000", 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7629e-05 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985668+0000", 2023-06-10T21:23:03.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985674+0000", 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001225133 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986899+0000", 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012472500000000001 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987024+0000", 2023-06-10T21:23:03.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000152232 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987176+0000", 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014119930000000001 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988588+0000", 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017869019999999999 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990375+0000", 2023-06-10T21:23:03.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28840961799999998 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278785+0000", 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9700000000000002e-05 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278834+0000", 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8235999999999998e-05 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278872+0000", 2023-06-10T21:23:03.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998 2023-06-10T21:23:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20074 9.3 9:cf5d96ce:::benchmark_data_smithi064_247471_object20073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.734213321, 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001, 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.798 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20074 2023-06-10T21:23:03.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985679+0000", 2023-06-10T21:23:03.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9383e-05 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985709+0000", 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985716+0000", 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.048e-06 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985720+0000", 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002102862 2023-06-10T21:23:03.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987822+0000", 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4449000000000004e-05 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987907+0000", 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134979 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988042+0000", 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064599799999999995 2023-06-10T21:23:03.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988688+0000", 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001729993 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990418+0000", 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28863366400000001 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279052+0000", 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3007999999999998e-05 2023-06-10T21:23:03.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279105+0000", 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8399000000000003e-05 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279143+0000", 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20077 9.d 9:bf31cec4:::benchmark_data_smithi064_247471_object20076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.733028671, 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002, 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.810 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20077 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:23:03.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986863+0000", 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1553000000000001e-05 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986895+0000", 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5759999999999995e-06 2023-06-10T21:23:03.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986905+0000", 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7969999999999997e-06 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986910+0000", 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1326000000000001e-05 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986982+0000", 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012606299999999999 2023-06-10T21:23:03.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987108+0000", 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014393300000000001 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987252+0000", 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001079055 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988331+0000", 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117989 2023-06-10T21:23:03.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988449+0000", 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29228185899999998 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280731+0000", 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4913000000000002e-05 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280776+0000", 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8541000000000003e-05 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280814+0000", 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20079 9.d 9:b60e4f03:::benchmark_data_smithi064_247471_object20078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.730900956, 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267, 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.821 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20079 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:23:03.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988991+0000", 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9812000000000001e-05 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989071+0000", 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1182000000000001e-05 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989082+0000", 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1869999999999996e-06 2023-06-10T21:23:03.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989087+0000", 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6656999999999998e-05 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989124+0000", 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013583000000000001 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989260+0000", 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9172999999999995e-05 2023-06-10T21:23:03.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989349+0000", 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00098969899999999992 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990339+0000", 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017819999999999999 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990517+0000", 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29222891400000001 2023-06-10T21:23:03.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282746+0000", 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8776000000000001e-05 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282805+0000", 2023-06-10T21:23:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4026000000000002e-05 2023-06-10T21:23:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282849+0000", 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.833 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20080 9.e 9:7e203f42:::benchmark_data_smithi064_247471_object20079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.730749905, 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28931462600000002, 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.834 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20080 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:23:03.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989142+0000", 2023-06-10T21:23:03.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0820000000000003e-05 2023-06-10T21:23:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989213+0000", 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0723000000000001e-05 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989224+0000", 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3220000000000004e-06 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989231+0000", 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3353999999999998e-05 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989285+0000", 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119684 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989404+0000", 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000166086 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989570+0000", 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096425099999999995 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990535+0000", 2023-06-10T21:23:03.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019061699999999999 2023-06-10T21:23:03.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990725+0000", 2023-06-10T21:23:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28764304899999998 2023-06-10T21:23:03.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278368+0000", 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4506999999999998e-05 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278423+0000", 2023-06-10T21:23:03.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5105999999999999e-05 2023-06-10T21:23:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278458+0000", 2023-06-10T21:23:03.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28931462600000002 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.850 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20081 9.e 9:7495eed3:::benchmark_data_smithi064_247471_object20080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 123.728414061, 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29004904599999998, 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20081 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991478+0000", 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6579999999999997e-05 2023-06-10T21:23:03.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991535+0000", 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1749999999999994e-06 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991544+0000", 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0550000000000001e-06 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991548+0000", 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7662e-05 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991586+0000", 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1367999999999997e-05 2023-06-10T21:23:03.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991677+0000", 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102032 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:23:03.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991779+0000", 2023-06-10T21:23:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00060211100000000001 2023-06-10T21:23:03.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.992381+0000", 2023-06-10T21:23:03.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000217801 2023-06-10T21:23:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.992599+0000", 2023-06-10T21:23:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.288837964 2023-06-10T21:23:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281437+0000", 2023-06-10T21:23:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0899000000000002e-05 2023-06-10T21:23:03.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281488+0000", 2023-06-10T21:23:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0327999999999997e-05 2023-06-10T21:23:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281528+0000", 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29004904599999998 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110551 9.3 9:cab0d1bf:::benchmark_data_smithi064_247471_object1563:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.660451+0000", 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.059442029000003, 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.273589154, 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.873 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110551 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660451+0000", 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660451+0000", 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660450+0000", 2023-06-10T21:23:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0289999999999996e-06 2023-06-10T21:23:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660455+0000", 2023-06-10T21:23:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0271e-05 2023-06-10T21:23:03.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660466+0000", 2023-06-10T21:23:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4269999999999999e-06 2023-06-10T21:23:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660471+0000", 2023-06-10T21:23:03.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033314239999999999 2023-06-10T21:23:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.663803+0000", 2023-06-10T21:23:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116922 2023-06-10T21:23:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.663919+0000", 2023-06-10T21:23:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134418 2023-06-10T21:23:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.664054+0000", 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071711789999999997 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.671225+0000", 2023-06-10T21:23:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051171410000000004 2023-06-10T21:23:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676342+0000", 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25762484699999999 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.933967+0000", 2023-06-10T21:23:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3556000000000001e-05 2023-06-10T21:23:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.934011+0000", 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9754e-05 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.934040+0000", 2023-06-10T21:23:03.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.273589154 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110554 9.3 9:c2ca445e:::benchmark_data_smithi064_247471_object1566:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.890 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.660510+0000", 2023-06-10T21:23:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.059383549000003, 2023-06-10T21:23:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28010542599999999, 2023-06-10T21:23:03.891 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.892 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110554 2023-06-10T21:23:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.893 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660510+0000", 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660510+0000", 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660509+0000", 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.864e-06 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660512+0000", 2023-06-10T21:23:03.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6949999999999997e-06 2023-06-10T21:23:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660518+0000", 2023-06-10T21:23:03.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7610000000000001e-06 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.660522+0000", 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0073019239999999996 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.667823+0000", 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119882 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.667943+0000", 2023-06-10T21:23:03.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000132643 2023-06-10T21:23:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.668076+0000", 2023-06-10T21:23:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003265422 2023-06-10T21:23:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.671341+0000", 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088878789999999996 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680229+0000", 2023-06-10T21:23:03.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.260296892 2023-06-10T21:23:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940526+0000", 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2614999999999997e-05 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940579+0000", 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6293e-05 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940615+0000", 2023-06-10T21:23:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28010542599999999 2023-06-10T21:23:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.905 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110566 9.e 9:70faa131:::benchmark_data_smithi064_247471_object1578:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.676267+0000", 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.043626189000001, 2023-06-10T21:23:03.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28474395800000002, 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110566 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.907 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676267+0000", 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676267+0000", 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676266+0000", 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7849999999999997e-06 2023-06-10T21:23:03.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676271+0000", 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4691999999999999e-05 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676286+0000", 2023-06-10T21:23:03.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9380000000000003e-06 2023-06-10T21:23:03.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676293+0000", 2023-06-10T21:23:03.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010314099999999999 2023-06-10T21:23:03.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676396+0000", 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112899 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676509+0000", 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6879999999999996e-05 2023-06-10T21:23:03.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676586+0000", 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069379389999999997 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.683524+0000", 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8651e-05 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.683563+0000", 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.27735464300000001 2023-06-10T21:23:03.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.960917+0000", 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9438000000000003e-05 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.960967+0000", 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4339999999999999e-05 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.961011+0000", 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28474395800000002 2023-06-10T21:23:03.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110569 9.3 9:cc1f0062:::benchmark_data_smithi064_247471_object1581:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.918 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.676306+0000", 2023-06-10T21:23:03.919 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.043587530000003, 2023-06-10T21:23:03.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26446874999999997, 2023-06-10T21:23:03.920 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.920 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.920 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.920 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.920 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110569 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676306+0000", 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676306+0000", 2023-06-10T21:23:03.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676305+0000", 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.422e-06 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676309+0000", 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2309999999999999e-06 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676316+0000", 2023-06-10T21:23:03.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.539e-05 2023-06-10T21:23:03.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676331+0000", 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0654000000000001e-05 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676372+0000", 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130961 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676503+0000", 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013323299999999999 2023-06-10T21:23:03.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676636+0000", 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038324660000000001 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680468+0000", 2023-06-10T21:23:03.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6943000000000002e-05 2023-06-10T21:23:03.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680505+0000", 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26020834199999998 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940714+0000", 2023-06-10T21:23:03.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5206999999999997e-05 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940749+0000", 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5493000000000001e-05 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940774+0000", 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26446874999999997 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110570 9.e 9:7bb910e9:::benchmark_data_smithi064_247471_object1582:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.676340+0000", 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.043553234999997, 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28479695999999999, 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110570 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676340+0000", 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676340+0000", 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676339+0000", 2023-06-10T21:23:03.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5439999999999999e-06 2023-06-10T21:23:03.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676343+0000", 2023-06-10T21:23:03.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3990000000000001e-06 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676349+0000", 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2860000000000001e-06 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.676354+0000", 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033533450000000002 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.679707+0000", 2023-06-10T21:23:03.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109831 2023-06-10T21:23:03.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.679817+0000", 2023-06-10T21:23:03.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119449 2023-06-10T21:23:03.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.679936+0000", 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037832619999999999 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.683720+0000", 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026155950000000001 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.686335+0000", 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.27472185900000001 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.961057+0000", 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4375999999999997e-05 2023-06-10T21:23:03.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.961101+0000", 2023-06-10T21:23:03.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5546999999999997e-05 2023-06-10T21:23:03.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.961137+0000", 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28479695999999999 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.946 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110573 9.d 9:be906241:::benchmark_data_smithi064_247471_object1585:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.680725+0000", 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.039167851000002, 2023-06-10T21:23:03.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25989394399999999, 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.948 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110573 2023-06-10T21:23:03.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.949 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680725+0000", 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680725+0000", 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:03.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680724+0000", 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7769999999999999e-06 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680730+0000", 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9589999999999995e-06 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680738+0000", 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3120000000000004e-06 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680742+0000", 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6449000000000003e-05 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680778+0000", 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8133000000000006e-05 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680846+0000", 2023-06-10T21:23:03.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8984000000000002e-05 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.680915+0000", 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056640709999999997 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.686580+0000", 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036208439999999998 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.690200+0000", 2023-06-10T21:23:03.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.250296615 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940497+0000", 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8461999999999996e-05 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940565+0000", 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3871e-05 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.940619+0000", 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25989394399999999 2023-06-10T21:23:03.958 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:110577 9.d 9:bcc85255:::benchmark_data_smithi064_247471_object1589:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:24.687205+0000", 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.032688536000002, 2023-06-10T21:23:03.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.257740682, 2023-06-10T21:23:03.960 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.960 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.960 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.961 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110577 2023-06-10T21:23:03.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.961 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687205+0000", 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687205+0000", 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687203+0000", 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2120000000000003e-06 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687211+0000", 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1786e-05 2023-06-10T21:23:03.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687223+0000", 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0549999999999999e-06 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687228+0000", 2023-06-10T21:23:03.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2487000000000002e-05 2023-06-10T21:23:03.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687271+0000", 2023-06-10T21:23:03.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8028000000000004e-05 2023-06-10T21:23:03.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687369+0000", 2023-06-10T21:23:03.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3732000000000003e-05 2023-06-10T21:23:03.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.687462+0000", 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028946499999999999 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.690357+0000", 2023-06-10T21:23:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026751599999999998 2023-06-10T21:23:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.690625+0000", 2023-06-10T21:23:03.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25424155500000001 2023-06-10T21:23:03.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.944866+0000", 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3792999999999999e-05 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.944910+0000", 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5451999999999999e-05 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:24.944945+0000", 2023-06-10T21:23:03.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.257740682 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137764 9.d 9:bf6cd1ea:::benchmark_data_smithi064_247471_object28776:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:23:03.974 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.278439686, 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998, 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.975 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137764 2023-06-10T21:23:03.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.976 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:23:03.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441452+0000", 2023-06-10T21:23:03.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7470000000000002e-06 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441458+0000", 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8309999999999996e-06 2023-06-10T21:23:03.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441468+0000", 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4750000000000001e-06 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441473+0000", 2023-06-10T21:23:03.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3132e-05 2023-06-10T21:23:03.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441516+0000", 2023-06-10T21:23:03.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114258 2023-06-10T21:23:03.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441631+0000", 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119329 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441750+0000", 2023-06-10T21:23:03.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041105380000000004 2023-06-10T21:23:03.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445860+0000", 2023-06-10T21:23:03.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074270170000000002 2023-06-10T21:23:03.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453288+0000", 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28097103499999998 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:03.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734259+0000", 2023-06-10T21:23:03.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2166000000000003e-05 2023-06-10T21:23:03.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734301+0000", 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6820000000000003e-05 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:03.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734338+0000", 2023-06-10T21:23:03.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998 2023-06-10T21:23:03.989 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.989 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:03.989 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137775 9.d 9:b6fe8ae7:::benchmark_data_smithi064_247471_object28787:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.453445+0000", 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.2664481519999999, 2023-06-10T21:23:03.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28598638900000001, 2023-06-10T21:23:03.991 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:03.991 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:03.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:03.991 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137775 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:03.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453445+0000", 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453445+0000", 2023-06-10T21:23:03.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:23:03.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:03.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453444+0000", 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7690000000000004e-06 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453449+0000", 2023-06-10T21:23:03.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4279999999999996e-06 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453457+0000", 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0830000000000003e-06 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453462+0000", 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0874999999999997e-05 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453503+0000", 2023-06-10T21:23:03.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0279999999999998e-05 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453574+0000", 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4617999999999996e-05 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453648+0000", 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030602659999999999 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.456709+0000", 2023-06-10T21:23:03.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003050695 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.459759+0000", 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.27958083500000003 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739340+0000", 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1947999999999998e-05 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:04.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739392+0000", 2023-06-10T21:23:04.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9461e-05 2023-06-10T21:23:04.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739431+0000", 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28598638900000001 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137779 9.3 9:c8aa9f8d:::benchmark_data_smithi064_247471_object28791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.460473+0000", 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.259420693, 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25179867900000003, 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:23:04.004 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:23:04.005 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:23:04.005 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:04.006 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137779 2023-06-10T21:23:04.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.007 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:23:04.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:23:04.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460473+0000", 2023-06-10T21:23:04.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:23:04.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:23:04.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460473+0000", 2023-06-10T21:23:04.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:04.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:23:04.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460472+0000", 2023-06-10T21:23:04.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3139999999999999e-06 2023-06-10T21:23:04.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:23:04.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460475+0000", 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4549999999999997e-06 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460480+0000", 2023-06-10T21:23:04.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.185e-06 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.460483+0000", 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029770909999999999 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.463460+0000", 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8684999999999997e-05 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:23:04.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.463529+0000", 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4296999999999997e-05 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.463603+0000", 2023-06-10T21:23:04.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002438768 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.466042+0000", 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24592871099999999 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.711970+0000", 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000211609 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:04.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.712182+0000", 2023-06-10T21:23:04.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5306000000000003e-05 2023-06-10T21:23:04.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:23:04.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.712227+0000", 2023-06-10T21:23:04.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3924000000000002e-05 2023-06-10T21:23:04.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:23:04.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:23:04.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.712271+0000", 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25179867900000003 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:23:04.023 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:23:04.024 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:23:04.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:04.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:04.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:04.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:04.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:04.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:04.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:04.261 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:04.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:23:04.585 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:04.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:04.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:23:04.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:04.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:04.916 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:04.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:05.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:05.130 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:05.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:05 smithi064 ceph-mon[180297]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:05.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:05 smithi064 ceph-mon[180297]: pgmap v603: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:05.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:05 smithi064 ceph-mon[186706]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:05.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:05 smithi064 ceph-mon[186706]: pgmap v603: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:05 smithi152 ceph-mon[156819]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:05.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:05 smithi152 ceph-mon[156819]: pgmap v603: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:05.610 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":602,"stamp":"2023-06-10T21:23:01.595516+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5437003686,"num_objects":82978,"num_object_clones":0,"num_object_copies":248934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82978,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":135279,"num_write_kb":6977487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128213,"ondisk_log_size":128213,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18440784,"kb_used_data":15933968,"kb_used_omap":20,"kb_used_meta":2506667,"kb_avail":731520432,"statfs":{"total":767960285184,"available":749076922368,"internally_reserved":0,"allocated":16316383232,"data_stored":16312156251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2566827662},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,11,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-358416384,"num_objects":-5469,"num_object_clones":0,"num_object_copies":-16407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5469,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.119569"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":134,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.231158+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:22:53.231158+0000","last_peered":"2023-06-10T21:22:53.231158+0000","last_clean":"2023-06-10T21:22:53.231158+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:22:53.231158+0000","last_undegraded":"2023-06-10T21:22:53.231158+0000","last_fullsized":"2023-06-10T21:22:53.231158+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.172189+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:22:51.172189+0000","last_peered":"2023-06-10T21:22:51.172189+0000","last_clean":"2023-06-10T21:22:51.172189+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:22:51.172189+0000","last_undegraded":"2023-06-10T21:22:51.172189+0000","last_fullsized":"2023-06-10T21:22:51.172189+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.202963+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:22:53.202963+0000","last_peered":"2023-06-10T21:22:53.202963+0000","last_clean":"2023-06-10T21:22:53.202963+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:22:53.202963+0000","last_undegraded":"2023-06-10T21:22:53.202963+0000","last_fullsized":"2023-06-10T21:22:53.202963+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218562+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:22:53.218562+0000","last_peered":"2023-06-10T21:22:53.218562+0000","last_clean":"2023-06-10T21:22:53.218562+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:22:53.218562+0000","last_undegraded":"2023-06-10T21:22:53.218562+0000","last_fullsized":"2023-06-10T21:22:53.218562+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169223+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:22:51.169223+0000","last_peered":"2023-06-10T21:22:51.169223+0000","last_clean":"2023-06-10T21:22:51.169223+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:22:51.169223+0000","last_undegraded":"2023-06-10T21:22:51.169223+0000","last_fullsized":"2023-06-10T21:22:51.169223+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.214491+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:22:53.214491+0000","last_peered":"2023-06-10T21:22:53.214491+0000","last_clean":"2023-06-10T21:22:53.214491+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:22:53.214491+0000","last_undegraded":"2023-06-10T21:22:53.214491+0000","last_fullsized":"2023-06-10T21:22:53.214491+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218949+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:22:53.218949+0000","last_peered":"2023-06-10T21:22:53.218949+0000","last_clean":"2023-06-10T21:22:53.218949+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:22:53.218949+0000","last_undegraded":"2023-06-10T21:22:53.218949+0000","last_fullsized":"2023-06-10T21:22:53.218949+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169180+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:22:51.169180+0000","last_peered":"2023-06-10T21:22:51.169180+0000","last_clean":"2023-06-10T21:22:51.169180+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:22:51.169180+0000","last_undegraded":"2023-06-10T21:22:51.169180+0000","last_fullsized":"2023-06-10T21:22:51.169180+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.214550+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:22:53.214550+0000","last_peered":"2023-06-10T21:22:53.214550+0000","last_clean":"2023-06-10T21:22:53.214550+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:22:53.214550+0000","last_undegraded":"2023-06-10T21:22:53.214550+0000","last_fullsized":"2023-06-10T21:22:53.214550+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":89,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.168990+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:22:51.168990+0000","last_peered":"2023-06-10T21:22:51.168990+0000","last_clean":"2023-06-10T21:22:51.168990+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:22:51.168990+0000","last_undegraded":"2023-06-10T21:22:51.168990+0000","last_fullsized":"2023-06-10T21:22:51.168990+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":121,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218030+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:22:53.218030+0000","last_peered":"2023-06-10T21:22:53.218030+0000","last_clean":"2023-06-10T21:22:53.218030+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:22:53.218030+0000","last_undegraded":"2023-06-10T21:22:53.218030+0000","last_fullsized":"2023-06-10T21:22:53.218030+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.205502+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:22:53.205502+0000","last_peered":"2023-06-10T21:22:53.205502+0000","last_clean":"2023-06-10T21:22:53.205502+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:22:53.205502+0000","last_undegraded":"2023-06-10T21:22:53.205502+0000","last_fullsized":"2023-06-10T21:22:53.205502+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":126,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.228186+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:22:53.228186+0000","last_peered":"2023-06-10T21:22:53.228186+0000","last_clean":"2023-06-10T21:22:53.228186+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:22:53.228186+0000","last_undegraded":"2023-06-10T21:22:53.228186+0000","last_fullsized":"2023-06-10T21:22:53.228186+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.227735+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:22:53.227735+0000","last_peered":"2023-06-10T21:22:53.227735+0000","last_clean":"2023-06-10T21:22:53.227735+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:22:53.227735+0000","last_undegraded":"2023-06-10T21:22:53.227735+0000","last_fullsized":"2023-06-10T21:22:53.227735+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":120,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217927+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:22:53.217927+0000","last_peered":"2023-06-10T21:22:53.217927+0000","last_clean":"2023-06-10T21:22:53.217927+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:22:53.217927+0000","last_undegraded":"2023-06-10T21:22:53.217927+0000","last_fullsized":"2023-06-10T21:22:53.217927+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169178+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:22:51.169178+0000","last_peered":"2023-06-10T21:22:51.169178+0000","last_clean":"2023-06-10T21:22:51.169178+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:22:51.169178+0000","last_undegraded":"2023-06-10T21:22:51.169178+0000","last_fullsized":"2023-06-10T21:22:51.169178+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217769+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:22:53.217769+0000","last_peered":"2023-06-10T21:22:53.217769+0000","last_clean":"2023-06-10T21:22:53.217769+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:22:53.217769+0000","last_undegraded":"2023-06-10T21:22:53.217769+0000","last_fullsized":"2023-06-10T21:22:53.217769+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212675+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:22:53.212675+0000","last_peered":"2023-06-10T21:22:53.212675+0000","last_clean":"2023-06-10T21:22:53.212675+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:22:53.212675+0000","last_undegraded":"2023-06-10T21:22:53.212675+0000","last_fullsized":"2023-06-10T21:22:53.212675+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":103,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217785+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:22:53.217785+0000","last_peered":"2023-06-10T21:22:53.217785+0000","last_clean":"2023-06-10T21:22:53.217785+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:22:53.217785+0000","last_undegraded":"2023-06-10T21:22:53.217785+0000","last_fullsized":"2023-06-10T21:22:53.217785+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218711+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:22:53.218711+0000","last_peered":"2023-06-10T21:22:53.218711+0000","last_clean":"2023-06-10T21:22:53.218711+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:22:53.218711+0000","last_undegraded":"2023-06-10T21:22:53.218711+0000","last_fullsized":"2023-06-10T21:22:53.218711+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":126,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.213575+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:22:53.213575+0000","last_peered":"2023-06-10T21:22:53.213575+0000","last_clean":"2023-06-10T21:22:53.213575+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:22:53.213575+0000","last_undegraded":"2023-06-10T21:22:53.213575+0000","last_fullsized":"2023-06-10T21:22:53.213575+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.172336+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:22:51.172336+0000","last_peered":"2023-06-10T21:22:51.172336+0000","last_clean":"2023-06-10T21:22:51.172336+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:22:51.172336+0000","last_undegraded":"2023-06-10T21:22:51.172336+0000","last_fullsized":"2023-06-10T21:22:51.172336+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"151'8406","reported_seq":8440,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.787601+0000","last_change":"2023-06-10T21:20:45.268089+0000","last_active":"2023-06-10T21:22:57.787601+0000","last_peered":"2023-06-10T21:22:57.787601+0000","last_clean":"2023-06-10T21:22:57.787601+0000","last_became_active":"2023-06-10T21:20:45.267772+0000","last_became_peered":"2023-06-10T21:20:45.267772+0000","last_unstale":"2023-06-10T21:22:57.787601+0000","last_undegraded":"2023-06-10T21:22:57.787601+0000","last_fullsized":"2023-06-10T21:22:57.787601+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8406,"log_dups_size":0,"ondisk_log_size":8406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:24:07.609015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341573632,"num_objects":5212,"num_object_clones":0,"num_object_copies":15636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8406,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218235+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:22:53.218235+0000","last_peered":"2023-06-10T21:22:53.218235+0000","last_clean":"2023-06-10T21:22:53.218235+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:22:53.218235+0000","last_undegraded":"2023-06-10T21:22:53.218235+0000","last_fullsized":"2023-06-10T21:22:53.218235+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212625+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:22:53.212625+0000","last_peered":"2023-06-10T21:22:53.212625+0000","last_clean":"2023-06-10T21:22:53.212625+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:22:53.212625+0000","last_undegraded":"2023-06-10T21:22:53.212625+0000","last_fullsized":"2023-06-10T21:22:53.212625+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"151'8586","reported_seq":8620,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:23:00.984228+0000","last_change":"2023-06-10T21:20:45.293144+0000","last_active":"2023-06-10T21:23:00.984228+0000","last_peered":"2023-06-10T21:23:00.984228+0000","last_clean":"2023-06-10T21:23:00.984228+0000","last_became_active":"2023-06-10T21:20:45.292848+0000","last_became_peered":"2023-06-10T21:20:45.292848+0000","last_unstale":"2023-06-10T21:23:00.984228+0000","last_undegraded":"2023-06-10T21:23:00.984228+0000","last_fullsized":"2023-06-10T21:23:00.984228+0000","mapping_epoch":127,"log_start":"131'1200","ondisk_log_start":"131'1200","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:53:38.966995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341442560,"num_objects":5210,"num_object_clones":0,"num_object_copies":15630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8586,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":120,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.220409+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:22:53.220409+0000","last_peered":"2023-06-10T21:22:53.220409+0000","last_clean":"2023-06-10T21:22:53.220409+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:22:53.220409+0000","last_undegraded":"2023-06-10T21:22:53.220409+0000","last_fullsized":"2023-06-10T21:22:53.220409+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.229711+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:22:53.229711+0000","last_peered":"2023-06-10T21:22:53.229711+0000","last_clean":"2023-06-10T21:22:53.229711+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:22:53.229711+0000","last_undegraded":"2023-06-10T21:22:53.229711+0000","last_fullsized":"2023-06-10T21:22:53.229711+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"151'8387","reported_seq":8419,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.379934+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:22:59.379934+0000","last_peered":"2023-06-10T21:22:59.379934+0000","last_clean":"2023-06-10T21:22:59.379934+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:22:59.379934+0000","last_undegraded":"2023-06-10T21:22:59.379934+0000","last_fullsized":"2023-06-10T21:22:59.379934+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335872000,"num_objects":5125,"num_object_clones":0,"num_object_copies":15375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8387,"num_write_kb":432384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217864+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:22:53.217864+0000","last_peered":"2023-06-10T21:22:53.217864+0000","last_clean":"2023-06-10T21:22:53.217864+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:22:53.217864+0000","last_undegraded":"2023-06-10T21:22:53.217864+0000","last_fullsized":"2023-06-10T21:22:53.217864+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.213953+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:22:53.213953+0000","last_peered":"2023-06-10T21:22:53.213953+0000","last_clean":"2023-06-10T21:22:53.213953+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:22:53.213953+0000","last_undegraded":"2023-06-10T21:22:53.213953+0000","last_fullsized":"2023-06-10T21:22:53.213953+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"151'8467","reported_seq":8498,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.542884+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:22:59.542884+0000","last_peered":"2023-06-10T21:22:59.542884+0000","last_clean":"2023-06-10T21:22:59.542884+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:22:59.542884+0000","last_undegraded":"2023-06-10T21:22:59.542884+0000","last_fullsized":"2023-06-10T21:22:59.542884+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8467,"log_dups_size":0,"ondisk_log_size":8467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336134144,"num_objects":5129,"num_object_clones":0,"num_object_copies":15387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212610+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:22:53.212610+0000","last_peered":"2023-06-10T21:22:53.212610+0000","last_clean":"2023-06-10T21:22:53.212610+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:22:53.212610+0000","last_undegraded":"2023-06-10T21:22:53.212610+0000","last_fullsized":"2023-06-10T21:22:53.212610+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218578+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:22:53.218578+0000","last_peered":"2023-06-10T21:22:53.218578+0000","last_clean":"2023-06-10T21:22:53.218578+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:22:53.218578+0000","last_undegraded":"2023-06-10T21:22:53.218578+0000","last_fullsized":"2023-06-10T21:22:53.218578+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"150'8335","reported_seq":8368,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:56.733862+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:22:56.733862+0000","last_peered":"2023-06-10T21:22:56.733862+0000","last_clean":"2023-06-10T21:22:56.733862+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:22:56.733862+0000","last_undegraded":"2023-06-10T21:22:56.733862+0000","last_fullsized":"2023-06-10T21:22:56.733862+0000","mapping_epoch":127,"log_start":"131'600","ondisk_log_start":"131'600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8335,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169023+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:22:51.169023+0000","last_peered":"2023-06-10T21:22:51.169023+0000","last_clean":"2023-06-10T21:22:51.169023+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:22:51.169023+0000","last_undegraded":"2023-06-10T21:22:51.169023+0000","last_fullsized":"2023-06-10T21:22:51.169023+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.228110+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:22:53.228110+0000","last_peered":"2023-06-10T21:22:53.228110+0000","last_clean":"2023-06-10T21:22:53.228110+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:22:53.228110+0000","last_undegraded":"2023-06-10T21:22:53.228110+0000","last_fullsized":"2023-06-10T21:22:53.228110+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"151'8290","reported_seq":8324,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:23:00.850637+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:23:00.850637+0000","last_peered":"2023-06-10T21:23:00.850637+0000","last_clean":"2023-06-10T21:23:00.850637+0000","last_became_active":"2023-06-10T21:20:45.291689+0000","last_became_peered":"2023-06-10T21:20:45.291689+0000","last_unstale":"2023-06-10T21:23:00.850637+0000","last_undegraded":"2023-06-10T21:23:00.850637+0000","last_fullsized":"2023-06-10T21:23:00.850637+0000","mapping_epoch":127,"log_start":"131'900","ondisk_log_start":"131'900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:11:53.302103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320471040,"num_objects":4890,"num_object_clones":0,"num_object_copies":14670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8290,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.213835+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:22:53.213835+0000","last_peered":"2023-06-10T21:22:53.213835+0000","last_clean":"2023-06-10T21:22:53.213835+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:22:53.213835+0000","last_undegraded":"2023-06-10T21:22:53.213835+0000","last_fullsized":"2023-06-10T21:22:53.213835+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":132,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.227711+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:22:53.227711+0000","last_peered":"2023-06-10T21:22:53.227711+0000","last_clean":"2023-06-10T21:22:53.227711+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:22:53.227711+0000","last_undegraded":"2023-06-10T21:22:53.227711+0000","last_fullsized":"2023-06-10T21:22:53.227711+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"151'8581","reported_seq":8613,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.321821+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:22:59.321821+0000","last_peered":"2023-06-10T21:22:59.321821+0000","last_clean":"2023-06-10T21:22:59.321821+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:22:59.321821+0000","last_undegraded":"2023-06-10T21:22:59.321821+0000","last_fullsized":"2023-06-10T21:22:59.321821+0000","mapping_epoch":127,"log_start":"131'800","ondisk_log_start":"131'800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7781,"log_dups_size":0,"ondisk_log_size":7781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341770240,"num_objects":5215,"num_object_clones":0,"num_object_copies":15645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8581,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.229522+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:22:53.229522+0000","last_peered":"2023-06-10T21:22:53.229522+0000","last_clean":"2023-06-10T21:22:53.229522+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:22:53.229522+0000","last_undegraded":"2023-06-10T21:22:53.229522+0000","last_fullsized":"2023-06-10T21:22:53.229522+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.230355+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:22:53.230355+0000","last_peered":"2023-06-10T21:22:53.230355+0000","last_clean":"2023-06-10T21:22:53.230355+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:22:53.230355+0000","last_undegraded":"2023-06-10T21:22:53.230355+0000","last_fullsized":"2023-06-10T21:22:53.230355+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"150'8455","reported_seq":8488,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:56.810319+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:22:56.810319+0000","last_peered":"2023-06-10T21:22:56.810319+0000","last_clean":"2023-06-10T21:22:56.810319+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:22:56.810319+0000","last_undegraded":"2023-06-10T21:22:56.810319+0000","last_fullsized":"2023-06-10T21:22:56.810319+0000","mapping_epoch":127,"log_start":"131'700","ondisk_log_start":"131'700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7755,"log_dups_size":0,"ondisk_log_size":7755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346357760,"num_objects":5285,"num_object_clones":0,"num_object_copies":15855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8455,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169304+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:22:51.169304+0000","last_peered":"2023-06-10T21:22:51.169304+0000","last_clean":"2023-06-10T21:22:51.169304+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:22:51.169304+0000","last_undegraded":"2023-06-10T21:22:51.169304+0000","last_fullsized":"2023-06-10T21:22:51.169304+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.204595+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:22:53.204595+0000","last_peered":"2023-06-10T21:22:53.204595+0000","last_clean":"2023-06-10T21:22:53.204595+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:22:53.204595+0000","last_undegraded":"2023-06-10T21:22:53.204595+0000","last_fullsized":"2023-06-10T21:22:53.204595+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"151'8417","reported_seq":8449,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.543425+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:22:59.543425+0000","last_peered":"2023-06-10T21:22:59.543425+0000","last_clean":"2023-06-10T21:22:59.543425+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:22:59.543425+0000","last_undegraded":"2023-06-10T21:22:59.543425+0000","last_fullsized":"2023-06-10T21:22:59.543425+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8417,"log_dups_size":0,"ondisk_log_size":8417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8417,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":120,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574885+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:22:57.574885+0000","last_peered":"2023-06-10T21:22:57.574885+0000","last_clean":"2023-06-10T21:22:57.574885+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:22:57.574885+0000","last_undegraded":"2023-06-10T21:22:57.574885+0000","last_fullsized":"2023-06-10T21:22:57.574885+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":133,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212746+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:22:53.212746+0000","last_peered":"2023-06-10T21:22:53.212746+0000","last_clean":"2023-06-10T21:22:53.212746+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:22:53.212746+0000","last_undegraded":"2023-06-10T21:22:53.212746+0000","last_fullsized":"2023-06-10T21:22:53.212746+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":96,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.220861+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:22:53.220861+0000","last_peered":"2023-06-10T21:22:53.220861+0000","last_clean":"2023-06-10T21:22:53.220861+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:22:53.220861+0000","last_undegraded":"2023-06-10T21:22:53.220861+0000","last_fullsized":"2023-06-10T21:22:53.220861+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"151'8296","reported_seq":8328,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.818514+0000","last_change":"2023-06-10T21:20:45.269372+0000","last_active":"2023-06-10T21:22:57.818514+0000","last_peered":"2023-06-10T21:22:57.818514+0000","last_clean":"2023-06-10T21:22:57.818514+0000","last_became_active":"2023-06-10T21:20:45.269098+0000","last_became_peered":"2023-06-10T21:20:45.269098+0000","last_unstale":"2023-06-10T21:22:57.818514+0000","last_undegraded":"2023-06-10T21:22:57.818514+0000","last_fullsized":"2023-06-10T21:22:57.818514+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8296,"log_dups_size":0,"ondisk_log_size":8296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:23:51.802626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8296,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.220383+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:22:53.220383+0000","last_peered":"2023-06-10T21:22:53.220383+0000","last_clean":"2023-06-10T21:22:53.220383+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:22:53.220383+0000","last_undegraded":"2023-06-10T21:22:53.220383+0000","last_fullsized":"2023-06-10T21:22:53.220383+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.227877+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:22:53.227877+0000","last_peered":"2023-06-10T21:22:53.227877+0000","last_clean":"2023-06-10T21:22:53.227877+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:22:53.227877+0000","last_undegraded":"2023-06-10T21:22:53.227877+0000","last_fullsized":"2023-06-10T21:22:53.227877+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":143,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574956+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:22:57.574956+0000","last_peered":"2023-06-10T21:22:57.574956+0000","last_clean":"2023-06-10T21:22:57.574956+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:22:57.574956+0000","last_undegraded":"2023-06-10T21:22:57.574956+0000","last_fullsized":"2023-06-10T21:22:57.574956+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"151'8544","reported_seq":8576,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.493707+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:22:59.493707+0000","last_peered":"2023-06-10T21:22:59.493707+0000","last_clean":"2023-06-10T21:22:59.493707+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:22:59.493707+0000","last_undegraded":"2023-06-10T21:22:59.493707+0000","last_fullsized":"2023-06-10T21:22:59.493707+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8544,"log_dups_size":0,"ondisk_log_size":8544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335937536,"num_objects":5126,"num_object_clones":0,"num_object_copies":15378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8544,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.220422+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:22:53.220422+0000","last_peered":"2023-06-10T21:22:53.220422+0000","last_clean":"2023-06-10T21:22:53.220422+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:22:53.220422+0000","last_undegraded":"2023-06-10T21:22:53.220422+0000","last_fullsized":"2023-06-10T21:22:53.220422+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":128,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.202625+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:22:53.202625+0000","last_peered":"2023-06-10T21:22:53.202625+0000","last_clean":"2023-06-10T21:22:53.202625+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:22:53.202625+0000","last_undegraded":"2023-06-10T21:22:53.202625+0000","last_fullsized":"2023-06-10T21:22:53.202625+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.204671+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:22:53.204671+0000","last_peered":"2023-06-10T21:22:53.204671+0000","last_clean":"2023-06-10T21:22:53.204671+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:22:53.204671+0000","last_undegraded":"2023-06-10T21:22:53.204671+0000","last_fullsized":"2023-06-10T21:22:53.204671+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"150'8244","reported_seq":8277,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:56.786089+0000","last_change":"2023-06-10T21:20:45.262983+0000","last_active":"2023-06-10T21:22:56.786089+0000","last_peered":"2023-06-10T21:22:56.786089+0000","last_clean":"2023-06-10T21:22:56.786089+0000","last_became_active":"2023-06-10T21:20:45.262641+0000","last_became_peered":"2023-06-10T21:20:45.262641+0000","last_unstale":"2023-06-10T21:22:56.786089+0000","last_undegraded":"2023-06-10T21:22:56.786089+0000","last_fullsized":"2023-06-10T21:22:56.786089+0000","mapping_epoch":127,"log_start":"131'500","ondisk_log_start":"131'500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:27:45.171697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334364672,"num_objects":5102,"num_object_clones":0,"num_object_copies":15306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8244,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212422+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:22:53.212422+0000","last_peered":"2023-06-10T21:22:53.212422+0000","last_clean":"2023-06-10T21:22:53.212422+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:22:53.212422+0000","last_undegraded":"2023-06-10T21:22:53.212422+0000","last_fullsized":"2023-06-10T21:22:53.212422+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":354,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217263+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:22:53.217263+0000","last_peered":"2023-06-10T21:22:53.217263+0000","last_clean":"2023-06-10T21:22:53.217263+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:22:53.217263+0000","last_undegraded":"2023-06-10T21:22:53.217263+0000","last_fullsized":"2023-06-10T21:22:53.217263+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.170674+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:22:51.170674+0000","last_peered":"2023-06-10T21:22:51.170674+0000","last_clean":"2023-06-10T21:22:51.170674+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:22:51.170674+0000","last_undegraded":"2023-06-10T21:22:51.170674+0000","last_fullsized":"2023-06-10T21:22:51.170674+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.214294+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:22:53.214294+0000","last_peered":"2023-06-10T21:22:53.214294+0000","last_clean":"2023-06-10T21:22:53.214294+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:22:53.214294+0000","last_undegraded":"2023-06-10T21:22:53.214294+0000","last_fullsized":"2023-06-10T21:22:53.214294+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"151'8504","reported_seq":8537,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.813182+0000","last_change":"2023-06-10T21:20:45.259115+0000","last_active":"2023-06-10T21:22:57.813182+0000","last_peered":"2023-06-10T21:22:57.813182+0000","last_clean":"2023-06-10T21:22:57.813182+0000","last_became_active":"2023-06-10T21:20:45.258813+0000","last_became_peered":"2023-06-10T21:20:45.258813+0000","last_unstale":"2023-06-10T21:22:57.813182+0000","last_undegraded":"2023-06-10T21:22:57.813182+0000","last_fullsized":"2023-06-10T21:22:57.813182+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8504,"log_dups_size":0,"ondisk_log_size":8504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:29.551738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342491136,"num_objects":5226,"num_object_clones":0,"num_object_copies":15678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8504,"num_write_kb":439360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":137,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217182+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:22:53.217182+0000","last_peered":"2023-06-10T21:22:53.217182+0000","last_clean":"2023-06-10T21:22:53.217182+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:22:53.217182+0000","last_undegraded":"2023-06-10T21:22:53.217182+0000","last_fullsized":"2023-06-10T21:22:53.217182+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218119+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:22:53.218119+0000","last_peered":"2023-06-10T21:22:53.218119+0000","last_clean":"2023-06-10T21:22:53.218119+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:22:53.218119+0000","last_undegraded":"2023-06-10T21:22:53.218119+0000","last_fullsized":"2023-06-10T21:22:53.218119+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":121,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218637+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:22:53.218637+0000","last_peered":"2023-06-10T21:22:53.218637+0000","last_clean":"2023-06-10T21:22:53.218637+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:22:53.218637+0000","last_undegraded":"2023-06-10T21:22:53.218637+0000","last_fullsized":"2023-06-10T21:22:53.218637+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"151'8525","reported_seq":8557,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.396198+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:22:59.396198+0000","last_peered":"2023-06-10T21:22:59.396198+0000","last_clean":"2023-06-10T21:22:59.396198+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:22:59.396198+0000","last_undegraded":"2023-06-10T21:22:59.396198+0000","last_fullsized":"2023-06-10T21:22:59.396198+0000","mapping_epoch":127,"log_start":"131'800","ondisk_log_start":"131'800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340721664,"num_objects":5199,"num_object_clones":0,"num_object_copies":15597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8525,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.217938+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:22:53.217938+0000","last_peered":"2023-06-10T21:22:53.217938+0000","last_clean":"2023-06-10T21:22:53.217938+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:22:53.217938+0000","last_undegraded":"2023-06-10T21:22:53.217938+0000","last_fullsized":"2023-06-10T21:22:53.217938+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":128,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575033+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:22:57.575033+0000","last_peered":"2023-06-10T21:22:57.575033+0000","last_clean":"2023-06-10T21:22:57.575033+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:22:57.575033+0000","last_undegraded":"2023-06-10T21:22:57.575033+0000","last_fullsized":"2023-06-10T21:22:57.575033+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":91,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218488+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:22:53.218488+0000","last_peered":"2023-06-10T21:22:53.218488+0000","last_clean":"2023-06-10T21:22:53.218488+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:22:53.218488+0000","last_undegraded":"2023-06-10T21:22:53.218488+0000","last_fullsized":"2023-06-10T21:22:53.218488+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"151'8675","reported_seq":8708,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:59.389909+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:22:59.389909+0000","last_peered":"2023-06-10T21:22:59.389909+0000","last_clean":"2023-06-10T21:22:59.389909+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:22:59.389909+0000","last_undegraded":"2023-06-10T21:22:59.389909+0000","last_fullsized":"2023-06-10T21:22:59.389909+0000","mapping_epoch":127,"log_start":"131'900","ondisk_log_start":"131'900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346292248,"num_objects":5285,"num_object_clones":0,"num_object_copies":15855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5285,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8675,"num_write_kb":446657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.172445+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:22:51.172445+0000","last_peered":"2023-06-10T21:22:51.172445+0000","last_clean":"2023-06-10T21:22:51.172445+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:22:51.172445+0000","last_undegraded":"2023-06-10T21:22:51.172445+0000","last_fullsized":"2023-06-10T21:22:51.172445+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.205583+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:22:53.205583+0000","last_peered":"2023-06-10T21:22:53.205583+0000","last_clean":"2023-06-10T21:22:53.205583+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:22:53.205583+0000","last_undegraded":"2023-06-10T21:22:53.205583+0000","last_fullsized":"2023-06-10T21:22:53.205583+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.169262+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:22:51.169262+0000","last_peered":"2023-06-10T21:22:51.169262+0000","last_clean":"2023-06-10T21:22:51.169262+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:22:51.169262+0000","last_undegraded":"2023-06-10T21:22:51.169262+0000","last_fullsized":"2023-06-10T21:22:51.169262+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"150'8306","reported_seq":8338,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:56.763836+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:22:56.763836+0000","last_peered":"2023-06-10T21:22:56.763836+0000","last_clean":"2023-06-10T21:22:56.763836+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:22:56.763836+0000","last_undegraded":"2023-06-10T21:22:56.763836+0000","last_fullsized":"2023-06-10T21:22:56.763836+0000","mapping_epoch":127,"log_start":"131'600","ondisk_log_start":"131'600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7706,"log_dups_size":0,"ondisk_log_size":7706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347602944,"num_objects":5304,"num_object_clones":0,"num_object_copies":15912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8306,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":122,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.214435+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:22:53.214435+0000","last_peered":"2023-06-10T21:22:53.214435+0000","last_clean":"2023-06-10T21:22:53.214435+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:22:53.214435+0000","last_undegraded":"2023-06-10T21:22:53.214435+0000","last_fullsized":"2023-06-10T21:22:53.214435+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218476+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:22:53.218476+0000","last_peered":"2023-06-10T21:22:53.218476+0000","last_clean":"2023-06-10T21:22:53.218476+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:22:53.218476+0000","last_undegraded":"2023-06-10T21:22:53.218476+0000","last_fullsized":"2023-06-10T21:22:53.218476+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":102,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.229595+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:22:53.229595+0000","last_peered":"2023-06-10T21:22:53.229595+0000","last_clean":"2023-06-10T21:22:53.229595+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:22:53.229595+0000","last_undegraded":"2023-06-10T21:22:53.229595+0000","last_fullsized":"2023-06-10T21:22:53.229595+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":94,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218592+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:22:53.218592+0000","last_peered":"2023-06-10T21:22:53.218592+0000","last_clean":"2023-06-10T21:22:53.218592+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:22:53.218592+0000","last_undegraded":"2023-06-10T21:22:53.218592+0000","last_fullsized":"2023-06-10T21:22:53.218592+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":123,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.220169+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:22:53.220169+0000","last_peered":"2023-06-10T21:22:53.220169+0000","last_clean":"2023-06-10T21:22:53.220169+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:22:53.220169+0000","last_undegraded":"2023-06-10T21:22:53.220169+0000","last_fullsized":"2023-06-10T21:22:53.220169+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":120,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212591+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:22:53.212591+0000","last_peered":"2023-06-10T21:22:53.212591+0000","last_clean":"2023-06-10T21:22:53.212591+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:22:53.212591+0000","last_undegraded":"2023-06-10T21:22:53.212591+0000","last_fullsized":"2023-06-10T21:22:53.212591+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.227948+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:22:53.227948+0000","last_peered":"2023-06-10T21:22:53.227948+0000","last_clean":"2023-06-10T21:22:53.227948+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:22:53.227948+0000","last_undegraded":"2023-06-10T21:22:53.227948+0000","last_fullsized":"2023-06-10T21:22:53.227948+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":105,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.212828+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:22:53.212828+0000","last_peered":"2023-06-10T21:22:53.212828+0000","last_clean":"2023-06-10T21:22:53.212828+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:22:53.212828+0000","last_undegraded":"2023-06-10T21:22:53.212828+0000","last_fullsized":"2023-06-10T21:22:53.212828+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.231276+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:22:53.231276+0000","last_peered":"2023-06-10T21:22:53.231276+0000","last_clean":"2023-06-10T21:22:53.231276+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:22:53.231276+0000","last_undegraded":"2023-06-10T21:22:53.231276+0000","last_fullsized":"2023-06-10T21:22:53.231276+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":126,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.172349+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:22:51.172349+0000","last_peered":"2023-06-10T21:22:51.172349+0000","last_clean":"2023-06-10T21:22:51.172349+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:22:51.172349+0000","last_undegraded":"2023-06-10T21:22:51.172349+0000","last_fullsized":"2023-06-10T21:22:51.172349+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":139,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-06-10T21:22:51.168933+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:22:51.168933+0000","last_peered":"2023-06-10T21:22:51.168933+0000","last_clean":"2023-06-10T21:22:51.168933+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:22:51.168933+0000","last_undegraded":"2023-06-10T21:22:51.168933+0000","last_fullsized":"2023-06-10T21:22:51.168933+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":110,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.218871+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:22:53.218871+0000","last_peered":"2023-06-10T21:22:53.218871+0000","last_clean":"2023-06-10T21:22:53.218871+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:22:53.218871+0000","last_undegraded":"2023-06-10T21:22:53.218871+0000","last_fullsized":"2023-06-10T21:22:53.218871+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":126,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-06-10T21:22:53.227570+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:22:53.227570+0000","last_peered":"2023-06-10T21:22:53.227570+0000","last_clean":"2023-06-10T21:22:53.227570+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:22:53.227570+0000","last_undegraded":"2023-06-10T21:22:53.227570+0000","last_fullsized":"2023-06-10T21:22:53.227570+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5436407832,"num_objects":82954,"num_object_clones":0,"num_object_copies":248862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82954,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135018,"num_write_kb":6975041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16250515456,"data_stored":16250503240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128018,"ondisk_log_size":128018,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430511,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3383136,"kb_used_data":3007768,"kb_used_omap":0,"kb_used_meta":375360,"kb_avail":90362016,"statfs":{"total":95995035648,"available":92530704384,"internally_reserved":0,"allocated":3079954432,"data_stored":3079451393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":384368640},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:12 2023","interfaces":[{"interface":"back","average":{"1min":4.7240000000000002,"5min":1.7909999999999999,"15min":1.5509999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.22800000000000001},"max":{"1min":53.706000000000003,"5min":53.706000000000003,"15min":53.706000000000003},"last":4.8390000000000004},{"interface":"front","average":{"1min":4.2599999999999998,"5min":1.776,"15min":1.998},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":53.777000000000001,"5min":53.777000000000001,"15min":53.777000000000001},"last":4.9000000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":10.936,"5min":3.6429999999999998,"15min":2.3359999999999999},"min":{"1min":0.32100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":130.19399999999999,"5min":130.19399999999999,"15min":130.19399999999999},"last":5.1929999999999996},{"interface":"front","average":{"1min":21.949000000000002,"5min":5.9269999999999996,"15min":3.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":212.20599999999999,"5min":212.20599999999999,"15min":212.20599999999999},"last":4.625}]},{"osd":2,"last update":"Sat Jun 10 21:22:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0979999999999999,"5min":0.84499999999999997,"15min":0.626},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":9.1609999999999996,"5min":9.1609999999999996,"15min":9.1609999999999996},"last":3.7999999999999998},{"interface":"front","average":{"1min":1.597,"5min":0.76100000000000001,"15min":0.60899999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":4.4669999999999996,"5min":4.4669999999999996,"15min":4.4669999999999996},"last":5.3650000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":6.6340000000000003,"5min":2.4249999999999998,"15min":1.1830000000000001},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":43.911000000000001,"5min":57.085000000000001,"15min":57.085000000000001},"last":6.1760000000000002},{"interface":"front","average":{"1min":6.1660000000000004,"5min":2.4430000000000001,"15min":1.2050000000000001},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":44.225999999999999,"5min":57.167999999999999,"15min":57.167999999999999},"last":4.931}]},{"osd":4,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.68500000000000005,"15min":1.929},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":4.0170000000000003,"5min":4.0170000000000003,"15min":219.17699999999999},"last":7.0010000000000003},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.0109999999999999,"15min":2.1680000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":9.2129999999999992,"5min":9.2129999999999992,"15min":219.13999999999999},"last":4.0800000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":0.91200000000000003,"15min":2.5419999999999998},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":4.7249999999999996,"5min":4.7619999999999996,"15min":311.202},"last":7.1299999999999999},{"interface":"front","average":{"1min":1.3859999999999999,"5min":0.94499999999999995,"15min":2.4239999999999999},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.29699999999999999},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":311.875},"last":4.649}]},{"osd":6,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.86699999999999999,"15min":1.0649999999999999},"min":{"1min":0.39900000000000002,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.4109999999999996,"5min":4.4109999999999996,"15min":8.6630000000000003},"last":4.8680000000000003},{"interface":"front","average":{"1min":1.2889999999999999,"5min":0.92400000000000004,"15min":1.016},"min":{"1min":0.40000000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":4.8680000000000003,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":7.9980000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626739,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285452,"kb_used_data":985220,"kb_used_omap":0,"kb_used_meta":300224,"kb_avail":92459700,"statfs":{"total":95995035648,"available":94678732800,"internally_reserved":0,"allocated":1008865280,"data_stored":1008382236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":307429376},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.173,"15min":2.1179999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":8.4730000000000008,"5min":9.5250000000000004,"15min":119.276},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.7509999999999999,"5min":1.226,"15min":2.609},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":4.0940000000000003,"5min":9.4580000000000002,"15min":119.22},"last":0.318}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.3340000000000001,"15min":1.746},"min":{"1min":0.32200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9130000000000003,"5min":14.897,"15min":14.897},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.0600000000000001,"5min":1.4770000000000001,"15min":1.8680000000000001},"min":{"1min":0.30599999999999999,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5369999999999999,"5min":8.0299999999999994,"15min":10.766999999999999},"last":2.6589999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.72999999999999998,"15min":0.61799999999999999},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.944,"5min":3.0089999999999999,"15min":3.0089999999999999},"last":4.593},{"interface":"front","average":{"1min":1.8069999999999999,"5min":0.79700000000000004,"15min":0.624},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9600000000000009,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":4.6059999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":5.8479999999999999,"5min":1.7569999999999999,"15min":0.95099999999999996},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":53.591000000000001,"5min":53.591000000000001,"15min":53.591000000000001},"last":2.5030000000000001},{"interface":"front","average":{"1min":5.1749999999999998,"5min":1.873,"15min":1.004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":53.616,"5min":53.616,"15min":53.616},"last":2.649}]},{"osd":4,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.0029999999999999,"15min":1.819},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":4.0049999999999999,"5min":6.7290000000000001,"15min":221.76900000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.3440000000000001,"5min":0.92900000000000005,"15min":1.974},"min":{"1min":0.34499999999999997,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":10.221,"5min":10.221,"15min":221.905},"last":0.307}]},{"osd":5,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6549999999999998,"5min":1.2889999999999999,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.25700000000000001},"max":{"1min":11.132999999999999,"5min":11.132999999999999,"15min":313.661},"last":198.154},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.1499999999999999,"15min":2.4020000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":10.17,"5min":10.17,"15min":314.38900000000001},"last":198.08199999999999}]},{"osd":7,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.367,"15min":1.196},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":4.0679999999999996,"5min":11.179,"15min":11.179},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.181,"5min":1.2190000000000001,"15min":1.2090000000000001},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":10.244999999999999,"5min":10.244999999999999,"15min":10.244999999999999},"last":0.34599999999999997}]}]},{"osd":1,"up_from":105,"seq":450971566182,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3231420,"kb_used_data":2946484,"kb_used_omap":4,"kb_used_meta":284859,"kb_avail":90513732,"statfs":{"total":95995035648,"available":92686061568,"internally_reserved":0,"allocated":3017199616,"data_stored":3016613334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":291696526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.3809999999999998,"15min":2.129},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":29.754999999999999,"5min":86.009,"15min":86.009},"last":4.4569999999999999},{"interface":"front","average":{"1min":3.3370000000000002,"5min":2.7719999999999998,"15min":2.669},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.22600000000000001},"max":{"1min":32.079000000000001,"5min":85.869,"15min":85.869},"last":2.1819999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":0.95099999999999996,"15min":0.64800000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.6080000000000005,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.58299999999999996,"15min":0.50700000000000001},"min":{"1min":0.23799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.3849999999999998,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":2.347}]},{"osd":3,"last update":"Sat Jun 10 21:22:31 2023","interfaces":[{"interface":"back","average":{"1min":12.256,"5min":2.9249999999999998,"15min":1.2150000000000001},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":118.348,"5min":118.348,"15min":118.348},"last":0.23300000000000001},{"interface":"front","average":{"1min":13.488,"5min":3.4420000000000002,"15min":1.4239999999999999},"min":{"1min":0.48499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":117.75,"5min":117.75,"15min":117.75},"last":1.806}]},{"osd":4,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.2350000000000001,"15min":1.7989999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.0199999999999996,"5min":8.0199999999999996,"15min":12.94},"last":1.756},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.22,"15min":2.343},"min":{"1min":0.40699999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.5720000000000001,"5min":7.5720000000000001,"15min":12.603},"last":4.4029999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.55,"15min":2.0720000000000001},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":8.2550000000000008,"5min":9.7219999999999995,"15min":11.007999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.8820000000000001,"5min":1.5249999999999999,"15min":2.3130000000000002},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.309},"max":{"1min":7.9669999999999996,"5min":8.9700000000000006,"15min":13.602},"last":6.5910000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":1.4930000000000001,"15min":2.0699999999999998},"min":{"1min":0.56499999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":9.6699999999999999,"15min":12.861000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.266,"5min":1.3799999999999999,"15min":2.2610000000000001},"min":{"1min":0.48199999999999998,"5min":0.40899999999999997,"15min":0.36699999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":12.970000000000001},"last":4.5739999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":1.363,"15min":2.8090000000000002},"min":{"1min":0.45300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.1750000000000007,"5min":8.1750000000000007,"15min":13.025},"last":6.5490000000000004},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.2549999999999999,"15min":1.984},"min":{"1min":0.49199999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.6920000000000002,"5min":7.6920000000000002,"15min":19.286999999999999},"last":4.4359999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187604,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1699144,"kb_used_data":1333696,"kb_used_omap":5,"kb_used_meta":365434,"kb_avail":92046008,"statfs":{"total":95995035648,"available":94255112192,"internally_reserved":0,"allocated":1365704704,"data_stored":1365100594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":374205050},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:22:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1070000000000002,"5min":3.165,"15min":2.6190000000000002},"min":{"1min":0.28000000000000003,"5min":0.20899999999999999,"15min":0.19700000000000001},"max":{"1min":13.58,"5min":137.761,"15min":137.761},"last":5.0350000000000001},{"interface":"front","average":{"1min":3.2519999999999998,"5min":3.1040000000000001,"15min":3.3620000000000001},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":15.045999999999999,"5min":137.88999999999999,"15min":137.88999999999999},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":0.71099999999999997,"15min":0.46300000000000002},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.797000000000001,"5min":10.797000000000001,"15min":10.797000000000001},"last":2.2890000000000001},{"interface":"front","average":{"1min":1.7889999999999999,"5min":0.69399999999999995,"15min":0.46899999999999997},"min":{"1min":0.29099999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.6609999999999996,"5min":5.6609999999999996,"15min":5.6609999999999996},"last":0.57999999999999996}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":10.523,"5min":4.3330000000000002,"15min":1.712},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":75.674999999999997,"5min":163.56999999999999,"15min":163.56999999999999},"last":2.3330000000000002},{"interface":"front","average":{"1min":10.789,"5min":4.4050000000000002,"15min":1.726},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":77.289000000000001,"5min":164.239,"15min":164.239},"last":5.0810000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.821,"5min":1.3600000000000001,"15min":2.089},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":10.733000000000001,"5min":10.733000000000001,"15min":10.948},"last":2.3730000000000002},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.1319999999999999,"15min":1.831},"min":{"1min":0.35899999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":4.593,"5min":8.8160000000000007,"15min":10.667},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.1919999999999999,"15min":2.1230000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.0529999999999999,"5min":8.1539999999999999,"15min":10.693},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.1150000000000002,"5min":1.6519999999999999,"15min":2.4119999999999999},"min":{"1min":0.46500000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0720000000000001,"5min":8.4819999999999993,"15min":17.460000000000001},"last":4.9100000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.1799999999999999,"15min":2.1829999999999998},"min":{"1min":0.49099999999999999,"5min":0.42799999999999999,"15min":0.35899999999999999},"max":{"1min":4.5570000000000004,"5min":11.848000000000001,"15min":11.848000000000001},"last":5.0650000000000004},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.615,"15min":2.4990000000000001},"min":{"1min":0.48699999999999999,"5min":0.41899999999999998,"15min":0.38500000000000001},"max":{"1min":5.5999999999999996,"5min":8.4350000000000005,"15min":17.411000000000001},"last":5.3239999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:03 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.593,"15min":2.4940000000000002},"min":{"1min":0.42899999999999999,"5min":0.39200000000000002,"15min":0.36399999999999999},"max":{"1min":5.5659999999999998,"5min":8.5269999999999992,"15min":17.486999999999998},"last":5.3570000000000002},{"interface":"front","average":{"1min":2.1899999999999999,"5min":1.522,"15min":2.0950000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.36799999999999999},"max":{"1min":10.691000000000001,"5min":10.691000000000001,"15min":10.867000000000001},"last":2.4009999999999998}]}]},{"osd":2,"up_from":120,"seq":515396075570,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1163088,"kb_used_data":988360,"kb_used_omap":7,"kb_used_meta":174712,"kb_avail":92582064,"statfs":{"total":95995035648,"available":94804033536,"internally_reserved":0,"allocated":1012080640,"data_stored":1011532972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":178905169},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":0.96099999999999997,"15min":0.57199999999999995},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":31.873000000000001,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.26200000000000001},{"interface":"front","average":{"1min":3.27,"5min":0.96299999999999997,"15min":0.56399999999999995},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.249},"max":{"1min":31.282,"5min":31.282,"15min":31.282},"last":5.6920000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3559999999999999,"5min":0.97899999999999998,"15min":0.58299999999999996},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":29.486000000000001,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.371},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.01,"15min":0.61199999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":28.013999999999999,"5min":28.013999999999999,"15min":28.013999999999999},"last":10.201000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":7.1050000000000004,"5min":2.1480000000000001,"15min":0.98199999999999998},"min":{"1min":0.27100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":96.295000000000002,"5min":96.295000000000002,"15min":96.295000000000002},"last":0.29599999999999999},{"interface":"front","average":{"1min":9.7989999999999995,"5min":2.633,"15min":1.161},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":99.504999999999995,"5min":99.504999999999995,"15min":99.504999999999995},"last":5.7469999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":0.85499999999999998,"15min":0.72199999999999998},"min":{"1min":0.39900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":4.8220000000000001,"5min":4.8220000000000001,"15min":4.8220000000000001},"last":1.26},{"interface":"front","average":{"1min":1.9470000000000001,"5min":0.91000000000000003,"15min":0.747},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4290000000000003,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.056,"15min":0.80900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":10.166,"5min":10.166,"15min":10.166},"last":10.254},{"interface":"front","average":{"1min":2.5,"5min":1.0389999999999999,"15min":0.79500000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":6.7489999999999997,"5min":6.7489999999999997,"15min":6.7489999999999997},"last":1.3160000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":0.996,"15min":0.79800000000000004},"min":{"1min":0.439,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":7.7130000000000001,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":6.016},{"interface":"front","average":{"1min":2.0910000000000002,"5min":0.998,"15min":0.81200000000000006},"min":{"1min":0.50800000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":7.4649999999999999,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":0.96299999999999997,"15min":0.81100000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.1070000000000002,"5min":7.3550000000000004,"15min":7.3550000000000004},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.992,"5min":0.995,"15min":0.81599999999999995},"min":{"1min":0.47399999999999998,"5min":0.44,"15min":0.44},"max":{"1min":7.6609999999999996,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":5.7809999999999997}]}]},{"osd":3,"up_from":125,"seq":536870912044,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2887036,"kb_used_data":2708788,"kb_used_omap":2,"kb_used_meta":178237,"kb_avail":90858116,"statfs":{"total":95995035648,"available":93038710784,"internally_reserved":0,"allocated":2773798912,"data_stored":2773263216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":182514741},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":0.93899999999999995,"15min":0.55400000000000005},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.6460000000000008,"5min":8.6460000000000008,"15min":8.6460000000000008},"last":0.98299999999999998},{"interface":"front","average":{"1min":2.758,"5min":1.016,"15min":0.57399999999999995},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.669,"5min":12.669,"15min":12.669},"last":9.0860000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":4.9820000000000002,"5min":1.4379999999999999,"15min":0.748},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.112000000000002,"5min":54.112000000000002,"15min":54.112000000000002},"last":1.016},{"interface":"front","average":{"1min":5.734,"5min":1.639,"15min":0.82299999999999995},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":57.814,"5min":57.814,"15min":57.814},"last":1.0880000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":0.96099999999999997,"15min":0.57999999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":12.726000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":9.1910000000000007},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.0409999999999999,"15min":0.60899999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.091,"15min":0.747},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":4.694},{"interface":"front","average":{"1min":2.4460000000000002,"5min":1.03,"15min":0.72599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.4800000000000004,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":2.6539999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.0620000000000001,"15min":0.752},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.975,"5min":10.975,"15min":10.975},"last":3.3580000000000001},{"interface":"front","average":{"1min":2.827,"5min":1.163,"15min":0.79400000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.423,"5min":8.423,"15min":8.423},"last":3.4169999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":1.1879999999999999,"15min":0.79000000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":8.5050000000000008,"5min":8.5050000000000008,"15min":8.5050000000000008},"last":9.1210000000000004},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.0529999999999999,"15min":0.79500000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":7.1799999999999997},"last":9.0380000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":1.1799999999999999,"15min":0.80100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":8.3849999999999998,"5min":8.3849999999999998,"15min":8.3849999999999998},"last":9.1679999999999993},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.085,"15min":0.81000000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.3579999999999997,"5min":7.3579999999999997,"15min":7.3579999999999997},"last":8.9930000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019195,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1362320,"kb_used_data":1004680,"kb_used_omap":0,"kb_used_meta":357632,"kb_avail":92382832,"statfs":{"total":95995035648,"available":94600019968,"internally_reserved":0,"allocated":1028792320,"data_stored":1028304712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":366215168},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":5.9900000000000002,"5min":1.857,"15min":2.1309999999999998},"min":{"1min":0.26800000000000002,"5min":0.23899999999999999,"15min":0.22},"max":{"1min":40.186999999999998,"5min":40.186999999999998,"15min":40.186999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":6.1429999999999998,"5min":2.3399999999999999,"15min":2.3719999999999999},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":39.395000000000003,"5min":39.395000000000003,"15min":39.395000000000003},"last":0.39500000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:21:58 2023","interfaces":[{"interface":"back","average":{"1min":5.7309999999999999,"5min":2.0899999999999999,"15min":1.984},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":79.731999999999999,"5min":79.731999999999999,"15min":79.731999999999999},"last":11.029},{"interface":"front","average":{"1min":5.5220000000000002,"5min":2.0230000000000001,"15min":2.4079999999999999},"min":{"1min":0.30499999999999999,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":79.146000000000001,"5min":79.146000000000001,"15min":79.146000000000001},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":0.64600000000000002,"15min":0.56599999999999995},"min":{"1min":0.34200000000000003,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.625,"5min":4.625,"15min":4.625},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.3600000000000001,"5min":0.77200000000000002,"15min":0.67600000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":5.3689999999999998,"5min":5.3689999999999998,"15min":5.3689999999999998},"last":11.099}]},{"osd":3,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":10.824,"5min":2.9609999999999999,"15min":1.3580000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":92.936000000000007,"5min":92.936000000000007,"15min":92.936000000000007},"last":0.41699999999999998},{"interface":"front","average":{"1min":9.3360000000000003,"5min":2.6629999999999998,"15min":1.3009999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":92.912000000000006,"5min":92.912000000000006,"15min":92.912000000000006},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.79100000000000004,"15min":1.1659999999999999},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.23599999999999999},"max":{"1min":3.7959999999999998,"5min":6.5010000000000003,"15min":92.653000000000006},"last":10.984999999999999},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.88600000000000001,"15min":1.4399999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":4.0380000000000003,"5min":4.4279999999999999,"15min":93.281999999999996},"last":3.7189999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8490000000000002,"5min":1.1799999999999999,"15min":1.0820000000000001},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.26700000000000002},"max":{"1min":8.2599999999999998,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.9119999999999999,"5min":1.0009999999999999,"15min":0.97099999999999997},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.30399999999999999},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.8420000000000005},"last":11.055}]},{"osd":7,"last update":"Sat Jun 10 21:22:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":1.1899999999999999,"15min":1.24},"min":{"1min":0.38600000000000001,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":4.9669999999999996,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":11.077},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.1499999999999999,"15min":1.083},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.312},"max":{"1min":8.3119999999999994,"5min":8.3119999999999994,"15min":8.3119999999999994},"last":0.437}]}]},{"osd":5,"up_from":36,"seq":154618822967,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3429188,"kb_used_data":2958972,"kb_used_omap":0,"kb_used_meta":470208,"kb_avail":90315964,"statfs":{"total":95995035648,"available":92483547136,"internally_reserved":0,"allocated":3029987328,"data_stored":3029507794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":481492992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":4.9279999999999999,"5min":1.659,"15min":1.365},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":57.700000000000003,"5min":57.700000000000003,"15min":57.700000000000003},"last":4.1669999999999998},{"interface":"front","average":{"1min":4.4550000000000001,"5min":1.8660000000000001,"15min":2.1110000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":57.890999999999998,"5min":57.890999999999998,"15min":57.890999999999998},"last":3.4849999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5310000000000001,"5min":2.1190000000000002,"15min":2.2069999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.99,"5min":57.899000000000001,"15min":57.899000000000001},"last":4.1219999999999999},{"interface":"front","average":{"1min":2.2349999999999999,"5min":1.8740000000000001,"15min":2.5670000000000002},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":13.875999999999999,"5min":57.837000000000003,"15min":57.837000000000003},"last":2.8889999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":0.75700000000000001,"15min":0.65600000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.6029999999999998,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":3.4289999999999998},{"interface":"front","average":{"1min":2.1960000000000002,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.895,"5min":10.895,"15min":10.895},"last":2.9609999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:22:29 2023","interfaces":[{"interface":"back","average":{"1min":10.441000000000001,"5min":2.9289999999999998,"15min":1.387},"min":{"1min":0.442,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":128.16800000000001,"5min":128.16800000000001,"15min":128.16800000000001},"last":2.9199999999999999},{"interface":"front","average":{"1min":9.4960000000000004,"5min":2.6709999999999998,"15min":1.248},"min":{"1min":0.314,"5min":0.311,"15min":0.311},"max":{"1min":128.58000000000001,"5min":128.58000000000001,"15min":128.58000000000001},"last":4.0629999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1539999999999999,"15min":1.0669999999999999},"min":{"1min":0.30499999999999999,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":11.039999999999999,"5min":11.039999999999999,"15min":11.039999999999999},"last":11.039999999999999},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.83199999999999996,"15min":1.129},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":5.016,"5min":6.2640000000000002,"15min":12.901},"last":2.9409999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:22:14 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":0.97899999999999998,"15min":1.1020000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7640000000000002,"5min":7.6609999999999996,"15min":8.1199999999999992},"last":10.991},{"interface":"front","average":{"1min":1.028,"5min":0.83599999999999997,"15min":1.157},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":4.7549999999999999,"5min":5.3810000000000002,"15min":7.9370000000000003},"last":4.5419999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.149,"15min":1.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":6.7080000000000002,"5min":8.5109999999999992,"15min":8.5109999999999992},"last":4.5670000000000002},{"interface":"front","average":{"1min":2.1349999999999998,"5min":1.1379999999999999,"15min":1.173},"min":{"1min":0.42099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":7.1539999999999999,"5min":7.8710000000000004,"15min":8.1809999999999992},"last":11.07}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1359151104,"data_stored":1359151104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3003584512,"data_stored":3003580440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1009319936,"data_stored":1009319936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2765619200,"data_stored":2765619200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1026162688,"data_stored":1026162688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3025932288,"data_stored":3025928216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1004535808,"data_stored":1004535808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3056209920,"data_stored":3056205848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:23:05.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:23:05.663 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:05.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:05.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:05.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:05.883 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:05.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:05.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:06.027 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:06.109 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:06.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:06.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:06.252 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:06.319 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:06.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:06.357 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:06.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:06.482 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:06.504 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:06.520 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:06.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:06.593 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:06.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:06.676 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:06.718 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:06.764 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:06.768 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:06.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:06.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:06.798 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:06.810 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:06.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.830 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.831 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:23:06.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:06.932 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:07.003 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:07.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:07.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:07.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:07.164 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:07 smithi152 ceph-mon[156819]: pgmap v604: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-06-10T21:23:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1023896463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:07.236 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:07.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:07.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:07.352 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:07 smithi064 ceph-mon[180297]: pgmap v604: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-06-10T21:23:07.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1023896463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:07.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:07 smithi064 ceph-mon[186706]: pgmap v604: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-06-10T21:23:07.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1023896463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:07.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:07.643 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:07.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:07.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:08.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:08.130 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:09.009 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:09.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:09.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:09.184 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:09.282 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:09.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:09.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:09.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:09 smithi152 ceph-mon[156819]: pgmap v605: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-10T21:23:09.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:09.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:09 smithi064 ceph-mon[180297]: pgmap v605: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-10T21:23:09.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:09 smithi064 ceph-mon[186706]: pgmap v605: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-10T21:23:09.498 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:09.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:09.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:09.687 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:10.349 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:23:10.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1287596612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1287596612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1287596612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:10.737 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:10.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:10.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:10.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:23:10.908 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:11.011 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:11.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:11.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:11.176 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:11.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:11.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: pgmap v606: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-06-10T21:23:11.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:11.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:11.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: osdmap e153: 8 total, 8 up, 8 in 2023-06-10T21:23:11.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: pgmap v606: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:11.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: osdmap e153: 8 total, 8 up, 8 in 2023-06-10T21:23:11.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:23:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:11.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: pgmap v606: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: failed to encode map e153 with expected crc 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: osdmap e153: 8 total, 8 up, 8 in 2023-06-10T21:23:11.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:23:11.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:11.824 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:11.919 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:11.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:11.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:12.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:12.101 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:12.316 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:12.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:12.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:12.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:12.510 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:12.817 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:23:12.817 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:23:12.825 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:12.888 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:12.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:13.007 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:13.079 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:13.137 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:13.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:12] "GET /metrics HTTP/1.1" 200 223527 "" "Prometheus/2.33.4" 2023-06-10T21:23:13.250 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:23:13.252 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:23:13.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:13.280 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:13.332 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:13.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:13 smithi152 ceph-mon[156819]: pgmap v608: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:13.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:13.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:13.500 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:13 smithi064 ceph-mon[180297]: pgmap v608: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:13.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:13 smithi064 ceph-mon[186706]: pgmap v608: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:13.521 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:14.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:23:14.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:23:14.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:14 smithi152 ceph-mon[156819]: from='client.55423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:14.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:14 smithi064 ceph-mon[186706]: from='client.55423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:14.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:14 smithi064 ceph-mon[180297]: from='client.55423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:14.585 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:14.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:14.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:14.755 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":608,"stamp":"2023-06-10T21:23:11.481724+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4993259430,"num_objects":76207,"num_object_clones":0,"num_object_copies":228621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76207,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":142050,"num_write_kb":6977487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131184,"ondisk_log_size":131184,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17219568,"kb_used_data":14675504,"kb_used_omap":20,"kb_used_meta":2543787,"kb_avail":732741648,"statfs":{"total":767960285184,"available":750327447552,"internally_reserved":0,"allocated":15027716096,"data_stored":15023476739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2604838542},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-443744256,"num_objects":-6771,"num_object_clones":0,"num_object_copies":-20313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6771,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.886208"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":135,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565941+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:22:57.565941+0000","last_peered":"2023-06-10T21:22:57.565941+0000","last_clean":"2023-06-10T21:22:57.565941+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:22:57.565941+0000","last_undegraded":"2023-06-10T21:22:57.565941+0000","last_fullsized":"2023-06-10T21:22:57.565941+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574158+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:22:57.574158+0000","last_peered":"2023-06-10T21:22:57.574158+0000","last_clean":"2023-06-10T21:22:57.574158+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:22:57.574158+0000","last_undegraded":"2023-06-10T21:22:57.574158+0000","last_fullsized":"2023-06-10T21:22:57.574158+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612518+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:23:01.612518+0000","last_peered":"2023-06-10T21:23:01.612518+0000","last_clean":"2023-06-10T21:23:01.612518+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:23:01.612518+0000","last_undegraded":"2023-06-10T21:23:01.612518+0000","last_fullsized":"2023-06-10T21:23:01.612518+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":108,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592537+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:22:57.592537+0000","last_peered":"2023-06-10T21:22:57.592537+0000","last_clean":"2023-06-10T21:22:57.592537+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:22:57.592537+0000","last_undegraded":"2023-06-10T21:22:57.592537+0000","last_fullsized":"2023-06-10T21:22:57.592537+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":128,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575730+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:22:57.575730+0000","last_peered":"2023-06-10T21:22:57.575730+0000","last_clean":"2023-06-10T21:22:57.575730+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:22:57.575730+0000","last_undegraded":"2023-06-10T21:22:57.575730+0000","last_fullsized":"2023-06-10T21:22:57.575730+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589254+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:22:57.589254+0000","last_peered":"2023-06-10T21:22:57.589254+0000","last_clean":"2023-06-10T21:22:57.589254+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:22:57.589254+0000","last_undegraded":"2023-06-10T21:22:57.589254+0000","last_fullsized":"2023-06-10T21:22:57.589254+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592928+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:22:57.592928+0000","last_peered":"2023-06-10T21:22:57.592928+0000","last_clean":"2023-06-10T21:22:57.592928+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:22:57.592928+0000","last_undegraded":"2023-06-10T21:22:57.592928+0000","last_fullsized":"2023-06-10T21:22:57.592928+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575641+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:22:57.575641+0000","last_peered":"2023-06-10T21:22:57.575641+0000","last_clean":"2023-06-10T21:22:57.575641+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:22:57.575641+0000","last_undegraded":"2023-06-10T21:22:57.575641+0000","last_fullsized":"2023-06-10T21:22:57.575641+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589125+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:22:57.589125+0000","last_peered":"2023-06-10T21:22:57.589125+0000","last_clean":"2023-06-10T21:22:57.589125+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:22:57.589125+0000","last_undegraded":"2023-06-10T21:22:57.589125+0000","last_fullsized":"2023-06-10T21:22:57.589125+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575110+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:22:57.575110+0000","last_peered":"2023-06-10T21:22:57.575110+0000","last_clean":"2023-06-10T21:22:57.575110+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:22:57.575110+0000","last_undegraded":"2023-06-10T21:22:57.575110+0000","last_fullsized":"2023-06-10T21:22:57.575110+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":122,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588352+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:22:57.588352+0000","last_peered":"2023-06-10T21:22:57.588352+0000","last_clean":"2023-06-10T21:22:57.588352+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:22:57.588352+0000","last_undegraded":"2023-06-10T21:22:57.588352+0000","last_fullsized":"2023-06-10T21:22:57.588352+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612788+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:23:01.612788+0000","last_peered":"2023-06-10T21:23:01.612788+0000","last_clean":"2023-06-10T21:23:01.612788+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:23:01.612788+0000","last_undegraded":"2023-06-10T21:23:01.612788+0000","last_fullsized":"2023-06-10T21:23:01.612788+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":127,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566485+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:22:57.566485+0000","last_peered":"2023-06-10T21:22:57.566485+0000","last_clean":"2023-06-10T21:22:57.566485+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:22:57.566485+0000","last_undegraded":"2023-06-10T21:22:57.566485+0000","last_fullsized":"2023-06-10T21:22:57.566485+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566049+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:22:57.566049+0000","last_peered":"2023-06-10T21:22:57.566049+0000","last_clean":"2023-06-10T21:22:57.566049+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:22:57.566049+0000","last_undegraded":"2023-06-10T21:22:57.566049+0000","last_fullsized":"2023-06-10T21:22:57.566049+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":121,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.598343+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:22:57.598343+0000","last_peered":"2023-06-10T21:22:57.598343+0000","last_clean":"2023-06-10T21:22:57.598343+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:22:57.598343+0000","last_undegraded":"2023-06-10T21:22:57.598343+0000","last_fullsized":"2023-06-10T21:22:57.598343+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.577418+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:22:57.577418+0000","last_peered":"2023-06-10T21:22:57.577418+0000","last_clean":"2023-06-10T21:22:57.577418+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:22:57.577418+0000","last_undegraded":"2023-06-10T21:22:57.577418+0000","last_fullsized":"2023-06-10T21:22:57.577418+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592777+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:22:57.592777+0000","last_peered":"2023-06-10T21:22:57.592777+0000","last_clean":"2023-06-10T21:22:57.592777+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:22:57.592777+0000","last_undegraded":"2023-06-10T21:22:57.592777+0000","last_fullsized":"2023-06-10T21:22:57.592777+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589442+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:22:57.589442+0000","last_peered":"2023-06-10T21:22:57.589442+0000","last_clean":"2023-06-10T21:22:57.589442+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:22:57.589442+0000","last_undegraded":"2023-06-10T21:22:57.589442+0000","last_fullsized":"2023-06-10T21:22:57.589442+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":104,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.591666+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:22:57.591666+0000","last_peered":"2023-06-10T21:22:57.591666+0000","last_clean":"2023-06-10T21:22:57.591666+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:22:57.591666+0000","last_undegraded":"2023-06-10T21:22:57.591666+0000","last_fullsized":"2023-06-10T21:22:57.591666+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592694+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:22:57.592694+0000","last_peered":"2023-06-10T21:22:57.592694+0000","last_clean":"2023-06-10T21:22:57.592694+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:22:57.592694+0000","last_undegraded":"2023-06-10T21:22:57.592694+0000","last_fullsized":"2023-06-10T21:22:57.592694+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":127,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.591656+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:22:57.591656+0000","last_peered":"2023-06-10T21:22:57.591656+0000","last_clean":"2023-06-10T21:22:57.591656+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:22:57.591656+0000","last_undegraded":"2023-06-10T21:22:57.591656+0000","last_fullsized":"2023-06-10T21:22:57.591656+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":112,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574493+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:22:57.574493+0000","last_peered":"2023-06-10T21:22:57.574493+0000","last_clean":"2023-06-10T21:22:57.574493+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:22:57.574493+0000","last_undegraded":"2023-06-10T21:22:57.574493+0000","last_fullsized":"2023-06-10T21:22:57.574493+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"152'8815","reported_seq":8850,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:07.802194+0000","last_change":"2023-06-10T21:20:45.268089+0000","last_active":"2023-06-10T21:23:07.802194+0000","last_peered":"2023-06-10T21:23:07.802194+0000","last_clean":"2023-06-10T21:23:07.802194+0000","last_became_active":"2023-06-10T21:20:45.267772+0000","last_became_peered":"2023-06-10T21:20:45.267772+0000","last_unstale":"2023-06-10T21:23:07.802194+0000","last_undegraded":"2023-06-10T21:23:07.802194+0000","last_fullsized":"2023-06-10T21:23:07.802194+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8815,"log_dups_size":0,"ondisk_log_size":8815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:24:07.609015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8815,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":120,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592345+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:22:57.592345+0000","last_peered":"2023-06-10T21:22:57.592345+0000","last_clean":"2023-06-10T21:22:57.592345+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:22:57.592345+0000","last_undegraded":"2023-06-10T21:22:57.592345+0000","last_fullsized":"2023-06-10T21:22:57.592345+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588862+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:22:57.588862+0000","last_peered":"2023-06-10T21:22:57.588862+0000","last_clean":"2023-06-10T21:22:57.588862+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:22:57.588862+0000","last_undegraded":"2023-06-10T21:22:57.588862+0000","last_fullsized":"2023-06-10T21:22:57.588862+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"153'8996","reported_seq":9032,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.971669+0000","last_change":"2023-06-10T21:20:45.293144+0000","last_active":"2023-06-10T21:23:10.971669+0000","last_peered":"2023-06-10T21:23:10.971669+0000","last_clean":"2023-06-10T21:23:10.971669+0000","last_became_active":"2023-06-10T21:20:45.292848+0000","last_became_peered":"2023-06-10T21:20:45.292848+0000","last_unstale":"2023-06-10T21:23:10.971669+0000","last_undegraded":"2023-06-10T21:23:10.971669+0000","last_fullsized":"2023-06-10T21:23:10.971669+0000","mapping_epoch":127,"log_start":"132'1600","ondisk_log_start":"132'1600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7396,"log_dups_size":0,"ondisk_log_size":7396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:53:38.966995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314572800,"num_objects":4800,"num_object_clones":0,"num_object_copies":14400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8996,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":121,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.587856+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:22:57.587856+0000","last_peered":"2023-06-10T21:22:57.587856+0000","last_clean":"2023-06-10T21:22:57.587856+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:22:57.587856+0000","last_undegraded":"2023-06-10T21:22:57.587856+0000","last_fullsized":"2023-06-10T21:22:57.587856+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":133,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565754+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:22:57.565754+0000","last_peered":"2023-06-10T21:22:57.565754+0000","last_clean":"2023-06-10T21:22:57.565754+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:22:57.565754+0000","last_undegraded":"2023-06-10T21:22:57.565754+0000","last_fullsized":"2023-06-10T21:22:57.565754+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"152'8825","reported_seq":8858,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.554026+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:23:09.554026+0000","last_peered":"2023-06-10T21:23:09.554026+0000","last_clean":"2023-06-10T21:23:09.554026+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:23:09.554026+0000","last_undegraded":"2023-06-10T21:23:09.554026+0000","last_fullsized":"2023-06-10T21:23:09.554026+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8825,"log_dups_size":0,"ondisk_log_size":8825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307167232,"num_objects":4687,"num_object_clones":0,"num_object_copies":14061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8825,"num_write_kb":432384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":105,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588214+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:22:57.588214+0000","last_peered":"2023-06-10T21:22:57.588214+0000","last_clean":"2023-06-10T21:22:57.588214+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:22:57.588214+0000","last_undegraded":"2023-06-10T21:22:57.588214+0000","last_fullsized":"2023-06-10T21:22:57.588214+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588519+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:22:57.588519+0000","last_peered":"2023-06-10T21:22:57.588519+0000","last_clean":"2023-06-10T21:22:57.588519+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:22:57.588519+0000","last_undegraded":"2023-06-10T21:22:57.588519+0000","last_fullsized":"2023-06-10T21:22:57.588519+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"152'8881","reported_seq":8914,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.552720+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:23:09.552720+0000","last_peered":"2023-06-10T21:23:09.552720+0000","last_clean":"2023-06-10T21:23:09.552720+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:23:09.552720+0000","last_undegraded":"2023-06-10T21:23:09.552720+0000","last_fullsized":"2023-06-10T21:23:09.552720+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8881,"log_dups_size":0,"ondisk_log_size":8881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309002240,"num_objects":4715,"num_object_clones":0,"num_object_copies":14145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8881,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588985+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:22:57.588985+0000","last_peered":"2023-06-10T21:22:57.588985+0000","last_clean":"2023-06-10T21:22:57.588985+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:22:57.588985+0000","last_undegraded":"2023-06-10T21:22:57.588985+0000","last_fullsized":"2023-06-10T21:22:57.588985+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.600592+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:22:57.600592+0000","last_peered":"2023-06-10T21:22:57.600592+0000","last_clean":"2023-06-10T21:22:57.600592+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:22:57.600592+0000","last_undegraded":"2023-06-10T21:22:57.600592+0000","last_fullsized":"2023-06-10T21:22:57.600592+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"152'8779","reported_seq":8814,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:06.814662+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:23:06.814662+0000","last_peered":"2023-06-10T21:23:06.814662+0000","last_clean":"2023-06-10T21:23:06.814662+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:23:06.814662+0000","last_undegraded":"2023-06-10T21:23:06.814662+0000","last_fullsized":"2023-06-10T21:23:06.814662+0000","mapping_epoch":127,"log_start":"131'1000","ondisk_log_start":"131'1000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7779,"log_dups_size":0,"ondisk_log_size":7779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320274432,"num_objects":4887,"num_object_clones":0,"num_object_copies":14661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8779,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":96,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574823+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:22:57.574823+0000","last_peered":"2023-06-10T21:22:57.574823+0000","last_clean":"2023-06-10T21:22:57.574823+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:22:57.574823+0000","last_undegraded":"2023-06-10T21:22:57.574823+0000","last_fullsized":"2023-06-10T21:22:57.574823+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566470+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:22:57.566470+0000","last_peered":"2023-06-10T21:22:57.566470+0000","last_clean":"2023-06-10T21:22:57.566470+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:22:57.566470+0000","last_undegraded":"2023-06-10T21:22:57.566470+0000","last_fullsized":"2023-06-10T21:22:57.566470+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"153'8707","reported_seq":8743,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.986687+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:23:10.986687+0000","last_peered":"2023-06-10T21:23:10.986687+0000","last_clean":"2023-06-10T21:23:10.986687+0000","last_became_active":"2023-06-10T21:20:45.291689+0000","last_became_peered":"2023-06-10T21:20:45.291689+0000","last_unstale":"2023-06-10T21:23:10.986687+0000","last_undegraded":"2023-06-10T21:23:10.986687+0000","last_fullsized":"2023-06-10T21:23:10.986687+0000","mapping_epoch":127,"log_start":"132'1300","ondisk_log_start":"132'1300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:11:53.302103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293142528,"num_objects":4473,"num_object_clones":0,"num_object_copies":13419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8707,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588410+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:22:57.588410+0000","last_peered":"2023-06-10T21:22:57.588410+0000","last_clean":"2023-06-10T21:22:57.588410+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:22:57.588410+0000","last_undegraded":"2023-06-10T21:22:57.588410+0000","last_fullsized":"2023-06-10T21:22:57.588410+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":133,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566501+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:22:57.566501+0000","last_peered":"2023-06-10T21:22:57.566501+0000","last_clean":"2023-06-10T21:22:57.566501+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:22:57.566501+0000","last_undegraded":"2023-06-10T21:22:57.566501+0000","last_fullsized":"2023-06-10T21:22:57.566501+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"152'8995","reported_seq":9028,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.401535+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:23:09.401535+0000","last_peered":"2023-06-10T21:23:09.401535+0000","last_clean":"2023-06-10T21:23:09.401535+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:23:09.401535+0000","last_undegraded":"2023-06-10T21:23:09.401535+0000","last_fullsized":"2023-06-10T21:23:09.401535+0000","mapping_epoch":127,"log_start":"131'1300","ondisk_log_start":"131'1300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7695,"log_dups_size":0,"ondisk_log_size":7695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314638336,"num_objects":4801,"num_object_clones":0,"num_object_copies":14403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8995,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565662+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:22:57.565662+0000","last_peered":"2023-06-10T21:22:57.565662+0000","last_clean":"2023-06-10T21:22:57.565662+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:22:57.565662+0000","last_undegraded":"2023-06-10T21:22:57.565662+0000","last_fullsized":"2023-06-10T21:22:57.565662+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566605+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:22:57.566605+0000","last_peered":"2023-06-10T21:22:57.566605+0000","last_clean":"2023-06-10T21:22:57.566605+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:22:57.566605+0000","last_undegraded":"2023-06-10T21:22:57.566605+0000","last_fullsized":"2023-06-10T21:22:57.566605+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"152'8880","reported_seq":8914,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:06.772299+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:23:06.772299+0000","last_peered":"2023-06-10T21:23:06.772299+0000","last_clean":"2023-06-10T21:23:06.772299+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:23:06.772299+0000","last_undegraded":"2023-06-10T21:23:06.772299+0000","last_fullsized":"2023-06-10T21:23:06.772299+0000","mapping_epoch":127,"log_start":"131'1100","ondisk_log_start":"131'1100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7780,"log_dups_size":0,"ondisk_log_size":7780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318504960,"num_objects":4860,"num_object_clones":0,"num_object_copies":14580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8880,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":113,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575819+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:22:57.575819+0000","last_peered":"2023-06-10T21:22:57.575819+0000","last_clean":"2023-06-10T21:22:57.575819+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:22:57.575819+0000","last_undegraded":"2023-06-10T21:22:57.575819+0000","last_fullsized":"2023-06-10T21:22:57.575819+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612837+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:23:01.612837+0000","last_peered":"2023-06-10T21:23:01.612837+0000","last_clean":"2023-06-10T21:23:01.612837+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:23:01.612837+0000","last_undegraded":"2023-06-10T21:23:01.612837+0000","last_fullsized":"2023-06-10T21:23:01.612837+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"152'8788","reported_seq":8821,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.535053+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:23:09.535053+0000","last_peered":"2023-06-10T21:23:09.535053+0000","last_clean":"2023-06-10T21:23:09.535053+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:23:09.535053+0000","last_undegraded":"2023-06-10T21:23:09.535053+0000","last_fullsized":"2023-06-10T21:23:09.535053+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8788,"log_dups_size":0,"ondisk_log_size":8788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308281344,"num_objects":4704,"num_object_clones":0,"num_object_copies":14112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8788,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613002+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:23:01.613002+0000","last_peered":"2023-06-10T21:23:01.613002+0000","last_clean":"2023-06-10T21:23:01.613002+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:23:01.613002+0000","last_undegraded":"2023-06-10T21:23:01.613002+0000","last_fullsized":"2023-06-10T21:23:01.613002+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":134,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589517+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:22:57.589517+0000","last_peered":"2023-06-10T21:22:57.589517+0000","last_clean":"2023-06-10T21:22:57.589517+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:22:57.589517+0000","last_undegraded":"2023-06-10T21:22:57.589517+0000","last_fullsized":"2023-06-10T21:22:57.589517+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":98,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.624396+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:23:01.624396+0000","last_peered":"2023-06-10T21:23:01.624396+0000","last_clean":"2023-06-10T21:23:01.624396+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:23:01.624396+0000","last_undegraded":"2023-06-10T21:23:01.624396+0000","last_fullsized":"2023-06-10T21:23:01.624396+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"152'8730","reported_seq":8765,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:07.822583+0000","last_change":"2023-06-10T21:20:45.269372+0000","last_active":"2023-06-10T21:23:07.822583+0000","last_peered":"2023-06-10T21:23:07.822583+0000","last_clean":"2023-06-10T21:23:07.822583+0000","last_became_active":"2023-06-10T21:20:45.269098+0000","last_became_peered":"2023-06-10T21:20:45.269098+0000","last_unstale":"2023-06-10T21:23:07.822583+0000","last_undegraded":"2023-06-10T21:23:07.822583+0000","last_fullsized":"2023-06-10T21:23:07.822583+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8730,"log_dups_size":0,"ondisk_log_size":8730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:23:51.802626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314966016,"num_objects":4806,"num_object_clones":0,"num_object_copies":14418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8730,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.591127+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:22:57.591127+0000","last_peered":"2023-06-10T21:22:57.591127+0000","last_clean":"2023-06-10T21:22:57.591127+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:22:57.591127+0000","last_undegraded":"2023-06-10T21:22:57.591127+0000","last_fullsized":"2023-06-10T21:22:57.591127+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566181+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:22:57.566181+0000","last_peered":"2023-06-10T21:22:57.566181+0000","last_clean":"2023-06-10T21:22:57.566181+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:22:57.566181+0000","last_undegraded":"2023-06-10T21:22:57.566181+0000","last_fullsized":"2023-06-10T21:22:57.566181+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613072+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:23:01.613072+0000","last_peered":"2023-06-10T21:23:01.613072+0000","last_clean":"2023-06-10T21:23:01.613072+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:23:01.613072+0000","last_undegraded":"2023-06-10T21:23:01.613072+0000","last_fullsized":"2023-06-10T21:23:01.613072+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"152'8983","reported_seq":9016,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.552127+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:23:09.552127+0000","last_peered":"2023-06-10T21:23:09.552127+0000","last_clean":"2023-06-10T21:23:09.552127+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:23:09.552127+0000","last_undegraded":"2023-06-10T21:23:09.552127+0000","last_fullsized":"2023-06-10T21:23:09.552127+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8983,"log_dups_size":0,"ondisk_log_size":8983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307167232,"num_objects":4687,"num_object_clones":0,"num_object_copies":14061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8983,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.623632+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:23:01.623632+0000","last_peered":"2023-06-10T21:23:01.623632+0000","last_clean":"2023-06-10T21:23:01.623632+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:23:01.623632+0000","last_undegraded":"2023-06-10T21:23:01.623632+0000","last_fullsized":"2023-06-10T21:23:01.623632+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613490+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:23:01.613490+0000","last_peered":"2023-06-10T21:23:01.613490+0000","last_clean":"2023-06-10T21:23:01.613490+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:23:01.613490+0000","last_undegraded":"2023-06-10T21:23:01.613490+0000","last_fullsized":"2023-06-10T21:23:01.613490+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":116,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612916+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:23:01.612916+0000","last_peered":"2023-06-10T21:23:01.612916+0000","last_clean":"2023-06-10T21:23:01.612916+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:23:01.612916+0000","last_undegraded":"2023-06-10T21:23:01.612916+0000","last_fullsized":"2023-06-10T21:23:01.612916+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"152'8665","reported_seq":8700,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:06.821593+0000","last_change":"2023-06-10T21:20:45.262983+0000","last_active":"2023-06-10T21:23:06.821593+0000","last_peered":"2023-06-10T21:23:06.821593+0000","last_clean":"2023-06-10T21:23:06.821593+0000","last_became_active":"2023-06-10T21:20:45.262641+0000","last_became_peered":"2023-06-10T21:20:45.262641+0000","last_unstale":"2023-06-10T21:23:06.821593+0000","last_undegraded":"2023-06-10T21:23:06.821593+0000","last_fullsized":"2023-06-10T21:23:06.821593+0000","mapping_epoch":127,"log_start":"131'900","ondisk_log_start":"131'900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7765,"log_dups_size":0,"ondisk_log_size":7765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:27:45.171697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306774016,"num_objects":4681,"num_object_clones":0,"num_object_copies":14043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8665,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.590758+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:22:57.590758+0000","last_peered":"2023-06-10T21:22:57.590758+0000","last_clean":"2023-06-10T21:22:57.590758+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:22:57.590758+0000","last_undegraded":"2023-06-10T21:22:57.590758+0000","last_fullsized":"2023-06-10T21:22:57.590758+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":355,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.587511+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:22:57.587511+0000","last_peered":"2023-06-10T21:22:57.587511+0000","last_clean":"2023-06-10T21:22:57.587511+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:22:57.587511+0000","last_undegraded":"2023-06-10T21:22:57.587511+0000","last_fullsized":"2023-06-10T21:22:57.587511+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575449+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:22:57.575449+0000","last_peered":"2023-06-10T21:22:57.575449+0000","last_clean":"2023-06-10T21:22:57.575449+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:22:57.575449+0000","last_undegraded":"2023-06-10T21:22:57.575449+0000","last_fullsized":"2023-06-10T21:22:57.575449+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588927+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:22:57.588927+0000","last_peered":"2023-06-10T21:22:57.588927+0000","last_clean":"2023-06-10T21:22:57.588927+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:22:57.588927+0000","last_undegraded":"2023-06-10T21:22:57.588927+0000","last_fullsized":"2023-06-10T21:22:57.588927+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"152'8937","reported_seq":8970,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:07.788146+0000","last_change":"2023-06-10T21:20:45.259115+0000","last_active":"2023-06-10T21:23:07.788146+0000","last_peered":"2023-06-10T21:23:07.788146+0000","last_clean":"2023-06-10T21:23:07.788146+0000","last_became_active":"2023-06-10T21:20:45.258813+0000","last_became_peered":"2023-06-10T21:20:45.258813+0000","last_unstale":"2023-06-10T21:23:07.788146+0000","last_undegraded":"2023-06-10T21:23:07.788146+0000","last_fullsized":"2023-06-10T21:23:07.788146+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8937,"log_dups_size":0,"ondisk_log_size":8937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:29.551738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314114048,"num_objects":4793,"num_object_clones":0,"num_object_copies":14379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8937,"num_write_kb":439360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":138,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.587505+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:22:57.587505+0000","last_peered":"2023-06-10T21:22:57.587505+0000","last_clean":"2023-06-10T21:22:57.587505+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:22:57.587505+0000","last_undegraded":"2023-06-10T21:22:57.587505+0000","last_fullsized":"2023-06-10T21:22:57.587505+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":107,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592701+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:22:57.592701+0000","last_peered":"2023-06-10T21:22:57.592701+0000","last_clean":"2023-06-10T21:22:57.592701+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:22:57.592701+0000","last_undegraded":"2023-06-10T21:22:57.592701+0000","last_fullsized":"2023-06-10T21:22:57.592701+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592613+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:22:57.592613+0000","last_peered":"2023-06-10T21:22:57.592613+0000","last_clean":"2023-06-10T21:22:57.592613+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:22:57.592613+0000","last_undegraded":"2023-06-10T21:22:57.592613+0000","last_fullsized":"2023-06-10T21:22:57.592613+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"152'8996","reported_seq":9029,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.429656+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:23:09.429656+0000","last_peered":"2023-06-10T21:23:09.429656+0000","last_clean":"2023-06-10T21:23:09.429656+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:23:09.429656+0000","last_undegraded":"2023-06-10T21:23:09.429656+0000","last_fullsized":"2023-06-10T21:23:09.429656+0000","mapping_epoch":127,"log_start":"132'1300","ondisk_log_start":"132'1300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7696,"log_dups_size":0,"ondisk_log_size":7696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309854208,"num_objects":4728,"num_object_clones":0,"num_object_copies":14184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8996,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.588279+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:22:57.588279+0000","last_peered":"2023-06-10T21:22:57.588279+0000","last_clean":"2023-06-10T21:22:57.588279+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:22:57.588279+0000","last_undegraded":"2023-06-10T21:22:57.588279+0000","last_fullsized":"2023-06-10T21:22:57.588279+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613146+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:23:01.613146+0000","last_peered":"2023-06-10T21:23:01.613146+0000","last_clean":"2023-06-10T21:23:01.613146+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:23:01.613146+0000","last_undegraded":"2023-06-10T21:23:01.613146+0000","last_fullsized":"2023-06-10T21:23:01.613146+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":92,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.600465+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:22:57.600465+0000","last_peered":"2023-06-10T21:22:57.600465+0000","last_clean":"2023-06-10T21:22:57.600465+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:22:57.600465+0000","last_undegraded":"2023-06-10T21:22:57.600465+0000","last_fullsized":"2023-06-10T21:22:57.600465+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"152'9077","reported_seq":9111,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:09.331537+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:23:09.331537+0000","last_peered":"2023-06-10T21:23:09.331537+0000","last_clean":"2023-06-10T21:23:09.331537+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:23:09.331537+0000","last_undegraded":"2023-06-10T21:23:09.331537+0000","last_fullsized":"2023-06-10T21:23:09.331537+0000","mapping_epoch":127,"log_start":"131'1300","ondisk_log_start":"131'1300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7777,"log_dups_size":0,"ondisk_log_size":7777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319946776,"num_objects":4883,"num_object_clones":0,"num_object_copies":14649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4883,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9077,"num_write_kb":446657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":96,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574611+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:22:57.574611+0000","last_peered":"2023-06-10T21:22:57.574611+0000","last_clean":"2023-06-10T21:22:57.574611+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:22:57.574611+0000","last_undegraded":"2023-06-10T21:22:57.574611+0000","last_fullsized":"2023-06-10T21:22:57.574611+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613070+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:23:01.613070+0000","last_peered":"2023-06-10T21:23:01.613070+0000","last_clean":"2023-06-10T21:23:01.613070+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:23:01.613070+0000","last_undegraded":"2023-06-10T21:23:01.613070+0000","last_fullsized":"2023-06-10T21:23:01.613070+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575726+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:22:57.575726+0000","last_peered":"2023-06-10T21:22:57.575726+0000","last_clean":"2023-06-10T21:22:57.575726+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:22:57.575726+0000","last_undegraded":"2023-06-10T21:22:57.575726+0000","last_fullsized":"2023-06-10T21:22:57.575726+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"152'8735","reported_seq":8770,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:06.780568+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:23:06.780568+0000","last_peered":"2023-06-10T21:23:06.780568+0000","last_clean":"2023-06-10T21:23:06.780568+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:23:06.780568+0000","last_undegraded":"2023-06-10T21:23:06.780568+0000","last_fullsized":"2023-06-10T21:23:06.780568+0000","mapping_epoch":127,"log_start":"131'1000","ondisk_log_start":"131'1000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8735,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":123,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589186+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:22:57.589186+0000","last_peered":"2023-06-10T21:22:57.589186+0000","last_clean":"2023-06-10T21:22:57.589186+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:22:57.589186+0000","last_undegraded":"2023-06-10T21:22:57.589186+0000","last_fullsized":"2023-06-10T21:22:57.589186+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":111,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592453+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:22:57.592453+0000","last_peered":"2023-06-10T21:22:57.592453+0000","last_clean":"2023-06-10T21:22:57.592453+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:22:57.592453+0000","last_undegraded":"2023-06-10T21:22:57.592453+0000","last_fullsized":"2023-06-10T21:22:57.592453+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565649+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:22:57.565649+0000","last_peered":"2023-06-10T21:22:57.565649+0000","last_clean":"2023-06-10T21:22:57.565649+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:22:57.565649+0000","last_undegraded":"2023-06-10T21:22:57.565649+0000","last_fullsized":"2023-06-10T21:22:57.565649+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":95,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.600668+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:22:57.600668+0000","last_peered":"2023-06-10T21:22:57.600668+0000","last_clean":"2023-06-10T21:22:57.600668+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:22:57.600668+0000","last_undegraded":"2023-06-10T21:22:57.600668+0000","last_fullsized":"2023-06-10T21:22:57.600668+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.563296+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:22:57.563296+0000","last_peered":"2023-06-10T21:22:57.563296+0000","last_clean":"2023-06-10T21:22:57.563296+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:22:57.563296+0000","last_undegraded":"2023-06-10T21:22:57.563296+0000","last_fullsized":"2023-06-10T21:22:57.563296+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":121,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589116+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:22:57.589116+0000","last_peered":"2023-06-10T21:22:57.589116+0000","last_clean":"2023-06-10T21:22:57.589116+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:22:57.589116+0000","last_undegraded":"2023-06-10T21:22:57.589116+0000","last_fullsized":"2023-06-10T21:22:57.589116+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":133,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.566223+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:22:57.566223+0000","last_peered":"2023-06-10T21:22:57.566223+0000","last_clean":"2023-06-10T21:22:57.566223+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:22:57.566223+0000","last_undegraded":"2023-06-10T21:22:57.566223+0000","last_fullsized":"2023-06-10T21:22:57.566223+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":106,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.589598+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:22:57.589598+0000","last_peered":"2023-06-10T21:22:57.589598+0000","last_clean":"2023-06-10T21:22:57.589598+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:22:57.589598+0000","last_undegraded":"2023-06-10T21:22:57.589598+0000","last_fullsized":"2023-06-10T21:22:57.589598+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565988+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:22:57.565988+0000","last_peered":"2023-06-10T21:22:57.565988+0000","last_clean":"2023-06-10T21:22:57.565988+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:22:57.565988+0000","last_undegraded":"2023-06-10T21:22:57.565988+0000","last_fullsized":"2023-06-10T21:22:57.565988+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":128,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.574098+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:22:57.574098+0000","last_peered":"2023-06-10T21:22:57.574098+0000","last_clean":"2023-06-10T21:22:57.574098+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:22:57.574098+0000","last_undegraded":"2023-06-10T21:22:57.574098+0000","last_fullsized":"2023-06-10T21:22:57.574098+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.575046+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:22:57.575046+0000","last_peered":"2023-06-10T21:22:57.575046+0000","last_clean":"2023-06-10T21:22:57.575046+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:22:57.575046+0000","last_undegraded":"2023-06-10T21:22:57.575046+0000","last_fullsized":"2023-06-10T21:22:57.575046+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":111,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.592800+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:22:57.592800+0000","last_peered":"2023-06-10T21:22:57.592800+0000","last_clean":"2023-06-10T21:22:57.592800+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:22:57.592800+0000","last_undegraded":"2023-06-10T21:22:57.592800+0000","last_fullsized":"2023-06-10T21:22:57.592800+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":151,"state":"active+clean","last_fresh":"2023-06-10T21:22:57.565646+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:22:57.565646+0000","last_peered":"2023-06-10T21:22:57.565646+0000","last_clean":"2023-06-10T21:22:57.565646+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:22:57.565646+0000","last_undegraded":"2023-06-10T21:22:57.565646+0000","last_fullsized":"2023-06-10T21:22:57.565646+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4992663576,"num_objects":76183,"num_object_clones":0,"num_object_copies":228549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76183,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141789,"num_write_kb":6975041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14923214848,"data_stored":14923202632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130989,"ondisk_log_size":130989,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430513,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3138600,"kb_used_data":2760608,"kb_used_omap":0,"kb_used_meta":377984,"kb_avail":90606552,"statfs":{"total":95995035648,"available":92781109248,"internally_reserved":0,"allocated":2826862592,"data_stored":2826359111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":387055616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:12 2023","interfaces":[{"interface":"back","average":{"1min":4.7240000000000002,"5min":1.7909999999999999,"15min":1.5509999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.22800000000000001},"max":{"1min":53.706000000000003,"5min":53.706000000000003,"15min":53.706000000000003},"last":0.42999999999999999},{"interface":"front","average":{"1min":4.2599999999999998,"5min":1.776,"15min":1.998},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.245},"max":{"1min":53.777000000000001,"5min":53.777000000000001,"15min":53.777000000000001},"last":6.0750000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":10.936,"5min":3.6429999999999998,"15min":2.3359999999999999},"min":{"1min":0.32100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":130.19399999999999,"5min":130.19399999999999,"15min":130.19399999999999},"last":2.012},{"interface":"front","average":{"1min":21.949000000000002,"5min":5.9269999999999996,"15min":3.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":212.20599999999999,"5min":212.20599999999999,"15min":212.20599999999999},"last":5.2889999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.204,"15min":0.746},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":6.6189999999999998,"5min":9.1609999999999996,"15min":9.1609999999999996},"last":0.55400000000000005},{"interface":"front","average":{"1min":13.19,"5min":3.2930000000000001,"15min":1.4530000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":212.13399999999999,"5min":212.13399999999999,"15min":212.13399999999999},"last":1.857}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":6.6340000000000003,"5min":2.4249999999999998,"15min":1.1830000000000001},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":43.911000000000001,"5min":57.085000000000001,"15min":57.085000000000001},"last":5.9329999999999998},{"interface":"front","average":{"1min":6.1660000000000004,"5min":2.4430000000000001,"15min":1.2050000000000001},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":44.225999999999999,"5min":57.167999999999999,"15min":57.167999999999999},"last":5.9779999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.68500000000000005,"15min":1.929},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":4.0170000000000003,"5min":4.0170000000000003,"15min":219.17699999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.0109999999999999,"15min":2.1680000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":9.2129999999999992,"5min":9.2129999999999992,"15min":219.13999999999999},"last":0.33700000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":0.91200000000000003,"15min":2.5419999999999998},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":4.7249999999999996,"5min":4.7619999999999996,"15min":311.202},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.3859999999999999,"5min":0.94499999999999995,"15min":2.4239999999999999},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.29699999999999999},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":311.875},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.86699999999999999,"15min":1.0649999999999999},"min":{"1min":0.39900000000000002,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.4109999999999996,"5min":4.4109999999999996,"15min":8.6630000000000003},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.2889999999999999,"5min":0.92400000000000004,"15min":1.016},"min":{"1min":0.40000000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":4.8680000000000003,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":0.39200000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626741,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213844,"kb_used_data":904460,"kb_used_omap":0,"kb_used_meta":309312,"kb_avail":92531308,"statfs":{"total":95995035648,"available":94752059392,"internally_reserved":0,"allocated":926167040,"data_stored":925683554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316735488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9650000000000001,"5min":1.173,"15min":2.1179999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":8.4730000000000008,"5min":9.5250000000000004,"15min":119.276},"last":3.153},{"interface":"front","average":{"1min":1.7509999999999999,"5min":1.226,"15min":2.609},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":4.0940000000000003,"5min":9.4580000000000002,"15min":119.22},"last":2.7890000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.3340000000000001,"15min":1.746},"min":{"1min":0.32200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9130000000000003,"5min":14.897,"15min":14.897},"last":2.2759999999999998},{"interface":"front","average":{"1min":3.0600000000000001,"5min":1.4770000000000001,"15min":1.8680000000000001},"min":{"1min":0.30599999999999999,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5369999999999999,"5min":8.0299999999999994,"15min":10.766999999999999},"last":5.1749999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.72999999999999998,"15min":0.61799999999999999},"min":{"1min":0.34899999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.944,"5min":3.0089999999999999,"15min":3.0089999999999999},"last":5.3570000000000002},{"interface":"front","average":{"1min":1.8069999999999999,"5min":0.79700000000000004,"15min":0.624},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9600000000000009,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":2.3450000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":5.8479999999999999,"5min":1.7569999999999999,"15min":0.95099999999999996},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":53.591000000000001,"5min":53.591000000000001,"15min":53.591000000000001},"last":5.3899999999999997},{"interface":"front","average":{"1min":5.1749999999999998,"5min":1.873,"15min":1.004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":53.616,"5min":53.616,"15min":53.616},"last":2.3180000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.0029999999999999,"15min":1.819},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":4.0049999999999999,"5min":6.7290000000000001,"15min":221.76900000000001},"last":2.8180000000000001},{"interface":"front","average":{"1min":1.3440000000000001,"5min":0.92900000000000005,"15min":1.974},"min":{"1min":0.34499999999999997,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":10.221,"5min":10.221,"15min":221.905},"last":5.4260000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6549999999999998,"5min":1.2889999999999999,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.25700000000000001},"max":{"1min":11.132999999999999,"5min":11.132999999999999,"15min":313.661},"last":2.8380000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.1499999999999999,"15min":2.4020000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":10.17,"5min":10.17,"15min":314.38900000000001},"last":5.4509999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.367,"15min":1.196},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":4.0679999999999996,"5min":11.179,"15min":11.179},"last":5.6760000000000002},{"interface":"front","average":{"1min":2.181,"5min":1.2190000000000001,"15min":1.2090000000000001},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":10.244999999999999,"5min":10.244999999999999,"15min":10.244999999999999},"last":5.4779999999999998}]}]},{"osd":1,"up_from":105,"seq":450971566184,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2992652,"kb_used_data":2711876,"kb_used_omap":4,"kb_used_meta":280763,"kb_avail":90752500,"statfs":{"total":95995035648,"available":92930560000,"internally_reserved":0,"allocated":2776961024,"data_stored":2776373858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":287502222},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.3809999999999998,"15min":2.129},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":29.754999999999999,"5min":86.009,"15min":86.009},"last":2.8069999999999999},{"interface":"front","average":{"1min":3.3370000000000002,"5min":2.7719999999999998,"15min":2.669},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.22600000000000001},"max":{"1min":32.079000000000001,"5min":85.869,"15min":85.869},"last":1.603}]},{"osd":2,"last update":"Sat Jun 10 21:22:11 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":0.95099999999999996,"15min":0.64800000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.6080000000000005,"5min":9.6080000000000005,"15min":9.6080000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.58299999999999996,"15min":0.50700000000000001},"min":{"1min":0.23799999999999999,"5min":0.218,"15min":0.218},"max":{"1min":6.3849999999999998,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.29599999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:22:31 2023","interfaces":[{"interface":"back","average":{"1min":12.256,"5min":2.9249999999999998,"15min":1.2150000000000001},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":118.348,"5min":118.348,"15min":118.348},"last":0.45900000000000002},{"interface":"front","average":{"1min":13.488,"5min":3.4420000000000002,"15min":1.4239999999999999},"min":{"1min":0.48499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":117.75,"5min":117.75,"15min":117.75},"last":2.911}]},{"osd":4,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.2350000000000001,"15min":1.7989999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.0199999999999996,"5min":8.0199999999999996,"15min":12.94},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.22,"15min":2.343},"min":{"1min":0.40699999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.5720000000000001,"5min":7.5720000000000001,"15min":12.603},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.55,"15min":2.0720000000000001},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":8.2550000000000008,"5min":9.7219999999999995,"15min":11.007999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.8820000000000001,"5min":1.5249999999999999,"15min":2.3130000000000002},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.309},"max":{"1min":7.9669999999999996,"5min":8.9700000000000006,"15min":13.602},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":1.4930000000000001,"15min":2.0699999999999998},"min":{"1min":0.56499999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":9.6699999999999999,"15min":12.861000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.266,"5min":1.3799999999999999,"15min":2.2610000000000001},"min":{"1min":0.48199999999999998,"5min":0.40899999999999997,"15min":0.36699999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":12.970000000000001},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":1.363,"15min":2.8090000000000002},"min":{"1min":0.45300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.1750000000000007,"5min":8.1750000000000007,"15min":13.025},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.2549999999999999,"15min":1.984},"min":{"1min":0.49199999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.6920000000000002,"5min":7.6920000000000002,"15min":19.286999999999999},"last":0.48999999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187606,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599068,"kb_used_data":1223764,"kb_used_omap":5,"kb_used_meta":375290,"kb_avail":92146084,"statfs":{"total":95995035648,"available":94357590016,"internally_reserved":0,"allocated":1253134336,"data_stored":1252525585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":384297594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:22:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1070000000000002,"5min":3.165,"15min":2.6190000000000002},"min":{"1min":0.28000000000000003,"5min":0.20899999999999999,"15min":0.19700000000000001},"max":{"1min":13.58,"5min":137.761,"15min":137.761},"last":3.044},{"interface":"front","average":{"1min":3.2519999999999998,"5min":3.1040000000000001,"15min":3.3620000000000001},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":15.045999999999999,"5min":137.88999999999999,"15min":137.88999999999999},"last":2.7320000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":4.1230000000000002,"5min":1.468,"15min":0.71499999999999997},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":8.0649999999999995},{"interface":"front","average":{"1min":3.0489999999999999,"5min":1.2330000000000001,"15min":0.64900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":16.782,"5min":16.782,"15min":16.782},"last":1.2390000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":10.523,"5min":4.3330000000000002,"15min":1.712},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":75.674999999999997,"5min":163.56999999999999,"15min":163.56999999999999},"last":7.9729999999999999},{"interface":"front","average":{"1min":10.789,"5min":4.4050000000000002,"15min":1.726},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":77.289000000000001,"5min":164.239,"15min":164.239},"last":3.2730000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":1.595,"15min":2.1469999999999998},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":9.3490000000000002,"5min":10.733000000000001,"15min":10.948},"last":8.1340000000000003},{"interface":"front","average":{"1min":2.7549999999999999,"5min":1.2609999999999999,"15min":1.849},"min":{"1min":0.39100000000000001,"5min":0.35899999999999999,"15min":0.29699999999999999},"max":{"1min":7.2309999999999999,"5min":7.2309999999999999,"15min":10.667},"last":2.9399999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.218,"15min":2.0710000000000002},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.9889999999999999,"5min":6.0529999999999999,"15min":10.693},"last":3.0070000000000001},{"interface":"front","average":{"1min":3.0379999999999998,"5min":1.534,"15min":2.4329999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.32400000000000001},"max":{"1min":7.2210000000000001,"5min":7.2210000000000001,"15min":17.460000000000001},"last":1.2150000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9900000000000002,"5min":1.3799999999999999,"15min":2.1760000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35899999999999999},"max":{"1min":10.305999999999999,"5min":10.305999999999999,"15min":11.848000000000001},"last":2.972},{"interface":"front","average":{"1min":3.3210000000000002,"5min":1.575,"15min":2.5249999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.38500000000000001},"max":{"1min":9.4109999999999996,"5min":9.4109999999999996,"15min":17.411000000000001},"last":1.175}]},{"osd":7,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":1.482,"15min":2.4969999999999999},"min":{"1min":0.45500000000000002,"5min":0.42899999999999999,"15min":0.36399999999999999},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":17.486999999999998},"last":1.264},{"interface":"front","average":{"1min":3.544,"5min":1.6240000000000001,"15min":2.1850000000000001},"min":{"1min":0.51700000000000002,"5min":0.42499999999999999,"15min":0.36799999999999999},"max":{"1min":9.3859999999999992,"5min":10.691000000000001,"15min":10.867000000000001},"last":8.1069999999999993}]}]},{"osd":2,"up_from":120,"seq":515396075572,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094360,"kb_used_data":915536,"kb_used_omap":7,"kb_used_meta":178808,"kb_avail":92650792,"statfs":{"total":95995035648,"available":94874411008,"internally_reserved":0,"allocated":937508864,"data_stored":936960754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":183099473},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":0.96099999999999997,"15min":0.57199999999999995},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":31.873000000000001,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":3.27,"5min":0.96299999999999997,"15min":0.56399999999999995},"min":{"1min":0.26300000000000001,"5min":0.249,"15min":0.249},"max":{"1min":31.282,"5min":31.282,"15min":31.282},"last":62.451999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3559999999999999,"5min":0.97899999999999998,"15min":0.58299999999999996},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":29.486000000000001,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.01,"15min":0.61199999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":28.013999999999999,"5min":28.013999999999999,"15min":28.013999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":7.1050000000000004,"5min":2.1480000000000001,"15min":0.98199999999999998},"min":{"1min":0.27100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":96.295000000000002,"5min":96.295000000000002,"15min":96.295000000000002},"last":57.802},{"interface":"front","average":{"1min":9.7989999999999995,"5min":2.633,"15min":1.161},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":99.504999999999995,"5min":99.504999999999995,"15min":99.504999999999995},"last":60.436999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":0.85499999999999998,"15min":0.72199999999999998},"min":{"1min":0.39900000000000002,"5min":0.371,"15min":0.371},"max":{"1min":4.8220000000000001,"5min":4.8220000000000001,"15min":4.8220000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.9470000000000001,"5min":0.91000000000000003,"15min":0.747},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4290000000000003,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.056,"15min":0.80900000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":10.166,"5min":10.166,"15min":10.166},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.5,"5min":1.0389999999999999,"15min":0.79500000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":6.7489999999999997,"5min":6.7489999999999997,"15min":6.7489999999999997},"last":0.496}]},{"osd":6,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":0.996,"15min":0.79800000000000004},"min":{"1min":0.439,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":7.7130000000000001,"5min":7.7130000000000001,"15min":7.7130000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.0910000000000002,"5min":0.998,"15min":0.81200000000000006},"min":{"1min":0.50800000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":7.4649999999999999,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5640000000000001,"5min":0.96299999999999997,"15min":0.81100000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":7.1070000000000002,"5min":7.3550000000000004,"15min":7.3550000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.992,"5min":0.995,"15min":0.81599999999999995},"min":{"1min":0.47399999999999998,"5min":0.44,"15min":0.44},"max":{"1min":7.6609999999999996,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.53200000000000003}]}]},{"osd":3,"up_from":125,"seq":536870912046,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2688208,"kb_used_data":2497608,"kb_used_omap":2,"kb_used_meta":190525,"kb_avail":91056944,"statfs":{"total":95995035648,"available":93242310656,"internally_reserved":0,"allocated":2557550592,"data_stored":2557010255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":195097653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":0.93899999999999995,"15min":0.55400000000000005},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.6460000000000008,"5min":8.6460000000000008,"15min":8.6460000000000008},"last":6.7489999999999997},{"interface":"front","average":{"1min":2.758,"5min":1.016,"15min":0.57399999999999995},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.669,"5min":12.669,"15min":12.669},"last":11.045999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":4.9820000000000002,"5min":1.4379999999999999,"15min":0.748},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.112000000000002,"5min":54.112000000000002,"15min":54.112000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":5.734,"5min":1.639,"15min":0.82299999999999995},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":57.814,"5min":57.814,"15min":57.814},"last":11.090999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":0.96099999999999997,"15min":0.57999999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":12.726000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":1.647},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.0409999999999999,"15min":0.60899999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":7.577}]},{"osd":4,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.091,"15min":0.747},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":1.72},{"interface":"front","average":{"1min":2.4460000000000002,"5min":1.03,"15min":0.72599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.4800000000000004,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":7.6369999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.0620000000000001,"15min":0.752},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.975,"5min":10.975,"15min":10.975},"last":1.6850000000000001},{"interface":"front","average":{"1min":2.827,"5min":1.163,"15min":0.79400000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.423,"5min":8.423,"15min":8.423},"last":7.6859999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":1.1879999999999999,"15min":0.79000000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":8.5050000000000008,"5min":8.5050000000000008,"15min":8.5050000000000008},"last":7.7089999999999996},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.0529999999999999,"15min":0.79500000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":7.1799999999999997},"last":6.6699999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":1.1799999999999999,"15min":0.80100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":8.3849999999999998,"5min":8.3849999999999998,"15min":8.3849999999999998},"last":7.6630000000000003},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.085,"15min":0.81000000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.3579999999999997,"5min":7.3579999999999997,"15min":7.3579999999999997},"last":6.7080000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019197,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285144,"kb_used_data":922448,"kb_used_omap":0,"kb_used_meta":362688,"kb_avail":92460008,"statfs":{"total":95995035648,"available":94679048192,"internally_reserved":0,"allocated":944586752,"data_stored":944098702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":371392512},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.8600000000000001,"15min":2.056},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22},"max":{"1min":7.3559999999999999,"5min":40.186999999999998,"15min":40.186999999999998},"last":1.538},{"interface":"front","average":{"1min":2.1930000000000001,"5min":2.1259999999999999,"15min":2.3330000000000002},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":6.4500000000000002,"5min":39.395000000000003,"15min":39.395000000000003},"last":4.4489999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:23:00 2023","interfaces":[{"interface":"back","average":{"1min":4.7279999999999998,"5min":2.5569999999999999,"15min":2.1739999999999999},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":24.803000000000001,"5min":79.731999999999999,"15min":79.731999999999999},"last":13.855},{"interface":"front","average":{"1min":3.2570000000000001,"5min":2.1960000000000002,"15min":2.452},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":24.613,"5min":79.146000000000001,"15min":79.146000000000001},"last":3.0910000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.88100000000000001,"15min":0.64500000000000002},"min":{"1min":0.32500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3330000000000002,"5min":7.3330000000000002,"15min":7.3330000000000002},"last":2.871},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.135,"15min":0.79700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11.099,"5min":11.099,"15min":11.099},"last":2.77}]},{"osd":3,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":10.824,"5min":2.9609999999999999,"15min":1.3580000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":92.936000000000007,"5min":92.936000000000007,"15min":92.936000000000007},"last":9.4009999999999998},{"interface":"front","average":{"1min":9.3360000000000003,"5min":2.6629999999999998,"15min":1.3009999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":92.912000000000006,"5min":92.912000000000006,"15min":92.912000000000006},"last":1.4990000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2709999999999999,"5min":1.1899999999999999,"15min":1.351},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":10.984999999999999,"5min":10.984999999999999,"15min":92.653000000000006},"last":2.7240000000000002},{"interface":"front","average":{"1min":2.681,"5min":1.1060000000000001,"15min":1.5589999999999999},"min":{"1min":0.38300000000000001,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":93.281999999999996},"last":2.556}]},{"osd":6,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8490000000000002,"5min":1.1799999999999999,"15min":1.0820000000000001},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.26700000000000002},"max":{"1min":8.2599999999999998,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":3.5},{"interface":"front","average":{"1min":1.9119999999999999,"5min":1.0009999999999999,"15min":0.97099999999999997},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.30399999999999999},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.8420000000000005},"last":2.7999999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":1.1899999999999999,"15min":1.24},"min":{"1min":0.38600000000000001,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":4.9669999999999996,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":7.157},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.1499999999999999,"15min":1.083},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.312},"max":{"1min":8.3119999999999994,"5min":8.3119999999999994,"15min":8.3119999999999994},"last":9.5060000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822969,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3207692,"kb_used_data":2739204,"kb_used_omap":0,"kb_used_meta":468416,"kb_avail":90537460,"statfs":{"total":95995035648,"available":92710359040,"internally_reserved":0,"allocated":2804944896,"data_stored":2804464920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":479657984},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":4.9279999999999999,"5min":1.659,"15min":1.365},"min":{"1min":0.246,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":57.700000000000003,"5min":57.700000000000003,"15min":57.700000000000003},"last":2.806},{"interface":"front","average":{"1min":4.4550000000000001,"5min":1.8660000000000001,"15min":2.1110000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":57.890999999999998,"5min":57.890999999999998,"15min":57.890999999999998},"last":8.8290000000000006}]},{"osd":1,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5310000000000001,"5min":2.1190000000000002,"15min":2.2069999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.99,"5min":57.899000000000001,"15min":57.899000000000001},"last":2.9750000000000001},{"interface":"front","average":{"1min":2.2349999999999999,"5min":1.8740000000000001,"15min":2.5670000000000002},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":13.875999999999999,"5min":57.837000000000003,"15min":57.837000000000003},"last":4.5869999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":0.75700000000000001,"15min":0.65600000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.6029999999999998,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":3.0209999999999999},{"interface":"front","average":{"1min":2.1960000000000002,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.895,"5min":10.895,"15min":10.895},"last":4.7110000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:22:29 2023","interfaces":[{"interface":"back","average":{"1min":10.441000000000001,"5min":2.9289999999999998,"15min":1.387},"min":{"1min":0.442,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":128.16800000000001,"5min":128.16800000000001,"15min":128.16800000000001},"last":8.8740000000000006},{"interface":"front","average":{"1min":9.4960000000000004,"5min":2.6709999999999998,"15min":1.248},"min":{"1min":0.314,"5min":0.311,"15min":0.311},"max":{"1min":128.58000000000001,"5min":128.58000000000001,"15min":128.58000000000001},"last":3.2160000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1539999999999999,"15min":1.0669999999999999},"min":{"1min":0.30499999999999999,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":11.039999999999999,"5min":11.039999999999999,"15min":11.039999999999999},"last":2.8460000000000001},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.83199999999999996,"15min":1.129},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":5.016,"5min":6.2640000000000002,"15min":12.901},"last":3.0739999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:22:14 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":0.97899999999999998,"15min":1.1020000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":4.7640000000000002,"5min":7.6609999999999996,"15min":8.1199999999999992},"last":2.8719999999999999},{"interface":"front","average":{"1min":1.028,"5min":0.83599999999999997,"15min":1.157},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":4.7549999999999999,"5min":5.3810000000000002,"15min":7.9370000000000003},"last":3.2719999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.149,"15min":1.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":6.7080000000000002,"5min":8.5109999999999992,"15min":8.5109999999999992},"last":3.2469999999999999},{"interface":"front","average":{"1min":2.1349999999999998,"5min":1.1379999999999999,"15min":1.173},"min":{"1min":0.42099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":7.1539999999999999,"5min":7.8710000000000004,"15min":8.1809999999999992},"last":9.5589999999999993}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1248460800,"data_stored":1248460800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2758545408,"data_stored":2758541336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":927334400,"data_stored":927334400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2542075904,"data_stored":2542075904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":942473216,"data_stored":942473216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2778468352,"data_stored":2778464280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":921108480,"data_stored":921108480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2804748288,"data_stored":2804744216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:23:14.762 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:23:14.763 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:23:14.763 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-06-10T21:23:14.763 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:23:14.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:14.992 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:14.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:15.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:15.174 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:15.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[180297]: pgmap v609: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-06-10T21:23:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:15.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[186706]: pgmap v609: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-06-10T21:23:15.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:15.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:15 smithi152 ceph-mon[156819]: pgmap v609: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-06-10T21:23:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:16.125 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:16.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:16.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:16.138 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:16.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:16.285 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:16.378 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:16.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:16.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:16.535 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:16.610 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:16.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:16.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:16.785 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:16.824 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:16.940 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:16.991 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:16.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:17.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:17.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:17.073 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:17.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:17.130 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:17.164 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:17.167 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:17.218 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:17.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:17.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:17.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:17.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:17.277 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.278 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.279 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:23:17.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:17.374 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:17.395 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:17 smithi064 ceph-mon[186706]: pgmap v610: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:23:17.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/889242916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:17 smithi064 ceph-mon[180297]: pgmap v610: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:23:17.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/889242916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:17 smithi152 ceph-mon[156819]: pgmap v610: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:23:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/889242916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:17.456 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:17.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:17.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:17.612 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:17.720 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:17.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:17.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:17.880 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:17.984 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:17.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:18.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:18.156 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:18.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:18.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:19.038 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:19.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:19.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:19.209 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:19.250 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:19.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:19.357 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:19.406 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:19.484 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:19.490 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:19 smithi152 ceph-mon[156819]: pgmap v611: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-10T21:23:19.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:19.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:19.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:19.649 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:19 smithi064 ceph-mon[186706]: pgmap v611: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-10T21:23:19.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:19 smithi064 ceph-mon[180297]: pgmap v611: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-10T21:23:19.714 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:20.071 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:20.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:20.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:20.255 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:20.355 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:20.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:20.460 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:20.526 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:21.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:21.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:21.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:21.486 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:21.598 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:23:21.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:21 smithi064 ceph-mon[186706]: pgmap v612: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:23:21.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:21.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:21 smithi064 ceph-mon[180297]: pgmap v612: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:23:21.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:21.690 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:21 smithi152 ceph-mon[156819]: pgmap v612: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:23:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:21.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:21.722 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:23:21.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:21.852 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:21.919 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:21.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:21.972 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:23:22.014 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:23:22.024 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:22.115 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:22.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:23:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:23:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: osdmap e154: 8 total, 8 up, 8 in 2023-06-10T21:23:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[186706]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:23:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:23:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: osdmap e154: 8 total, 8 up, 8 in 2023-06-10T21:23:22.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:22 smithi064 ceph-mon[180297]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:23:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:23:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/579875227' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: osdmap e154: 8 total, 8 up, 8 in 2023-06-10T21:23:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: failed to encode map e154 with expected crc 2023-06-10T21:23:22.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:22 smithi152 ceph-mon[156819]: failed to encode map e154 with expected crc 2023-06-10T21:23:23.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:22] "GET /metrics HTTP/1.1" 200 223531 "" "Prometheus/2.33.4" 2023-06-10T21:23:23.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:23.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:23.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:23.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:23 smithi064 ceph-mon[180297]: pgmap v613: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-10T21:23:23.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:23 smithi064 ceph-mon[180297]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:23.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:23 smithi064 ceph-mon[186706]: pgmap v613: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-10T21:23:23.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:23 smithi064 ceph-mon[186706]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:23.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:23 smithi152 ceph-mon[156819]: pgmap v613: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-10T21:23:23.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:23 smithi152 ceph-mon[156819]: from='client.45611 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:23:23.743 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:23:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:23:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:23:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:23:23.743 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:23:24.660 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":614,"stamp":"2023-06-10T21:23:21.564018+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4552923046,"num_objects":69488,"num_object_clones":0,"num_object_copies":208464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69488,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":148769,"num_write_kb":6977487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133903,"ondisk_log_size":133903,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15907432,"kb_used_data":13370536,"kb_used_omap":20,"kb_used_meta":2536107,"kb_avail":734053784,"statfs":{"total":767960285184,"available":751671074816,"internally_reserved":0,"allocated":13691428864,"data_stored":13687186413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2596974222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4],"upper_bound":8},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-440336384,"num_objects":-6719,"num_object_clones":0,"num_object_copies":-20157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6719,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.082294"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":136,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608167+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:23:01.608167+0000","last_peered":"2023-06-10T21:23:01.608167+0000","last_clean":"2023-06-10T21:23:01.608167+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:23:01.608167+0000","last_undegraded":"2023-06-10T21:23:01.608167+0000","last_fullsized":"2023-06-10T21:23:01.608167+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608598+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:23:01.608598+0000","last_peered":"2023-06-10T21:23:01.608598+0000","last_clean":"2023-06-10T21:23:01.608598+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:23:01.608598+0000","last_undegraded":"2023-06-10T21:23:01.608598+0000","last_fullsized":"2023-06-10T21:23:01.608598+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612518+0000","last_change":"2023-06-10T21:19:28.485563+0000","last_active":"2023-06-10T21:23:01.612518+0000","last_peered":"2023-06-10T21:23:01.612518+0000","last_clean":"2023-06-10T21:23:01.612518+0000","last_became_active":"2023-06-10T21:19:28.485045+0000","last_became_peered":"2023-06-10T21:19:28.485045+0000","last_unstale":"2023-06-10T21:23:01.612518+0000","last_undegraded":"2023-06-10T21:23:01.612518+0000","last_fullsized":"2023-06-10T21:23:01.612518+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:05.504691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.617540+0000","last_change":"2023-06-10T21:19:28.483727+0000","last_active":"2023-06-10T21:23:01.617540+0000","last_peered":"2023-06-10T21:23:01.617540+0000","last_clean":"2023-06-10T21:23:01.617540+0000","last_became_active":"2023-06-10T21:19:28.483373+0000","last_became_peered":"2023-06-10T21:19:28.483373+0000","last_unstale":"2023-06-10T21:23:01.617540+0000","last_undegraded":"2023-06-10T21:23:01.617540+0000","last_fullsized":"2023-06-10T21:23:01.617540+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:32:19.958547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.607927+0000","last_change":"2023-06-10T21:19:28.480757+0000","last_active":"2023-06-10T21:23:01.607927+0000","last_peered":"2023-06-10T21:23:01.607927+0000","last_clean":"2023-06-10T21:23:01.607927+0000","last_became_active":"2023-06-10T21:19:28.480396+0000","last_became_peered":"2023-06-10T21:19:28.480396+0000","last_unstale":"2023-06-10T21:23:01.607927+0000","last_undegraded":"2023-06-10T21:23:01.607927+0000","last_fullsized":"2023-06-10T21:23:01.607927+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:23:22.387582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.607530+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:23:01.607530+0000","last_peered":"2023-06-10T21:23:01.607530+0000","last_clean":"2023-06-10T21:23:01.607530+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:23:01.607530+0000","last_undegraded":"2023-06-10T21:23:01.607530+0000","last_fullsized":"2023-06-10T21:23:01.607530+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.618532+0000","last_change":"2023-06-10T21:19:28.486071+0000","last_active":"2023-06-10T21:23:01.618532+0000","last_peered":"2023-06-10T21:23:01.618532+0000","last_clean":"2023-06-10T21:23:01.618532+0000","last_became_active":"2023-06-10T21:19:28.485749+0000","last_became_peered":"2023-06-10T21:19:28.485749+0000","last_unstale":"2023-06-10T21:23:01.618532+0000","last_undegraded":"2023-06-10T21:23:01.618532+0000","last_fullsized":"2023-06-10T21:23:01.618532+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:09:40.736343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.607819+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:23:01.607819+0000","last_peered":"2023-06-10T21:23:01.607819+0000","last_clean":"2023-06-10T21:23:01.607819+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:23:01.607819+0000","last_undegraded":"2023-06-10T21:23:01.607819+0000","last_fullsized":"2023-06-10T21:23:01.607819+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.614342+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:23:01.614342+0000","last_peered":"2023-06-10T21:23:01.614342+0000","last_clean":"2023-06-10T21:23:01.614342+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:23:01.614342+0000","last_undegraded":"2023-06-10T21:23:01.614342+0000","last_fullsized":"2023-06-10T21:23:01.614342+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608693+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:23:01.608693+0000","last_peered":"2023-06-10T21:23:01.608693+0000","last_clean":"2023-06-10T21:23:01.608693+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:23:01.608693+0000","last_undegraded":"2023-06-10T21:23:01.608693+0000","last_fullsized":"2023-06-10T21:23:01.608693+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.614461+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:23:01.614461+0000","last_peered":"2023-06-10T21:23:01.614461+0000","last_clean":"2023-06-10T21:23:01.614461+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:23:01.614461+0000","last_undegraded":"2023-06-10T21:23:01.614461+0000","last_fullsized":"2023-06-10T21:23:01.614461+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612788+0000","last_change":"2023-06-10T21:19:00.496929+0000","last_active":"2023-06-10T21:23:01.612788+0000","last_peered":"2023-06-10T21:23:01.612788+0000","last_clean":"2023-06-10T21:23:01.612788+0000","last_became_active":"2023-06-10T21:19:00.496570+0000","last_became_peered":"2023-06-10T21:19:00.496570+0000","last_unstale":"2023-06-10T21:23:01.612788+0000","last_undegraded":"2023-06-10T21:23:01.612788+0000","last_fullsized":"2023-06-10T21:23:01.612788+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:48:56.873004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":128,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.604487+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:23:01.604487+0000","last_peered":"2023-06-10T21:23:01.604487+0000","last_clean":"2023-06-10T21:23:01.604487+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:23:01.604487+0000","last_undegraded":"2023-06-10T21:23:01.604487+0000","last_fullsized":"2023-06-10T21:23:01.604487+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608445+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:23:01.608445+0000","last_peered":"2023-06-10T21:23:01.608445+0000","last_clean":"2023-06-10T21:23:01.608445+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:23:01.608445+0000","last_undegraded":"2023-06-10T21:23:01.608445+0000","last_fullsized":"2023-06-10T21:23:01.608445+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.609661+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:23:01.609661+0000","last_peered":"2023-06-10T21:23:01.609661+0000","last_clean":"2023-06-10T21:23:01.609661+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:23:01.609661+0000","last_undegraded":"2023-06-10T21:23:01.609661+0000","last_fullsized":"2023-06-10T21:23:01.609661+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.611540+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:23:01.611540+0000","last_peered":"2023-06-10T21:23:01.611540+0000","last_clean":"2023-06-10T21:23:01.611540+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:23:01.611540+0000","last_undegraded":"2023-06-10T21:23:01.611540+0000","last_fullsized":"2023-06-10T21:23:01.611540+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.617420+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:23:01.617420+0000","last_peered":"2023-06-10T21:23:01.617420+0000","last_clean":"2023-06-10T21:23:01.617420+0000","last_became_active":"2023-06-10T21:19:28.483443+0000","last_became_peered":"2023-06-10T21:19:28.483443+0000","last_unstale":"2023-06-10T21:23:01.617420+0000","last_undegraded":"2023-06-10T21:23:01.617420+0000","last_fullsized":"2023-06-10T21:23:01.617420+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:58:28.539479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.619674+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:23:01.619674+0000","last_peered":"2023-06-10T21:23:01.619674+0000","last_clean":"2023-06-10T21:23:01.619674+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:23:01.619674+0000","last_undegraded":"2023-06-10T21:23:01.619674+0000","last_fullsized":"2023-06-10T21:23:01.619674+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":105,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608749+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:23:01.608749+0000","last_peered":"2023-06-10T21:23:01.608749+0000","last_clean":"2023-06-10T21:23:01.608749+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:23:01.608749+0000","last_undegraded":"2023-06-10T21:23:01.608749+0000","last_fullsized":"2023-06-10T21:23:01.608749+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.617689+0000","last_change":"2023-06-10T21:19:28.483808+0000","last_active":"2023-06-10T21:23:01.617689+0000","last_peered":"2023-06-10T21:23:01.617689+0000","last_clean":"2023-06-10T21:23:01.617689+0000","last_became_active":"2023-06-10T21:19:28.483439+0000","last_became_peered":"2023-06-10T21:19:28.483439+0000","last_unstale":"2023-06-10T21:23:01.617689+0000","last_undegraded":"2023-06-10T21:23:01.617689+0000","last_fullsized":"2023-06-10T21:23:01.617689+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:32:15.369326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":128,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608238+0000","last_change":"2023-06-10T21:10:48.968807+0000","last_active":"2023-06-10T21:23:01.608238+0000","last_peered":"2023-06-10T21:23:01.608238+0000","last_clean":"2023-06-10T21:23:01.608238+0000","last_became_active":"2023-06-10T21:10:48.968400+0000","last_became_peered":"2023-06-10T21:10:48.968400+0000","last_unstale":"2023-06-10T21:23:01.608238+0000","last_undegraded":"2023-06-10T21:23:01.608238+0000","last_fullsized":"2023-06-10T21:23:01.608238+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608952+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:23:01.608952+0000","last_peered":"2023-06-10T21:23:01.608952+0000","last_clean":"2023-06-10T21:23:01.608952+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:23:01.608952+0000","last_undegraded":"2023-06-10T21:23:01.608952+0000","last_fullsized":"2023-06-10T21:23:01.608952+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"153'9234","reported_seq":9270,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:17.822683+0000","last_change":"2023-06-10T21:20:45.268089+0000","last_active":"2023-06-10T21:23:17.822683+0000","last_peered":"2023-06-10T21:23:17.822683+0000","last_clean":"2023-06-10T21:23:17.822683+0000","last_became_active":"2023-06-10T21:20:45.267772+0000","last_became_peered":"2023-06-10T21:20:45.267772+0000","last_unstale":"2023-06-10T21:23:17.822683+0000","last_undegraded":"2023-06-10T21:23:17.822683+0000","last_fullsized":"2023-06-10T21:23:17.822683+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9234,"log_dups_size":0,"ondisk_log_size":9234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:24:07.609015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287309824,"num_objects":4384,"num_object_clones":0,"num_object_copies":13152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9234,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608909+0000","last_change":"2023-06-10T21:10:48.989212+0000","last_active":"2023-06-10T21:23:01.608909+0000","last_peered":"2023-06-10T21:23:01.608909+0000","last_clean":"2023-06-10T21:23:01.608909+0000","last_became_active":"2023-06-10T21:10:48.988782+0000","last_became_peered":"2023-06-10T21:10:48.988782+0000","last_unstale":"2023-06-10T21:23:01.608909+0000","last_undegraded":"2023-06-10T21:23:01.608909+0000","last_fullsized":"2023-06-10T21:23:01.608909+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612470+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:23:01.612470+0000","last_peered":"2023-06-10T21:23:01.612470+0000","last_clean":"2023-06-10T21:23:01.612470+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:23:01.612470+0000","last_undegraded":"2023-06-10T21:23:01.612470+0000","last_fullsized":"2023-06-10T21:23:01.612470+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"153'9448","reported_seq":9484,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:21.022239+0000","last_change":"2023-06-10T21:20:45.293144+0000","last_active":"2023-06-10T21:23:21.022239+0000","last_peered":"2023-06-10T21:23:21.022239+0000","last_clean":"2023-06-10T21:23:21.022239+0000","last_became_active":"2023-06-10T21:20:45.292848+0000","last_became_peered":"2023-06-10T21:20:45.292848+0000","last_unstale":"2023-06-10T21:23:21.022239+0000","last_undegraded":"2023-06-10T21:23:21.022239+0000","last_fullsized":"2023-06-10T21:23:21.022239+0000","mapping_epoch":127,"log_start":"132'2100","ondisk_log_start":"132'2100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:53:38.966995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284950528,"num_objects":4348,"num_object_clones":0,"num_object_copies":13044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9448,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.609116+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:23:01.609116+0000","last_peered":"2023-06-10T21:23:01.609116+0000","last_clean":"2023-06-10T21:23:01.609116+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:23:01.609116+0000","last_undegraded":"2023-06-10T21:23:01.609116+0000","last_fullsized":"2023-06-10T21:23:01.609116+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606668+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:23:01.606668+0000","last_peered":"2023-06-10T21:23:01.606668+0000","last_clean":"2023-06-10T21:23:01.606668+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:23:01.606668+0000","last_undegraded":"2023-06-10T21:23:01.606668+0000","last_fullsized":"2023-06-10T21:23:01.606668+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"153'9250","reported_seq":9284,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.546968+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:23:19.546968+0000","last_peered":"2023-06-10T21:23:19.546968+0000","last_clean":"2023-06-10T21:23:19.546968+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:23:19.546968+0000","last_undegraded":"2023-06-10T21:23:19.546968+0000","last_fullsized":"2023-06-10T21:23:19.546968+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9250,"log_dups_size":0,"ondisk_log_size":9250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279314432,"num_objects":4262,"num_object_clones":0,"num_object_copies":12786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9250,"num_write_kb":432384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":106,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.614016+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:23:01.614016+0000","last_peered":"2023-06-10T21:23:01.614016+0000","last_clean":"2023-06-10T21:23:01.614016+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:23:01.614016+0000","last_undegraded":"2023-06-10T21:23:01.614016+0000","last_fullsized":"2023-06-10T21:23:01.614016+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606896+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:23:01.606896+0000","last_peered":"2023-06-10T21:23:01.606896+0000","last_clean":"2023-06-10T21:23:01.606896+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:23:01.606896+0000","last_undegraded":"2023-06-10T21:23:01.606896+0000","last_fullsized":"2023-06-10T21:23:01.606896+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"153'9293","reported_seq":9327,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.545243+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:23:19.545243+0000","last_peered":"2023-06-10T21:23:19.545243+0000","last_clean":"2023-06-10T21:23:19.545243+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:23:19.545243+0000","last_undegraded":"2023-06-10T21:23:19.545243+0000","last_fullsized":"2023-06-10T21:23:19.545243+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9293,"log_dups_size":0,"ondisk_log_size":9293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282001408,"num_objects":4303,"num_object_clones":0,"num_object_copies":12909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9293,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612620+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:23:01.612620+0000","last_peered":"2023-06-10T21:23:01.612620+0000","last_clean":"2023-06-10T21:23:01.612620+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:23:01.612620+0000","last_undegraded":"2023-06-10T21:23:01.612620+0000","last_fullsized":"2023-06-10T21:23:01.612620+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.620033+0000","last_change":"2023-06-10T21:14:42.426180+0000","last_active":"2023-06-10T21:23:01.620033+0000","last_peered":"2023-06-10T21:23:01.620033+0000","last_clean":"2023-06-10T21:23:01.620033+0000","last_became_active":"2023-06-10T21:14:42.425872+0000","last_became_peered":"2023-06-10T21:14:42.425872+0000","last_unstale":"2023-06-10T21:23:01.620033+0000","last_undegraded":"2023-06-10T21:23:01.620033+0000","last_fullsized":"2023-06-10T21:23:01.620033+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:17.146206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"153'9207","reported_seq":9243,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:16.829213+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:23:16.829213+0000","last_peered":"2023-06-10T21:23:16.829213+0000","last_clean":"2023-06-10T21:23:16.829213+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:23:16.829213+0000","last_undegraded":"2023-06-10T21:23:16.829213+0000","last_fullsized":"2023-06-10T21:23:16.829213+0000","mapping_epoch":127,"log_start":"132'1500","ondisk_log_start":"132'1500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292225024,"num_objects":4459,"num_object_clones":0,"num_object_copies":13377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9207,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":97,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608412+0000","last_change":"2023-06-10T21:19:28.481924+0000","last_active":"2023-06-10T21:23:01.608412+0000","last_peered":"2023-06-10T21:23:01.608412+0000","last_clean":"2023-06-10T21:23:01.608412+0000","last_became_active":"2023-06-10T21:19:28.481463+0000","last_became_peered":"2023-06-10T21:19:28.481463+0000","last_unstale":"2023-06-10T21:23:01.608412+0000","last_undegraded":"2023-06-10T21:23:01.608412+0000","last_fullsized":"2023-06-10T21:23:01.608412+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:28.811194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.604393+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:23:01.604393+0000","last_peered":"2023-06-10T21:23:01.604393+0000","last_clean":"2023-06-10T21:23:01.604393+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:23:01.604393+0000","last_undegraded":"2023-06-10T21:23:01.604393+0000","last_fullsized":"2023-06-10T21:23:01.604393+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"153'9130","reported_seq":9166,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:20.995095+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:23:20.995095+0000","last_peered":"2023-06-10T21:23:20.995095+0000","last_clean":"2023-06-10T21:23:20.995095+0000","last_became_active":"2023-06-10T21:20:45.291689+0000","last_became_peered":"2023-06-10T21:20:45.291689+0000","last_unstale":"2023-06-10T21:23:20.995095+0000","last_undegraded":"2023-06-10T21:23:20.995095+0000","last_fullsized":"2023-06-10T21:23:20.995095+0000","mapping_epoch":127,"log_start":"132'1800","ondisk_log_start":"132'1800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:11:53.302103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265420800,"num_objects":4050,"num_object_clones":0,"num_object_copies":12150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9130,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606770+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:23:01.606770+0000","last_peered":"2023-06-10T21:23:01.606770+0000","last_clean":"2023-06-10T21:23:01.606770+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:23:01.606770+0000","last_undegraded":"2023-06-10T21:23:01.606770+0000","last_fullsized":"2023-06-10T21:23:01.606770+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.611414+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:23:01.611414+0000","last_peered":"2023-06-10T21:23:01.611414+0000","last_clean":"2023-06-10T21:23:01.611414+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:23:01.611414+0000","last_undegraded":"2023-06-10T21:23:01.611414+0000","last_fullsized":"2023-06-10T21:23:01.611414+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"153'9406","reported_seq":9439,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.376319+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:23:19.376319+0000","last_peered":"2023-06-10T21:23:19.376319+0000","last_clean":"2023-06-10T21:23:19.376319+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:23:19.376319+0000","last_undegraded":"2023-06-10T21:23:19.376319+0000","last_fullsized":"2023-06-10T21:23:19.376319+0000","mapping_epoch":127,"log_start":"132'1700","ondisk_log_start":"132'1700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7706,"log_dups_size":0,"ondisk_log_size":7706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287703040,"num_objects":4390,"num_object_clones":0,"num_object_copies":13170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9406,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606479+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:23:01.606479+0000","last_peered":"2023-06-10T21:23:01.606479+0000","last_clean":"2023-06-10T21:23:01.606479+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:23:01.606479+0000","last_undegraded":"2023-06-10T21:23:01.606479+0000","last_fullsized":"2023-06-10T21:23:01.606479+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.614456+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:23:01.614456+0000","last_peered":"2023-06-10T21:23:01.614456+0000","last_clean":"2023-06-10T21:23:01.614456+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:23:01.614456+0000","last_undegraded":"2023-06-10T21:23:01.614456+0000","last_fullsized":"2023-06-10T21:23:01.614456+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"153'9316","reported_seq":9352,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:16.781391+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:23:16.781391+0000","last_peered":"2023-06-10T21:23:16.781391+0000","last_clean":"2023-06-10T21:23:16.781391+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:23:16.781391+0000","last_undegraded":"2023-06-10T21:23:16.781391+0000","last_fullsized":"2023-06-10T21:23:16.781391+0000","mapping_epoch":127,"log_start":"132'1600","ondisk_log_start":"132'1600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9316,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608084+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:23:01.608084+0000","last_peered":"2023-06-10T21:23:01.608084+0000","last_clean":"2023-06-10T21:23:01.608084+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:23:01.608084+0000","last_undegraded":"2023-06-10T21:23:01.608084+0000","last_fullsized":"2023-06-10T21:23:01.608084+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612837+0000","last_change":"2023-06-10T21:19:28.484999+0000","last_active":"2023-06-10T21:23:01.612837+0000","last_peered":"2023-06-10T21:23:01.612837+0000","last_clean":"2023-06-10T21:23:01.612837+0000","last_became_active":"2023-06-10T21:19:28.484283+0000","last_became_peered":"2023-06-10T21:19:28.484283+0000","last_unstale":"2023-06-10T21:23:01.612837+0000","last_undegraded":"2023-06-10T21:23:01.612837+0000","last_fullsized":"2023-06-10T21:23:01.612837+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:41.796509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"153'9196","reported_seq":9227,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.547326+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:23:19.547326+0000","last_peered":"2023-06-10T21:23:19.547326+0000","last_clean":"2023-06-10T21:23:19.547326+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:23:19.547326+0000","last_undegraded":"2023-06-10T21:23:19.547326+0000","last_fullsized":"2023-06-10T21:23:19.547326+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9196,"log_dups_size":0,"ondisk_log_size":9196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281542656,"num_objects":4296,"num_object_clones":0,"num_object_copies":12888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9196,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613002+0000","last_change":"2023-06-10T21:14:42.430800+0000","last_active":"2023-06-10T21:23:01.613002+0000","last_peered":"2023-06-10T21:23:01.613002+0000","last_clean":"2023-06-10T21:23:01.613002+0000","last_became_active":"2023-06-10T21:14:42.429860+0000","last_became_peered":"2023-06-10T21:14:42.429860+0000","last_unstale":"2023-06-10T21:23:01.613002+0000","last_undegraded":"2023-06-10T21:23:01.613002+0000","last_fullsized":"2023-06-10T21:23:01.613002+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:10:56.715114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.619750+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:23:01.619750+0000","last_peered":"2023-06-10T21:23:01.619750+0000","last_clean":"2023-06-10T21:23:01.619750+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:23:01.619750+0000","last_undegraded":"2023-06-10T21:23:01.619750+0000","last_fullsized":"2023-06-10T21:23:01.619750+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":99,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:08.535320+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:23:08.535320+0000","last_peered":"2023-06-10T21:23:08.535320+0000","last_clean":"2023-06-10T21:23:08.535320+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:23:08.535320+0000","last_undegraded":"2023-06-10T21:23:08.535320+0000","last_fullsized":"2023-06-10T21:23:08.535320+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"153'9150","reported_seq":9186,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:17.807579+0000","last_change":"2023-06-10T21:20:45.269372+0000","last_active":"2023-06-10T21:23:17.807579+0000","last_peered":"2023-06-10T21:23:17.807579+0000","last_clean":"2023-06-10T21:23:17.807579+0000","last_became_active":"2023-06-10T21:20:45.269098+0000","last_became_peered":"2023-06-10T21:20:45.269098+0000","last_unstale":"2023-06-10T21:23:17.807579+0000","last_undegraded":"2023-06-10T21:23:17.807579+0000","last_fullsized":"2023-06-10T21:23:17.807579+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:23:51.802626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287440896,"num_objects":4386,"num_object_clones":0,"num_object_copies":13158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9150,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.620064+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:23:01.620064+0000","last_peered":"2023-06-10T21:23:01.620064+0000","last_clean":"2023-06-10T21:23:01.620064+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:23:01.620064+0000","last_undegraded":"2023-06-10T21:23:01.620064+0000","last_fullsized":"2023-06-10T21:23:01.620064+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608614+0000","last_change":"2023-06-10T21:10:48.976831+0000","last_active":"2023-06-10T21:23:01.608614+0000","last_peered":"2023-06-10T21:23:01.608614+0000","last_clean":"2023-06-10T21:23:01.608614+0000","last_became_active":"2023-06-10T21:10:48.976507+0000","last_became_peered":"2023-06-10T21:10:48.976507+0000","last_unstale":"2023-06-10T21:23:01.608614+0000","last_undegraded":"2023-06-10T21:23:01.608614+0000","last_fullsized":"2023-06-10T21:23:01.608614+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":145,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613072+0000","last_change":"2023-06-10T21:19:28.486084+0000","last_active":"2023-06-10T21:23:01.613072+0000","last_peered":"2023-06-10T21:23:01.613072+0000","last_clean":"2023-06-10T21:23:01.613072+0000","last_became_active":"2023-06-10T21:19:28.485531+0000","last_became_peered":"2023-06-10T21:19:28.485531+0000","last_unstale":"2023-06-10T21:23:01.613072+0000","last_undegraded":"2023-06-10T21:23:01.613072+0000","last_fullsized":"2023-06-10T21:23:01.613072+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:45.145706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"153'9423","reported_seq":9456,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.547348+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:23:19.547348+0000","last_peered":"2023-06-10T21:23:19.547348+0000","last_clean":"2023-06-10T21:23:19.547348+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:23:19.547348+0000","last_undegraded":"2023-06-10T21:23:19.547348+0000","last_fullsized":"2023-06-10T21:23:19.547348+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9423,"log_dups_size":0,"ondisk_log_size":9423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278331392,"num_objects":4247,"num_object_clones":0,"num_object_copies":12741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9423,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:08.535289+0000","last_change":"2023-06-10T21:19:28.482444+0000","last_active":"2023-06-10T21:23:08.535289+0000","last_peered":"2023-06-10T21:23:08.535289+0000","last_clean":"2023-06-10T21:23:08.535289+0000","last_became_active":"2023-06-10T21:19:28.481950+0000","last_became_peered":"2023-06-10T21:19:28.481950+0000","last_unstale":"2023-06-10T21:23:08.535289+0000","last_undegraded":"2023-06-10T21:23:08.535289+0000","last_fullsized":"2023-06-10T21:23:08.535289+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:43.450778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613490+0000","last_change":"2023-06-10T21:19:00.495462+0000","last_active":"2023-06-10T21:23:01.613490+0000","last_peered":"2023-06-10T21:23:01.613490+0000","last_clean":"2023-06-10T21:23:01.613490+0000","last_became_active":"2023-06-10T21:19:00.495152+0000","last_became_peered":"2023-06-10T21:19:00.495152+0000","last_unstale":"2023-06-10T21:23:01.613490+0000","last_undegraded":"2023-06-10T21:23:01.613490+0000","last_fullsized":"2023-06-10T21:23:01.613490+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:04:15.222235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.220395+0000","last_change":"2023-06-10T21:14:42.426043+0000","last_active":"2023-06-10T21:23:10.220395+0000","last_peered":"2023-06-10T21:23:10.220395+0000","last_clean":"2023-06-10T21:23:10.220395+0000","last_became_active":"2023-06-10T21:14:42.425714+0000","last_became_peered":"2023-06-10T21:14:42.425714+0000","last_unstale":"2023-06-10T21:23:10.220395+0000","last_undegraded":"2023-06-10T21:23:10.220395+0000","last_fullsized":"2023-06-10T21:23:10.220395+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:38.204967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"153'9067","reported_seq":9103,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:16.768124+0000","last_change":"2023-06-10T21:20:45.262983+0000","last_active":"2023-06-10T21:23:16.768124+0000","last_peered":"2023-06-10T21:23:16.768124+0000","last_clean":"2023-06-10T21:23:16.768124+0000","last_became_active":"2023-06-10T21:20:45.262641+0000","last_became_peered":"2023-06-10T21:20:45.262641+0000","last_unstale":"2023-06-10T21:23:16.768124+0000","last_undegraded":"2023-06-10T21:23:16.768124+0000","last_fullsized":"2023-06-10T21:23:16.768124+0000","mapping_epoch":127,"log_start":"131'1300","ondisk_log_start":"131'1300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7767,"log_dups_size":0,"ondisk_log_size":7767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:27:45.171697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280428544,"num_objects":4279,"num_object_clones":0,"num_object_copies":12837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9067,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612779+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:23:01.612779+0000","last_peered":"2023-06-10T21:23:01.612779+0000","last_clean":"2023-06-10T21:23:01.612779+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:23:01.612779+0000","last_undegraded":"2023-06-10T21:23:01.612779+0000","last_fullsized":"2023-06-10T21:23:01.612779+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":356,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608640+0000","last_change":"2023-06-10T21:10:48.989117+0000","last_active":"2023-06-10T21:23:01.608640+0000","last_peered":"2023-06-10T21:23:01.608640+0000","last_clean":"2023-06-10T21:23:01.608640+0000","last_became_active":"2023-06-10T21:10:48.988648+0000","last_became_peered":"2023-06-10T21:10:48.988648+0000","last_unstale":"2023-06-10T21:23:01.608640+0000","last_undegraded":"2023-06-10T21:23:01.608640+0000","last_fullsized":"2023-06-10T21:23:01.608640+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608279+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:23:01.608279+0000","last_peered":"2023-06-10T21:23:01.608279+0000","last_clean":"2023-06-10T21:23:01.608279+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:23:01.608279+0000","last_undegraded":"2023-06-10T21:23:01.608279+0000","last_fullsized":"2023-06-10T21:23:01.608279+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":104,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.607040+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:23:01.607040+0000","last_peered":"2023-06-10T21:23:01.607040+0000","last_clean":"2023-06-10T21:23:01.607040+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:23:01.607040+0000","last_undegraded":"2023-06-10T21:23:01.607040+0000","last_fullsized":"2023-06-10T21:23:01.607040+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"153'9335","reported_seq":9371,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:17.824477+0000","last_change":"2023-06-10T21:20:45.259115+0000","last_active":"2023-06-10T21:23:17.824477+0000","last_peered":"2023-06-10T21:23:17.824477+0000","last_clean":"2023-06-10T21:23:17.824477+0000","last_became_active":"2023-06-10T21:20:45.258813+0000","last_became_peered":"2023-06-10T21:20:45.258813+0000","last_unstale":"2023-06-10T21:23:17.824477+0000","last_undegraded":"2023-06-10T21:23:17.824477+0000","last_fullsized":"2023-06-10T21:23:17.824477+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9335,"log_dups_size":0,"ondisk_log_size":9335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:29.551738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288030720,"num_objects":4395,"num_object_clones":0,"num_object_copies":13185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9335,"num_write_kb":439360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":139,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608484+0000","last_change":"2023-06-10T21:14:42.435949+0000","last_active":"2023-06-10T21:23:01.608484+0000","last_peered":"2023-06-10T21:23:01.608484+0000","last_clean":"2023-06-10T21:23:01.608484+0000","last_became_active":"2023-06-10T21:14:42.435552+0000","last_became_peered":"2023-06-10T21:14:42.435552+0000","last_unstale":"2023-06-10T21:23:01.608484+0000","last_undegraded":"2023-06-10T21:23:01.608484+0000","last_fullsized":"2023-06-10T21:23:01.608484+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":109,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.231777+0000","last_change":"2023-06-10T21:19:28.487756+0000","last_active":"2023-06-10T21:23:10.231777+0000","last_peered":"2023-06-10T21:23:10.231777+0000","last_clean":"2023-06-10T21:23:10.231777+0000","last_became_active":"2023-06-10T21:19:28.487139+0000","last_became_peered":"2023-06-10T21:19:28.487139+0000","last_unstale":"2023-06-10T21:23:10.231777+0000","last_undegraded":"2023-06-10T21:23:10.231777+0000","last_fullsized":"2023-06-10T21:23:10.231777+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:06.356598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.617616+0000","last_change":"2023-06-10T21:19:00.494431+0000","last_active":"2023-06-10T21:23:01.617616+0000","last_peered":"2023-06-10T21:23:01.617616+0000","last_clean":"2023-06-10T21:23:01.617616+0000","last_became_active":"2023-06-10T21:19:00.494062+0000","last_became_peered":"2023-06-10T21:19:00.494062+0000","last_unstale":"2023-06-10T21:23:01.617616+0000","last_undegraded":"2023-06-10T21:23:01.617616+0000","last_fullsized":"2023-06-10T21:23:01.617616+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:29.312922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"153'9420","reported_seq":9454,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.403438+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:23:19.403438+0000","last_peered":"2023-06-10T21:23:19.403438+0000","last_clean":"2023-06-10T21:23:19.403438+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:23:19.403438+0000","last_undegraded":"2023-06-10T21:23:19.403438+0000","last_fullsized":"2023-06-10T21:23:19.403438+0000","mapping_epoch":127,"log_start":"132'1700","ondisk_log_start":"132'1700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282066944,"num_objects":4304,"num_object_clones":0,"num_object_copies":12912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9420,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.614113+0000","last_change":"2023-06-10T21:10:48.969727+0000","last_active":"2023-06-10T21:23:01.614113+0000","last_peered":"2023-06-10T21:23:01.614113+0000","last_clean":"2023-06-10T21:23:01.614113+0000","last_became_active":"2023-06-10T21:10:48.969376+0000","last_became_peered":"2023-06-10T21:10:48.969376+0000","last_unstale":"2023-06-10T21:23:01.614113+0000","last_undegraded":"2023-06-10T21:23:01.614113+0000","last_fullsized":"2023-06-10T21:23:01.614113+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.613146+0000","last_change":"2023-06-10T21:19:00.496213+0000","last_active":"2023-06-10T21:23:01.613146+0000","last_peered":"2023-06-10T21:23:01.613146+0000","last_clean":"2023-06-10T21:23:01.613146+0000","last_became_active":"2023-06-10T21:19:00.495889+0000","last_became_peered":"2023-06-10T21:19:00.495889+0000","last_unstale":"2023-06-10T21:23:01.613146+0000","last_undegraded":"2023-06-10T21:23:01.613146+0000","last_fullsized":"2023-06-10T21:23:01.613146+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:19.626298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.231032+0000","last_change":"2023-06-10T21:10:48.971561+0000","last_active":"2023-06-10T21:23:10.231032+0000","last_peered":"2023-06-10T21:23:10.231032+0000","last_clean":"2023-06-10T21:23:10.231032+0000","last_became_active":"2023-06-10T21:10:48.971172+0000","last_became_peered":"2023-06-10T21:10:48.971172+0000","last_unstale":"2023-06-10T21:23:10.231032+0000","last_undegraded":"2023-06-10T21:23:10.231032+0000","last_fullsized":"2023-06-10T21:23:10.231032+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:26.372457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"153'9483","reported_seq":9518,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:19.216842+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:23:19.216842+0000","last_peered":"2023-06-10T21:23:19.216842+0000","last_clean":"2023-06-10T21:23:19.216842+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:23:19.216842+0000","last_undegraded":"2023-06-10T21:23:19.216842+0000","last_fullsized":"2023-06-10T21:23:19.216842+0000","mapping_epoch":127,"log_start":"132'1700","ondisk_log_start":"132'1700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7783,"log_dups_size":0,"ondisk_log_size":7783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293339160,"num_objects":4477,"num_object_clones":0,"num_object_copies":13431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9483,"num_write_kb":446657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":97,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.609004+0000","last_change":"2023-06-10T21:19:28.484610+0000","last_active":"2023-06-10T21:23:01.609004+0000","last_peered":"2023-06-10T21:23:01.609004+0000","last_clean":"2023-06-10T21:23:01.609004+0000","last_became_active":"2023-06-10T21:19:28.484222+0000","last_became_peered":"2023-06-10T21:19:28.484222+0000","last_unstale":"2023-06-10T21:23:01.609004+0000","last_undegraded":"2023-06-10T21:23:01.609004+0000","last_fullsized":"2023-06-10T21:23:01.609004+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:52:17.843059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.223109+0000","last_change":"2023-06-10T21:14:42.429890+0000","last_active":"2023-06-10T21:23:10.223109+0000","last_peered":"2023-06-10T21:23:10.223109+0000","last_clean":"2023-06-10T21:23:10.223109+0000","last_became_active":"2023-06-10T21:14:42.429551+0000","last_became_peered":"2023-06-10T21:14:42.429551+0000","last_unstale":"2023-06-10T21:23:10.223109+0000","last_undegraded":"2023-06-10T21:23:10.223109+0000","last_fullsized":"2023-06-10T21:23:10.223109+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:45:04.665110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608004+0000","last_change":"2023-06-10T21:19:28.487200+0000","last_active":"2023-06-10T21:23:01.608004+0000","last_peered":"2023-06-10T21:23:01.608004+0000","last_clean":"2023-06-10T21:23:01.608004+0000","last_became_active":"2023-06-10T21:19:28.482519+0000","last_became_peered":"2023-06-10T21:19:28.482519+0000","last_unstale":"2023-06-10T21:23:01.608004+0000","last_undegraded":"2023-06-10T21:23:01.608004+0000","last_fullsized":"2023-06-10T21:23:01.608004+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:02.707185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"153'9150","reported_seq":9186,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:16.750660+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:23:16.750660+0000","last_peered":"2023-06-10T21:23:16.750660+0000","last_clean":"2023-06-10T21:23:16.750660+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:23:16.750660+0000","last_undegraded":"2023-06-10T21:23:16.750660+0000","last_fullsized":"2023-06-10T21:23:16.750660+0000","mapping_epoch":127,"log_start":"132'1400","ondisk_log_start":"132'1400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7750,"log_dups_size":0,"ondisk_log_size":7750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292290560,"num_objects":4460,"num_object_clones":0,"num_object_copies":13380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9150,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.607457+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:23:01.607457+0000","last_peered":"2023-06-10T21:23:01.607457+0000","last_clean":"2023-06-10T21:23:01.607457+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:23:01.607457+0000","last_undegraded":"2023-06-10T21:23:01.607457+0000","last_fullsized":"2023-06-10T21:23:01.607457+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":112,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.617459+0000","last_change":"2023-06-10T21:14:42.425260+0000","last_active":"2023-06-10T21:23:01.617459+0000","last_peered":"2023-06-10T21:23:01.617459+0000","last_clean":"2023-06-10T21:23:01.617459+0000","last_became_active":"2023-06-10T21:14:42.424731+0000","last_became_peered":"2023-06-10T21:14:42.424731+0000","last_unstale":"2023-06-10T21:23:01.617459+0000","last_undegraded":"2023-06-10T21:23:01.617459+0000","last_fullsized":"2023-06-10T21:23:01.617459+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:24:40.516334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":104,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606547+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:23:01.606547+0000","last_peered":"2023-06-10T21:23:01.606547+0000","last_clean":"2023-06-10T21:23:01.606547+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:23:01.606547+0000","last_undegraded":"2023-06-10T21:23:01.606547+0000","last_fullsized":"2023-06-10T21:23:01.606547+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":97,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.231241+0000","last_change":"2023-06-10T21:10:48.974095+0000","last_active":"2023-06-10T21:23:10.231241+0000","last_peered":"2023-06-10T21:23:10.231241+0000","last_clean":"2023-06-10T21:23:10.231241+0000","last_became_active":"2023-06-10T21:10:48.971533+0000","last_became_peered":"2023-06-10T21:10:48.971533+0000","last_unstale":"2023-06-10T21:23:10.231241+0000","last_undegraded":"2023-06-10T21:23:10.231241+0000","last_fullsized":"2023-06-10T21:23:10.231241+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:26.091661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.623853+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:23:01.623853+0000","last_peered":"2023-06-10T21:23:01.623853+0000","last_clean":"2023-06-10T21:23:01.623853+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:23:01.623853+0000","last_undegraded":"2023-06-10T21:23:01.623853+0000","last_fullsized":"2023-06-10T21:23:01.623853+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.612691+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:23:01.612691+0000","last_peered":"2023-06-10T21:23:01.612691+0000","last_clean":"2023-06-10T21:23:01.612691+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:23:01.612691+0000","last_undegraded":"2023-06-10T21:23:01.612691+0000","last_fullsized":"2023-06-10T21:23:01.612691+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608694+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:23:01.608694+0000","last_peered":"2023-06-10T21:23:01.608694+0000","last_clean":"2023-06-10T21:23:01.608694+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:23:01.608694+0000","last_undegraded":"2023-06-10T21:23:01.608694+0000","last_fullsized":"2023-06-10T21:23:01.608694+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":107,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.619836+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:23:01.619836+0000","last_peered":"2023-06-10T21:23:01.619836+0000","last_clean":"2023-06-10T21:23:01.619836+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:23:01.619836+0000","last_undegraded":"2023-06-10T21:23:01.619836+0000","last_fullsized":"2023-06-10T21:23:01.619836+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608306+0000","last_change":"2023-06-10T21:19:28.482280+0000","last_active":"2023-06-10T21:23:01.608306+0000","last_peered":"2023-06-10T21:23:01.608306+0000","last_clean":"2023-06-10T21:23:01.608306+0000","last_became_active":"2023-06-10T21:19:28.481936+0000","last_became_peered":"2023-06-10T21:19:28.481936+0000","last_unstale":"2023-06-10T21:23:01.608306+0000","last_undegraded":"2023-06-10T21:23:01.608306+0000","last_fullsized":"2023-06-10T21:23:01.608306+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608046+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:23:01.608046+0000","last_peered":"2023-06-10T21:23:01.608046+0000","last_clean":"2023-06-10T21:23:01.608046+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:23:01.608046+0000","last_undegraded":"2023-06-10T21:23:01.608046+0000","last_fullsized":"2023-06-10T21:23:01.608046+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":142,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.608623+0000","last_change":"2023-06-10T21:19:28.481801+0000","last_active":"2023-06-10T21:23:01.608623+0000","last_peered":"2023-06-10T21:23:01.608623+0000","last_clean":"2023-06-10T21:23:01.608623+0000","last_became_active":"2023-06-10T21:19:28.481440+0000","last_became_peered":"2023-06-10T21:19:28.481440+0000","last_unstale":"2023-06-10T21:23:01.608623+0000","last_undegraded":"2023-06-10T21:23:01.608623+0000","last_fullsized":"2023-06-10T21:23:01.608623+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:10.297137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-06-10T21:23:10.231060+0000","last_change":"2023-06-10T21:19:00.493733+0000","last_active":"2023-06-10T21:23:10.231060+0000","last_peered":"2023-06-10T21:23:10.231060+0000","last_clean":"2023-06-10T21:23:10.231060+0000","last_became_active":"2023-06-10T21:19:00.493345+0000","last_became_peered":"2023-06-10T21:19:00.493345+0000","last_unstale":"2023-06-10T21:23:10.231060+0000","last_undegraded":"2023-06-10T21:23:10.231060+0000","last_fullsized":"2023-06-10T21:23:10.231060+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:57.464085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":128,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-06-10T21:23:01.606367+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:23:01.606367+0000","last_peered":"2023-06-10T21:23:01.606367+0000","last_clean":"2023-06-10T21:23:01.606367+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:23:01.606367+0000","last_undegraded":"2023-06-10T21:23:01.606367+0000","last_fullsized":"2023-06-10T21:23:01.606367+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4552327192,"num_objects":69464,"num_object_clones":0,"num_object_copies":208392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69464,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148508,"num_write_kb":6975041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13608169472,"data_stored":13608157256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133708,"ondisk_log_size":133708,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430515,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2892016,"kb_used_data":2515240,"kb_used_omap":0,"kb_used_meta":376576,"kb_avail":90853136,"statfs":{"total":95995035648,"available":93033611264,"internally_reserved":0,"allocated":2575605760,"data_stored":2575101837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":385613824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.8620000000000001,"15min":1.6439999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":7.6719999999999997,"5min":53.706000000000003,"15min":53.706000000000003},"last":10.132999999999999},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.8280000000000001,"15min":2.0390000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.245},"max":{"1min":7.3579999999999997,"5min":53.777000000000001,"15min":53.777000000000001},"last":10.102}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":10.936,"5min":3.6429999999999998,"15min":2.3359999999999999},"min":{"1min":0.32100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":130.19399999999999,"5min":130.19399999999999,"15min":130.19399999999999},"last":10.176},{"interface":"front","average":{"1min":21.949000000000002,"5min":5.9269999999999996,"15min":3.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":212.20599999999999,"5min":212.20599999999999,"15min":212.20599999999999},"last":2.23}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3100000000000001,"5min":1.204,"15min":0.746},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":6.6189999999999998,"5min":9.1609999999999996,"15min":9.1609999999999996},"last":5.3810000000000002},{"interface":"front","average":{"1min":13.19,"5min":3.2930000000000001,"15min":1.4530000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":212.13399999999999,"5min":212.13399999999999,"15min":212.13399999999999},"last":2.1819999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":6.6340000000000003,"5min":2.4249999999999998,"15min":1.1830000000000001},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":43.911000000000001,"5min":57.085000000000001,"15min":57.085000000000001},"last":5.4290000000000003},{"interface":"front","average":{"1min":6.1660000000000004,"5min":2.4430000000000001,"15min":1.2050000000000001},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":44.225999999999999,"5min":57.167999999999999,"15min":57.167999999999999},"last":10.073}]},{"osd":4,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":0.68500000000000005,"15min":1.929},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":4.0170000000000003,"5min":4.0170000000000003,"15min":219.17699999999999},"last":5.3159999999999998},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.0109999999999999,"15min":2.1680000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":9.2129999999999992,"5min":9.2129999999999992,"15min":219.13999999999999},"last":5.5069999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":0.91200000000000003,"15min":2.5419999999999998},"min":{"1min":0.28499999999999998,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":4.7249999999999996,"5min":4.7619999999999996,"15min":311.202},"last":10.042},{"interface":"front","average":{"1min":1.3859999999999999,"5min":0.94499999999999995,"15min":2.4239999999999999},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.29699999999999999},"max":{"1min":4.8239999999999998,"5min":4.8239999999999998,"15min":311.875},"last":4.5510000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":0.86699999999999999,"15min":1.0649999999999999},"min":{"1min":0.39900000000000002,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.4109999999999996,"5min":4.4109999999999996,"15min":8.6630000000000003},"last":10.159000000000001},{"interface":"front","average":{"1min":1.2889999999999999,"5min":0.92400000000000004,"15min":1.016},"min":{"1min":0.40000000000000002,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":4.8680000000000003,"5min":7.6900000000000004,"15min":7.6900000000000004},"last":3.5670000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626743,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1137820,"kb_used_data":823508,"kb_used_omap":0,"kb_used_meta":314304,"kb_avail":92607332,"statfs":{"total":95995035648,"available":94829907968,"internally_reserved":0,"allocated":843272192,"data_stored":842788264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321847296},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.3180000000000001,"15min":2.1640000000000001},"min":{"1min":0.25,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":7.609,"5min":8.4730000000000008,"15min":119.276},"last":5.5940000000000003},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.2430000000000001,"15min":2.5710000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":7.2270000000000003,"5min":7.2270000000000003,"15min":119.22},"last":0.495}]},{"osd":1,"last update":"Sat Jun 10 21:22:54 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.3340000000000001,"15min":1.746},"min":{"1min":0.32200000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.9130000000000003,"5min":14.897,"15min":14.897},"last":0.51200000000000001},{"interface":"front","average":{"1min":3.0600000000000001,"5min":1.4770000000000001,"15min":1.8680000000000001},"min":{"1min":0.30599999999999999,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5369999999999999,"5min":8.0299999999999994,"15min":10.766999999999999},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9260000000000002,"5min":1.2030000000000001,"15min":0.77500000000000002},"min":{"1min":0.42199999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.125999999999999,"5min":10.125999999999999,"15min":10.125999999999999},"last":3.2850000000000001},{"interface":"front","average":{"1min":2.1179999999999999,"5min":1.113,"15min":0.72899999999999998},"min":{"1min":0.29699999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.5529999999999999,"5min":8.9600000000000009,"15min":8.9600000000000009},"last":3.2959999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":5.8479999999999999,"5min":1.7569999999999999,"15min":0.95099999999999996},"min":{"1min":0.37,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":53.591000000000001,"5min":53.591000000000001,"15min":53.591000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":5.1749999999999998,"5min":1.873,"15min":1.004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":53.616,"5min":53.616,"15min":53.616},"last":4.3630000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":1.0029999999999999,"15min":1.819},"min":{"1min":0.311,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":4.0049999999999999,"5min":6.7290000000000001,"15min":221.76900000000001},"last":5.633},{"interface":"front","average":{"1min":1.3440000000000001,"5min":0.92900000000000005,"15min":1.974},"min":{"1min":0.34499999999999997,"5min":0.27700000000000002,"15min":0.27300000000000002},"max":{"1min":10.221,"5min":10.221,"15min":221.905},"last":1.5549999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:22:27 2023","interfaces":[{"interface":"back","average":{"1min":2.6549999999999998,"5min":1.2889999999999999,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.25700000000000001},"max":{"1min":11.132999999999999,"5min":11.132999999999999,"15min":313.661},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.1499999999999999,"15min":2.4020000000000001},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":10.17,"5min":10.17,"15min":314.38900000000001},"last":0.436}]},{"osd":7,"last update":"Sat Jun 10 21:22:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.367,"15min":1.196},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":4.0679999999999996,"5min":11.179,"15min":11.179},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.181,"5min":1.2190000000000001,"15min":1.2090000000000001},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":10.244999999999999,"5min":10.244999999999999,"15min":10.244999999999999},"last":3.4849999999999999}]}]},{"osd":1,"up_from":105,"seq":450971566186,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2740172,"kb_used_data":2463492,"kb_used_omap":4,"kb_used_meta":276667,"kb_avail":91004980,"statfs":{"total":95995035648,"available":93189099520,"internally_reserved":0,"allocated":2522615808,"data_stored":2522028642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":283307918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":2.3809999999999998,"15min":2.129},"min":{"1min":0.25900000000000001,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":29.754999999999999,"5min":86.009,"15min":86.009},"last":3.2250000000000001},{"interface":"front","average":{"1min":3.3370000000000002,"5min":2.7719999999999998,"15min":2.669},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.22600000000000001},"max":{"1min":32.079000000000001,"5min":85.869,"15min":85.869},"last":3.294}]},{"osd":2,"last update":"Sat Jun 10 21:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.3540000000000001,"15min":0.78200000000000003},"min":{"1min":0.26000000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":13.589,"5min":13.589,"15min":13.589},"last":6.8209999999999997},{"interface":"front","average":{"1min":2.548,"5min":0.999,"15min":0.64600000000000002},"min":{"1min":0.24299999999999999,"5min":0.218,"15min":0.218},"max":{"1min":8.6940000000000008,"5min":8.6940000000000008,"15min":8.6940000000000008},"last":0.29999999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:22:31 2023","interfaces":[{"interface":"back","average":{"1min":12.256,"5min":2.9249999999999998,"15min":1.2150000000000001},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.251},"max":{"1min":118.348,"5min":118.348,"15min":118.348},"last":2.9409999999999998},{"interface":"front","average":{"1min":13.488,"5min":3.4420000000000002,"15min":1.4239999999999999},"min":{"1min":0.48499999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":117.75,"5min":117.75,"15min":117.75},"last":6.9210000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.2350000000000001,"15min":1.7989999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":8.0199999999999996,"5min":8.0199999999999996,"15min":12.94},"last":7.8280000000000003},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.22,"15min":2.343},"min":{"1min":0.40699999999999997,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.5720000000000001,"5min":7.5720000000000001,"15min":12.603},"last":8.1199999999999992}]},{"osd":5,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.55,"15min":2.0720000000000001},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":8.2550000000000008,"5min":9.7219999999999995,"15min":11.007999999999999},"last":6.8650000000000002},{"interface":"front","average":{"1min":2.8820000000000001,"5min":1.5249999999999999,"15min":2.3130000000000002},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.309},"max":{"1min":7.9669999999999996,"5min":8.9700000000000006,"15min":13.602},"last":2.7839999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":1.4930000000000001,"15min":2.0699999999999998},"min":{"1min":0.56499999999999995,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.4889999999999999,"5min":9.6699999999999999,"15min":12.861000000000001},"last":3.2530000000000001},{"interface":"front","average":{"1min":2.266,"5min":1.3799999999999999,"15min":2.2610000000000001},"min":{"1min":0.48199999999999998,"5min":0.40899999999999997,"15min":0.36699999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":12.970000000000001},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:22:55 2023","interfaces":[{"interface":"back","average":{"1min":2.444,"5min":1.363,"15min":2.8090000000000002},"min":{"1min":0.45300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.1750000000000007,"5min":8.1750000000000007,"15min":13.025},"last":2.8300000000000001},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.2549999999999999,"15min":1.984},"min":{"1min":0.49199999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.6920000000000002,"5min":7.6920000000000002,"15min":19.286999999999999},"last":8.2509999999999994}]}]},{"osd":0,"up_from":86,"seq":369367187608,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520804,"kb_used_data":1119260,"kb_used_omap":5,"kb_used_meta":401530,"kb_avail":92224348,"statfs":{"total":95995035648,"available":94437732352,"internally_reserved":0,"allocated":1146122240,"data_stored":1145513047,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":411167354},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:22:56 2023","interfaces":[{"interface":"back","average":{"1min":4.1070000000000002,"5min":3.165,"15min":2.6190000000000002},"min":{"1min":0.28000000000000003,"5min":0.20899999999999999,"15min":0.19700000000000001},"max":{"1min":13.58,"5min":137.761,"15min":137.761},"last":7.3470000000000004},{"interface":"front","average":{"1min":3.2519999999999998,"5min":3.1040000000000001,"15min":3.3620000000000001},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.22700000000000001},"max":{"1min":15.045999999999999,"5min":137.88999999999999,"15min":137.88999999999999},"last":3.032}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":4.1230000000000002,"5min":1.468,"15min":0.71499999999999997},"min":{"1min":0.30099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":14.542999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":7.2309999999999999},{"interface":"front","average":{"1min":3.0489999999999999,"5min":1.2330000000000001,"15min":0.64900000000000002},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":16.782,"5min":16.782,"15min":16.782},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:22:32 2023","interfaces":[{"interface":"back","average":{"1min":10.523,"5min":4.3330000000000002,"15min":1.712},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":75.674999999999997,"5min":163.56999999999999,"15min":163.56999999999999},"last":7.3650000000000002},{"interface":"front","average":{"1min":10.789,"5min":4.4050000000000002,"15min":1.726},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":77.289000000000001,"5min":164.239,"15min":164.239},"last":9.3819999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":1.595,"15min":2.1469999999999998},"min":{"1min":0.34999999999999998,"5min":0.32000000000000001,"15min":0.29599999999999999},"max":{"1min":9.3490000000000002,"5min":10.733000000000001,"15min":10.948},"last":4.194},{"interface":"front","average":{"1min":2.7549999999999999,"5min":1.2609999999999999,"15min":1.849},"min":{"1min":0.39100000000000001,"5min":0.35899999999999999,"15min":0.29699999999999999},"max":{"1min":7.2309999999999999,"5min":7.2309999999999999,"15min":10.667},"last":4.2560000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.218,"15min":2.0710000000000002},"min":{"1min":0.34799999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.9889999999999999,"5min":6.0529999999999999,"15min":10.693},"last":9.4320000000000004},{"interface":"front","average":{"1min":3.0379999999999998,"5min":1.534,"15min":2.4329999999999998},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.32400000000000001},"max":{"1min":7.2210000000000001,"5min":7.2210000000000001,"15min":17.460000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9900000000000002,"5min":1.3799999999999999,"15min":2.1760000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35899999999999999},"max":{"1min":10.305999999999999,"5min":10.305999999999999,"15min":11.848000000000001},"last":4.3029999999999999},{"interface":"front","average":{"1min":3.3210000000000002,"5min":1.575,"15min":2.5249999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.38500000000000001},"max":{"1min":9.4109999999999996,"5min":9.4109999999999996,"15min":17.411000000000001},"last":1.4039999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.0640000000000001,"5min":1.482,"15min":2.4969999999999999},"min":{"1min":0.45500000000000002,"5min":0.42899999999999999,"15min":0.36399999999999999},"max":{"1min":6.8200000000000003,"5min":6.8200000000000003,"15min":17.486999999999998},"last":0.44800000000000001},{"interface":"front","average":{"1min":3.544,"5min":1.6240000000000001,"15min":2.1850000000000001},"min":{"1min":0.51700000000000002,"5min":0.42499999999999999,"15min":0.36799999999999999},"max":{"1min":9.3859999999999992,"5min":10.691000000000001,"15min":10.867000000000001},"last":4.5519999999999996}]}]},{"osd":2,"up_from":120,"seq":515396075574,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991008,"kb_used_data":824472,"kb_used_omap":7,"kb_used_meta":166520,"kb_avail":92754144,"statfs":{"total":95995035648,"available":94980243456,"internally_reserved":0,"allocated":844259328,"data_stored":843710776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":170516561},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.3049999999999999,"15min":0.68700000000000006},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.311,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":5.1989999999999998,"5min":1.9299999999999999,"15min":0.88600000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":62.451999999999998,"5min":62.451999999999998,"15min":62.451999999999998},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.1919999999999999,"15min":0.65400000000000003},"min":{"1min":0.252,"5min":0.217,"15min":0.217},"max":{"1min":5.6390000000000002,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.23100000000000001},{"interface":"front","average":{"1min":2.4809999999999999,"5min":1.423,"15min":0.749},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":13.391,"5min":28.013999999999999,"15min":28.013999999999999},"last":0.63300000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:22:28 2023","interfaces":[{"interface":"back","average":{"1min":7.1050000000000004,"5min":2.1480000000000001,"15min":0.98199999999999998},"min":{"1min":0.27100000000000002,"5min":0.247,"15min":0.247},"max":{"1min":96.295000000000002,"5min":96.295000000000002,"15min":96.295000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":9.7989999999999995,"5min":2.633,"15min":1.161},"min":{"1min":0.32800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":99.504999999999995,"5min":99.504999999999995,"15min":99.504999999999995},"last":1.722}]},{"osd":4,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":1.0009999999999999,"15min":0.77100000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.0009999999999994,"5min":8.0009999999999994,"15min":8.0009999999999994},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.3440000000000001,"5min":1.0449999999999999,"15min":0.79200000000000004},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.351,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":0.30399999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.3280000000000001,"15min":0.90000000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.254,"5min":10.254,"15min":10.254},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.296,"15min":0.88},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.0310000000000006,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":2.1160000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.266,"15min":0.88800000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.048,"5min":8.048,"15min":8.048},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.845,"5min":1.2230000000000001,"15min":0.88700000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":5.7009999999999996,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.33600000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:23:10 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.073,"15min":0.84699999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2889999999999997,"5min":7.3550000000000004,"15min":7.3550000000000004},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.966,"5min":1.2430000000000001,"15min":0.89900000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":8.0679999999999996,"5min":8.0679999999999996,"15min":8.0679999999999996},"last":0.51900000000000002}]}]},{"osd":3,"up_from":125,"seq":536870912048,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2471384,"kb_used_data":2288656,"kb_used_omap":2,"kb_used_meta":182333,"kb_avail":91273768,"statfs":{"total":95995035648,"available":93464338432,"internally_reserved":0,"allocated":2343583744,"data_stored":2343042965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":186709045},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8079999999999998,"5min":0.93899999999999995,"15min":0.55400000000000005},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.6460000000000008,"5min":8.6460000000000008,"15min":8.6460000000000008},"last":3.5590000000000002},{"interface":"front","average":{"1min":2.758,"5min":1.016,"15min":0.57399999999999995},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.669,"5min":12.669,"15min":12.669},"last":0.82099999999999995}]},{"osd":1,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":4.9820000000000002,"5min":1.4379999999999999,"15min":0.748},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":54.112000000000002,"5min":54.112000000000002,"15min":54.112000000000002},"last":3.593},{"interface":"front","average":{"1min":5.734,"5min":1.639,"15min":0.82299999999999995},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":57.814,"5min":57.814,"15min":57.814},"last":1.9199999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":0.96099999999999997,"15min":0.57999999999999996},"min":{"1min":0.27200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":12.726000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":1.0569999999999999},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.0409999999999999,"15min":0.60899999999999999},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":9.8000000000000007,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":3.6019999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.091,"15min":0.747},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":10.92,"5min":10.92,"15min":10.92},"last":0.68300000000000005},{"interface":"front","average":{"1min":2.4460000000000002,"5min":1.03,"15min":0.72599999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.4800000000000004,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":3.4420000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.0620000000000001,"15min":0.752},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.975,"5min":10.975,"15min":10.975},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.827,"5min":1.163,"15min":0.79400000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.423,"5min":8.423,"15min":8.423},"last":3.5019999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":1.1879999999999999,"15min":0.79000000000000004},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":8.5050000000000008,"5min":8.5050000000000008,"15min":8.5050000000000008},"last":3.5409999999999999},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.0529999999999999,"15min":0.79500000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":7.1799999999999997,"5min":7.1799999999999997,"15min":7.1799999999999997},"last":3.4870000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:22:37 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":1.1799999999999999,"15min":0.80100000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":8.3849999999999998,"5min":8.3849999999999998,"15min":8.3849999999999998},"last":3.5739999999999998},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.085,"15min":0.81000000000000005},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.3579999999999997,"5min":7.3579999999999997,"15min":7.3579999999999997},"last":3.5270000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019199,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191136,"kb_used_data":839704,"kb_used_omap":0,"kb_used_meta":351296,"kb_avail":92554016,"statfs":{"total":95995035648,"available":94775312384,"internally_reserved":0,"allocated":859856896,"data_stored":859368404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":359727104},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.8600000000000001,"15min":2.056},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22},"max":{"1min":7.3559999999999999,"5min":40.186999999999998,"15min":40.186999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.1930000000000001,"5min":2.1259999999999999,"15min":2.3330000000000002},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.23100000000000001},"max":{"1min":6.4500000000000002,"5min":39.395000000000003,"15min":39.395000000000003},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:23:00 2023","interfaces":[{"interface":"back","average":{"1min":4.7279999999999998,"5min":2.5569999999999999,"15min":2.1739999999999999},"min":{"1min":0.26800000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":24.803000000000001,"5min":79.731999999999999,"15min":79.731999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.2570000000000001,"5min":2.1960000000000002,"15min":2.452},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":24.613,"5min":79.146000000000001,"15min":79.146000000000001},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":1.7030000000000001,"5min":0.88100000000000001,"15min":0.64500000000000002},"min":{"1min":0.32500000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3330000000000002,"5min":7.3330000000000002,"15min":7.3330000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.4420000000000002,"5min":1.135,"15min":0.79700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":11.099,"5min":11.099,"15min":11.099},"last":0.73299999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":10.824,"5min":2.9609999999999999,"15min":1.3580000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":92.936000000000007,"5min":92.936000000000007,"15min":92.936000000000007},"last":0.97699999999999998},{"interface":"front","average":{"1min":9.3360000000000003,"5min":2.6629999999999998,"15min":1.3009999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":92.912000000000006,"5min":92.912000000000006,"15min":92.912000000000006},"last":0.98799999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:23:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2709999999999999,"5min":1.1899999999999999,"15min":1.351},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":10.984999999999999,"5min":10.984999999999999,"15min":92.653000000000006},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.681,"5min":1.1060000000000001,"15min":1.5589999999999999},"min":{"1min":0.38300000000000001,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":93.281999999999996},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:22:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8490000000000002,"5min":1.1799999999999999,"15min":1.0820000000000001},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.26700000000000002},"max":{"1min":8.2599999999999998,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.9119999999999999,"5min":1.0009999999999999,"15min":0.97099999999999997},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.30399999999999999},"max":{"1min":8.0600000000000005,"5min":8.0600000000000005,"15min":8.8420000000000005},"last":0.432}]},{"osd":7,"last update":"Sat Jun 10 21:22:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9510000000000001,"5min":1.1899999999999999,"15min":1.24},"min":{"1min":0.38600000000000001,"5min":0.318,"15min":0.30099999999999999},"max":{"1min":4.9669999999999996,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.502},{"interface":"front","average":{"1min":2.0449999999999999,"5min":1.1499999999999999,"15min":1.083},"min":{"1min":0.42699999999999999,"5min":0.34000000000000002,"15min":0.312},"max":{"1min":8.3119999999999994,"5min":8.3119999999999994,"15min":8.3119999999999994},"last":0.41399999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822971,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2963092,"kb_used_data":2496204,"kb_used_omap":0,"kb_used_meta":466880,"kb_avail":90782060,"statfs":{"total":95995035648,"available":92960829440,"internally_reserved":0,"allocated":2556112896,"data_stored":2555632478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":478085120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5070000000000001,"5min":1.859,"15min":1.476},"min":{"1min":0.252,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":8.0310000000000006,"5min":57.700000000000003,"15min":57.700000000000003},"last":0.44},{"interface":"front","average":{"1min":2.661,"5min":1.869,"15min":2.1760000000000002},"min":{"1min":0.312,"5min":0.223,"15min":0.223},"max":{"1min":8.8290000000000006,"5min":57.890999999999998,"15min":57.890999999999998},"last":0.51000000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5310000000000001,"5min":2.1190000000000002,"15min":2.2069999999999999},"min":{"1min":0.28199999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.99,"5min":57.899000000000001,"15min":57.899000000000001},"last":1.8500000000000001},{"interface":"front","average":{"1min":2.2349999999999999,"5min":1.8740000000000001,"15min":2.5670000000000002},"min":{"1min":0.28199999999999997,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":13.875999999999999,"5min":57.837000000000003,"15min":57.837000000000003},"last":2.2269999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.004,"15min":0.73799999999999999},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.0389999999999997,"5min":6.6029999999999998,"15min":6.6029999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.292,"15min":0.83499999999999996},"min":{"1min":0.433,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.9139999999999997,"5min":10.895,"15min":10.895},"last":0.628}]},{"osd":3,"last update":"Sat Jun 10 21:22:29 2023","interfaces":[{"interface":"back","average":{"1min":10.441000000000001,"5min":2.9289999999999998,"15min":1.387},"min":{"1min":0.442,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":128.16800000000001,"5min":128.16800000000001,"15min":128.16800000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":9.4960000000000004,"5min":2.6709999999999998,"15min":1.248},"min":{"1min":0.314,"5min":0.311,"15min":0.311},"max":{"1min":128.58000000000001,"5min":128.58000000000001,"15min":128.58000000000001},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:22:57 2023","interfaces":[{"interface":"back","average":{"1min":2.79,"5min":1.1539999999999999,"15min":1.0669999999999999},"min":{"1min":0.30499999999999999,"5min":0.26600000000000001,"15min":0.25700000000000001},"max":{"1min":11.039999999999999,"5min":11.039999999999999,"15min":11.039999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.1899999999999999,"5min":0.83199999999999996,"15min":1.129},"min":{"1min":0.34000000000000002,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":5.016,"5min":6.2640000000000002,"15min":12.901},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:23:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7010000000000001,"5min":1.206,"15min":1.2150000000000001},"min":{"1min":0.372,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.991,"5min":10.991,"15min":10.991},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.1920000000000002,"5min":1.0369999999999999,"15min":1.2410000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.0429999999999993,"5min":8.0429999999999993,"15min":8.0429999999999993},"last":0.61299999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:22:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":1.149,"15min":1.3220000000000001},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.308},"max":{"1min":6.7080000000000002,"5min":8.5109999999999992,"15min":8.5109999999999992},"last":1.4339999999999999},{"interface":"front","average":{"1min":2.1349999999999998,"5min":1.1379999999999999,"15min":1.173},"min":{"1min":0.42099999999999999,"5min":0.318,"15min":0.318},"max":{"1min":7.1539999999999999,"5min":7.8710000000000004,"15min":8.1809999999999992},"last":0.46999999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1140326400,"data_stored":1140326400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2511474688,"data_stored":2511470616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":842465280,"data_stored":842465280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2324692992,"data_stored":2324692992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":857800704,"data_stored":857800704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2535854080,"data_stored":2535850008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":838860800,"data_stored":838860800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2556694528,"data_stored":2556690456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:23:24.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_4 pgp_num 16 2023-06-10T21:23:24.772 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:24.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:24.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:24.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:23:24.887 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:24.981 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:23:25.067 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:25.174 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:25.358 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:25.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:25.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:25.504 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:25.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:25.568 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:25 smithi152 ceph-mon[156819]: pgmap v615: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:23:25.569 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:25 smithi064 ceph-mon[180297]: pgmap v615: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:23:25.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:25 smithi064 ceph-mon[186706]: pgmap v615: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:23:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:25.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:25.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:25.717 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:25.782 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:25.821 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:25.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:25.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:26.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:26.706 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:26.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:26.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:26.817 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:26.877 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:26.887 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:27.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:27.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:27.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:27.138 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:27.260 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:27.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[180297]: pgmap v616: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-10T21:23:27.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:27.319 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:27.456 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:27.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[186706]: pgmap v616: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-10T21:23:27.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:27.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:27 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:27 smithi152 ceph-mon[156819]: pgmap v616: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-10T21:23:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:27 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:27 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:28.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:28.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:28.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:23:28.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:23:28.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:28.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:23:28.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:23:28.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:28.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:28.607 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:23:28.608 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:23:28.608 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:28.610 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:28.707 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:28.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:28.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:28.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:28.873 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:28.885 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:28.929 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:28.973 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:29.005 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:29.037 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:29.072 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:29.111 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:29.128 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.129 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.130 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:23:29.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:29 smithi064 ceph-mon[180297]: pgmap v617: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:29.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4228027568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:29.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:29 smithi064 ceph-mon[186706]: pgmap v617: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:29.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4228027568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:29 smithi152 ceph-mon[156819]: pgmap v617: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-10T21:23:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4228027568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:29.856 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:29.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:29.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:30.010 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:30.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:30.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:30.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:30.976 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:31.014 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:31.099 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:31.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:31.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:31.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:31.274 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:31.461 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:23:31.461 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:23:31.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[186706]: pgmap v618: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-06-10T21:23:31.580 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1710120885' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:31.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:31.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[180297]: pgmap v618: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-06-10T21:23:31.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1710120885' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:31.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:31 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:31.601 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 9 pgp_num to 16 2023-06-10T21:23:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:31 smithi152 ceph-mon[156819]: pgmap v618: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-06-10T21:23:31.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1710120885' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:31.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:31 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:23:32.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:32.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:32.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:32.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:32.322 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:32.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:32.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:32.493 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:32.537 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:32.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:32.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:32.689 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[180297]: pgmap v619: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:32.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[180297]: failed to encode map e155 with expected crc 2023-06-10T21:23:32.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:23:32.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[180297]: osdmap e155: 8 total, 8 up, 8 in 2023-06-10T21:23:32.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[180297]: failed to encode map e155 with expected crc 2023-06-10T21:23:32.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[186706]: pgmap v619: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:32.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[186706]: failed to encode map e155 with expected crc 2023-06-10T21:23:32.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:23:32.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[186706]: osdmap e155: 8 total, 8 up, 8 in 2023-06-10T21:23:32.691 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:32 smithi064 ceph-mon[186706]: failed to encode map e155 with expected crc 2023-06-10T21:23:32.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:23:32.695 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:32.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:32 smithi152 ceph-mon[156819]: pgmap v619: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-06-10T21:23:32.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:32 smithi152 ceph-mon[156819]: failed to encode map e155 with expected crc 2023-06-10T21:23:32.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:32 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:23:32.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:32 smithi152 ceph-mon[156819]: osdmap e155: 8 total, 8 up, 8 in 2023-06-10T21:23:32.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:32 smithi152 ceph-mon[156819]: failed to encode map e155 with expected crc 2023-06-10T21:23:33.005 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:32] "GET /metrics HTTP/1.1" 200 223531 "" "Prometheus/2.33.4" 2023-06-10T21:23:33.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:33.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:33.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:23:33.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:33.290 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:33.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:33.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:33.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:33.658 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:33.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:33.803 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:33.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:33.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:33.986 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:34.070 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:34.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:34.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:34.181 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:23:34.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:34.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:34.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:34.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3182859722' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.252 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.265 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:34.268 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:34.338 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:34.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:34.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:34.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:34.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3182859722' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:34.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:34.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3182859722' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:23:34.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:34.495 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:34.579 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:34.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:35.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:35.384 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: pgmap v621: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:23:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: osdmap e156: 8 total, 8 up, 8 in 2023-06-10T21:23:35.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[180297]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: pgmap v621: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: osdmap e156: 8 total, 8 up, 8 in 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:35 smithi064 ceph-mon[186706]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.433 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: pgmap v621: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: osdmap e156: 8 total, 8 up, 8 in 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:35 smithi152 ceph-mon[156819]: failed to encode map e156 with expected crc 2023-06-10T21:23:35.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:35.571 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:35.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:35.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:35.681 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:35.712 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:36.160 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:36.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:36.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:36.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:36.345 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:36.544 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:36.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:36.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:36.678 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:36.719 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:36.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:36.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:36.887 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:36.982 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:36.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:37.100 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:37.163 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:37.244 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:37.259 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:37 smithi152 ceph-mon[156819]: pgmap v623: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-06-10T21:23:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:37.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:37.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:37.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:37 smithi064 ceph-mon[180297]: pgmap v623: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-06-10T21:23:37.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:37 smithi064 ceph-mon[186706]: pgmap v623: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-06-10T21:23:37.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:37.610 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:37.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:37.614 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:37.650 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:37.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:37.715 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:37.740 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:37.763 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:37.773 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:37.830 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:37.855 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:37.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:37.867 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:37.899 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:23:37.916 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:37.936 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:23:37.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:38.014 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:38.023 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:23:38.024 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:23:38.024 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-10T21:23:38.025 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-06-10T21:23:38.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 1 0 2023-06-10T21:23:38.043 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:23:38.043 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:23:38.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.044 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:161803 9.0 9:0cb4de6c:::benchmark_data_smithi064_247471_object52815:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-06-10T21:23:38.044 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:38.016453+0000", 2023-06-10T21:23:38.044 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013964781000000001, 2023-06-10T21:23:38.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013997747, 2023-06-10T21:23:38.045 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.046 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:23:38.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:23:38.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:23:38.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 161803 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016453+0000", 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016453+0000", 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016451+0000", 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016457+0000", 2023-06-10T21:23:38.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9190000000000004e-06 2023-06-10T21:23:38.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016471+0000", 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3816e-05 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.016480+0000", 2023-06-10T21:23:38.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0100000000000001e-06 2023-06-10T21:23:38.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:23:38.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028156+0000", 2023-06-10T21:23:38.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011676130999999999 2023-06-10T21:23:38.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:23:38.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028285+0000", 2023-06-10T21:23:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012851200000000001 2023-06-10T21:23:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:23:38.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028416+0000", 2023-06-10T21:23:38.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130999 2023-06-10T21:23:38.055 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.055 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.055 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:161806 9.6 e156/127 9:6671d476:::benchmark_data_smithi064_247471_object52818:head v 156'10038, mlcod=156'10036)", 2023-06-10T21:23:38.056 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:38.020381+0000", 2023-06-10T21:23:38.057 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.010036653, 2023-06-10T21:23:38.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010157282, 2023-06-10T21:23:38.057 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.057 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:23:38.058 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020381+0000", 2023-06-10T21:23:38.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020381+0000", 2023-06-10T21:23:38.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020380+0000", 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020390+0000", 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3700000000000001e-06 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020402+0000", 2023-06-10T21:23:38.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2585999999999999e-05 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.020408+0000", 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1589999999999998e-06 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:23:38.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.023433+0000", 2023-06-10T21:23:38.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030244320000000001 2023-06-10T21:23:38.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.023453+0000", 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0038999999999999e-05 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35400.0:161795 9.0 e156/127)", 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:38.024479+0000", 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0059389389999999999, 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061203359999999997, 2023-06-10T21:23:38.065 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024479+0000", 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024479+0000", 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024478+0000", 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024484+0000", 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5650000000000001e-06 2023-06-10T21:23:38.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024495+0000", 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0855e-05 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024502+0000", 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3799999999999996e-06 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.070 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.35400.0:161800 9.0 e156/127)", 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:38.024652+0000", 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0057655609999999998, 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059948290000000001, 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024652+0000", 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024652+0000", 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024652+0000", 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:38.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024656+0000", 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8990000000000003e-06 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024664+0000", 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4039999999999995e-06 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.024670+0000", 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3369999999999996e-06 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.075 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:161809 9.e e156/127)", 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:38.028553+0000", 2023-06-10T21:23:38.076 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0018647889999999999, 2023-06-10T21:23:38.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002142396, 2023-06-10T21:23:38.077 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.077 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:23:38.077 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028553+0000", 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028553+0000", 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028552+0000", 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028560+0000", 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8539999999999997e-06 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028569+0000", 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7620000000000001e-06 2023-06-10T21:23:38.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.028576+0000", 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6800000000000004e-06 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.082 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:161790 9.0 9:0cd96e9d:::benchmark_data_smithi064_247471_object52802:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:37.997910+0000", 2023-06-10T21:23:38.083 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.032507435000000001, 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032838818999999998, 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:23:38.084 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 161790 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997910+0000", 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997910+0000", 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997909+0000", 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997916+0000", 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1420000000000004e-06 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997926+0000", 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0226e-05 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997933+0000", 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5930000000000002e-06 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:23:38.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.010253+0000", 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012319801999999999 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.010388+0000", 2023-06-10T21:23:38.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013551500000000001 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.010497+0000", 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108937 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.017749+0000", 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072522309999999996 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:161795 9.0 9:062254fd:::benchmark_data_smithi064_247471_object52807:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:37.997974+0000", 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.032444239, 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032858353999999999, 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.093 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 161795 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997974+0000", 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997974+0000", 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997973+0000", 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997976+0000", 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4109999999999999e-06 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997982+0000", 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9399999999999999e-06 2023-06-10T21:23:38.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997987+0000", 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.994e-06 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:23:38.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.014203+0000", 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016216556999999999 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.014315+0000", 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011183099999999999 2023-06-10T21:23:38.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.014434+0000", 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011867900000000001 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.100 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:161800 9.0 9:07f4fc8b:::benchmark_data_smithi064_247471_object52812:head [delete] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:23:37.997991+0000", 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.032426918999999998, 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032915967999999997, 2023-06-10T21:23:38.101 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 161800 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:23:38.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997991+0000", 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997991+0000", 2023-06-10T21:23:38.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997991+0000", 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997993+0000", 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1210000000000001e-06 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.997998+0000", 2023-06-10T21:23:38.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9660000000000004e-06 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:37.998002+0000", 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3780000000000003e-06 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.017763+0000", 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019760550000000002 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.017839+0000", 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6748000000000004e-05 2023-06-10T21:23:38.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:23:38.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:23:38.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:23:38.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:23:38.018286+0000", 2023-06-10T21:23:38.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00044700700000000002 2023-06-10T21:23:38.108 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2023-06-10T21:23:38.109 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:23:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:38.111 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:38.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:38.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:38.257 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1897072075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:38.262 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1897072075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:38.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:38.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1897072075' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:39.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:39.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:39.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:39.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:39.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:39 smithi064 ceph-mon[180297]: pgmap v624: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:39.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:39 smithi064 ceph-mon[186706]: pgmap v624: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:39.558 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:39.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:23:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:39.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:39.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:39 smithi152 ceph-mon[156819]: pgmap v624: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:39.740 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:39.761 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:39.808 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:39.943 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:39.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:39.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:40.059 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:40.167 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:40.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:40.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:40.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:40.694 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:40.795 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:40.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:40.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:40.929 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:40.980 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:40.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:41.050 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:41.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:41.146 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:41.264 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:41.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:41.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:41.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:23:41.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:41 smithi064 ceph-mon[186706]: pgmap v625: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-10T21:23:41.370 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:41 smithi064 ceph-mon[180297]: pgmap v625: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-10T21:23:41.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:41.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:41.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:41 smithi152 ceph-mon[156819]: pgmap v625: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-10T21:23:41.902 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:41.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:42.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:42.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:42.764 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:42.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:42.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:42.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:42.889 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:42] "GET /metrics HTTP/1.1" 200 223525 "" "Prometheus/2.33.4" 2023-06-10T21:23:42.955 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:43.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:43 smithi064 ceph-mon[180297]: pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:43.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:43 smithi064 ceph-mon[186706]: pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:43.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:43.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:43 smithi152 ceph-mon[156819]: pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-06-10T21:23:43.790 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:43.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:44.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:44.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:44.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:44.255 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:44.325 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:44.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:44.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:44.341 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:44.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:44.481 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:44.572 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:44.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:44.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:44.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:44.741 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:44.820 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:44.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:44.866 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:23:44.867 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:23:44.867 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:23:44.867 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:23:44.868 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:23:44.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:44.985 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:45.119 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:45.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:45.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:45.317 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 0 (802) 2023-06-10T21:23:45.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:45.328 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:23:45.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[180297]: pgmap v627: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-10T21:23:45.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1218478579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3242911669' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[186706]: pgmap v627: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1218478579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3242911669' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:45.646 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:45.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:45.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:45 smithi152 ceph-mon[156819]: pgmap v627: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-10T21:23:45.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1218478579' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:23:45.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3242911669' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-06-10T21:23:45.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:45.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:46.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:46.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:46.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-06-10T21:23:46.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: osdmap e157: 8 total, 8 up, 8 in 2023-06-10T21:23:46.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[180297]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:46.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:46.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-06-10T21:23:46.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: osdmap e157: 8 total, 8 up, 8 in 2023-06-10T21:23:46.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:46 smithi064 ceph-mon[186706]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:23:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-06-10T21:23:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: osdmap e157: 8 total, 8 up, 8 in 2023-06-10T21:23:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:46 smithi152 ceph-mon[156819]: failed to encode map e157 with expected crc 2023-06-10T21:23:46.812 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:46.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:46.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:46.956 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:46.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:23:47.055 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:47.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:47.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:47.190 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:47.231 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:47.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:47.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:47.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:47.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: pgmap v629: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-10T21:23:47.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: osdmap e158: 8 total, 8 up, 8 in 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.341 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[186706]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: pgmap v629: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-10T21:23:47.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: osdmap e158: 8 total, 8 up, 8 in 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:47 smithi064 ceph-mon[180297]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:47.634 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:47.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: pgmap v629: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-10T21:23:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: osdmap e158: 8 total, 8 up, 8 in 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:47 smithi152 ceph-mon[156819]: failed to encode map e158 with expected crc 2023-06-10T21:23:47.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:47.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:48.060 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:48.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:48.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:48.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:48.245 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:48.255 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:48.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:48.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:48.399 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:48.429 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:48.458 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:48.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:48.497 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:48.532 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:48.556 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:48.564 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:48.599 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:48.634 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:48.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:23:48.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:48.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:48.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:48.890 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:49.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[180297]: pgmap v631: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-10T21:23:49.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:49.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:49.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1147957126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:49.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[186706]: pgmap v631: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-10T21:23:49.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:49.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:49.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1147957126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:49 smithi152 ceph-mon[156819]: pgmap v631: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-10T21:23:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:23:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:23:49.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1147957126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:49.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:23:50.163 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:50.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:50.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:50.330 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:50.403 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:50.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:50.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:50.574 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:50.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:50.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:50.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:50.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:50.892 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:51.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:51.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:51.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:51.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:51.210 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:51.262 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:23:51.263 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:23:51.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:51 smithi064 ceph-mon[180297]: pgmap v632: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:51.264 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:51 smithi064 ceph-mon[186706]: pgmap v632: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:51.284 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:51.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:51.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:51.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:51 smithi152 ceph-mon[156819]: pgmap v632: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:51.467 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:51.551 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:51.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:51.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:51.696 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:52.363 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:23:52.364 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:23:52.364 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-06-10T21:23:52.364 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:23:52.364 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:23:52.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-06-10T21:23:52.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:23:52.856 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:52.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:52.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:23:52.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:53.024 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:53.103 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:53.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:23:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:23:52] "GET /metrics HTTP/1.1" 200 223535 "" "Prometheus/2.33.4" 2023-06-10T21:23:53.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:23:53.279 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:53 smithi152 ceph-mon[156819]: pgmap v633: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:53.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:53.357 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:23:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:53.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:23:53.400 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:53.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:53.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:53 smithi064 ceph-mon[180297]: pgmap v633: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:53.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:53 smithi064 ceph-mon[186706]: pgmap v633: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:53.552 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:53.625 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:53.829 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:53.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:53.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:53.986 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:54.101 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:54.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:54.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:54.209 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:23:54.214 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:23:54.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1932805274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.264 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.271 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:54.384 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:23:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:54.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:23:54.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1932805274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1932805274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:23:54.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:54.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:55.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: pgmap v634: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:55.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:55.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: osdmap e159: 8 total, 8 up, 8 in 2023-06-10T21:23:55.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[180297]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: pgmap v634: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: osdmap e159: 8 total, 8 up, 8 in 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.473 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:55 smithi064 ceph-mon[186706]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: pgmap v634: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:55.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: osdmap e159: 8 total, 8 up, 8 in 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: failed to encode map e159 with expected crc 2023-06-10T21:23:55.474 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:55 smithi152 ceph-mon[156819]: failed to encode map e159 with expected crc 2023-06-10T21:23:56.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:56.137 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:23:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:56.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:56.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:23:56.282 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:56.324 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:23:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:56.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:23:56.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:23:56.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:57.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:23:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:57.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:23:57.118 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:23:57.126 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:23:57.127 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:23:57.127 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:23:57.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:57.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:57 smithi064 ceph-mon[180297]: pgmap v636: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:57.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:57 smithi064 ceph-mon[186706]: pgmap v636: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:57.577 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:23:57.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:57.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:23:57.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:57.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:57 smithi152 ceph-mon[156819]: pgmap v636: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:57.767 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:57.849 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:57.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:57.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:57.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:58.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:58.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:23:58.305 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:23:58.306 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:23:58.306 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:23:58.306 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:23:58.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:23:58.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:23:58.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:23:58.308 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:23:58.309 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.309 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.310 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.311 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.311 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.311 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.312 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.312 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:23:58.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:58.323 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2362162276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:58.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2362162276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:58.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:23:58.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:23:58.456 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:58.497 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:23:58.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2362162276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:23:58.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:58.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:23:58.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:23:58.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:59.223 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:23:59 smithi064 ceph-mon[180297]: pgmap v637: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:23:59.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:59.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:23:59.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:23:59.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:23:59.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:23:59 smithi064 ceph-mon[186706]: pgmap v637: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:23:59 smithi152 ceph-mon[156819]: pgmap v637: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:23:59.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:23:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:23:59.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:00.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:00.103 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:00.142 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:00.195 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:00.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:00.251 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:24:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:24:00.252 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:24:00.252 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:24:00.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:00.360 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:00.469 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:00.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:00.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:00.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:00.675 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:01.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:01 smithi064 ceph-mon[186706]: pgmap v638: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:24:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:01 smithi064 ceph-mon[180297]: pgmap v638: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:24:01.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:01 smithi152 ceph-mon[156819]: pgmap v638: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-06-10T21:24:01.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:24:01.882 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:01.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:01.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:24:01.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:02.105 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:02.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:02.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:02.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:02.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:02.497 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:02.579 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:02.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:02.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:02.754 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:02.820 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:02.831 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:02.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:02.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:02.963 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:03.002 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:03.084 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:03.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:03.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:02] "GET /metrics HTTP/1.1" 200 223535 "" "Prometheus/2.33.4" 2023-06-10T21:24:03.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:03.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:03 smithi152 ceph-mon[156819]: pgmap v639: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:24:03.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:03.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:03 smithi064 ceph-mon[180297]: pgmap v639: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:24:03.457 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:03.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:03 smithi064 ceph-mon[186706]: pgmap v639: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:24:03.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:03.635 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:03.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:03.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:03.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:03.792 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:03.893 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:03.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:03.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:03.999 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:24:03.999 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:24:03.999 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:24:04.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 0 2023-06-10T21:24:04.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:04.071 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:04.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:04.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:04.268 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:24:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:04.760 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:04.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:04.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:04.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:04.919 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:04.974 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:04.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:05.080 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:05.179 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:05.290 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:05.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:05.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:05.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: pgmap v640: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:24:05.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:05.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:24:05.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:05.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: osdmap e160: 8 total, 8 up, 8 in 2023-06-10T21:24:05.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:05 smithi152 ceph-mon[156819]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.471 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:05.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: pgmap v640: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:24:05.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:05.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:24:05.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:05.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: osdmap e160: 8 total, 8 up, 8 in 2023-06-10T21:24:05.523 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[186706]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: pgmap v640: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-06-10T21:24:05.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:05.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:24:05.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/64191204' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:05.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: osdmap e160: 8 total, 8 up, 8 in 2023-06-10T21:24:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: failed to encode map e160 with expected crc 2023-06-10T21:24:05.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:05 smithi064 ceph-mon[180297]: failed to encode map e160 with expected crc 2023-06-10T21:24:06.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:06.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:06.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:06.212 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:06.324 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:06.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:06.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:06.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:06 smithi064 ceph-mon[180297]: failed to encode map e160 with expected crc 2023-06-10T21:24:06.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:06 smithi064 ceph-mon[186706]: failed to encode map e160 with expected crc 2023-06-10T21:24:06.522 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:06.596 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:06.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:06.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:06.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:06.676 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:06 smithi152 ceph-mon[156819]: failed to encode map e160 with expected crc 2023-06-10T21:24:06.713 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:06.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:06.764 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:06.809 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:06.810 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:06.843 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:06.880 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:06.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:06.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:06.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:06.916 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:06.924 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:06.925 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:06.966 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:06.976 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.976 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.976 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.976 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:24:06.977 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:24:07.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:07.068 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:07.132 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:07.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:07.238 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:07.302 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:07.412 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:07.418 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:07 smithi152 ceph-mon[156819]: pgmap v642: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-10T21:24:07.418 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1409142697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:07.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:07.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:07.573 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:07.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:07 smithi064 ceph-mon[186706]: pgmap v642: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-10T21:24:07.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1409142697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:07.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:07 smithi064 ceph-mon[180297]: pgmap v642: 89 pgs: 89 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-10T21:24:07.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1409142697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:07.663 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:07.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:07.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:07.854 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:07.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:24:08.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:24:08.092 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:08.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:08.203 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:08.262 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:08.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:08.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:08.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:08.962 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:09.051 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:09.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:09.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:09.210 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:09.287 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:09.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:09.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:09.417 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:09 smithi064 ceph-mon[186706]: pgmap v643: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:24:09.417 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:09 smithi064 ceph-mon[180297]: pgmap v643: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:24:09.448 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:09.529 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:09.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:09 smithi152 ceph-mon[156819]: pgmap v643: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2023-06-10T21:24:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:09.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:09.542 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:09.609 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:09.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:09.692 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:09.790 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:09.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:09.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:09.991 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:10 smithi064 conmon[180272]: 2023-06-10T21:24:10.446+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e160 definitely_dead 0 2023-06-10T21:24:11.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:24:11.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:11.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:11.313 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.0. 2023-06-10T21:24:11.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:11.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:11.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[180297]: pgmap v644: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:24:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/661877976' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[180297]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:24:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[186706]: pgmap v644: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:24:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/661877976' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:11 smithi064 ceph-mon[186706]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:24:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:11 smithi152 ceph-mon[156819]: pgmap v644: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-06-10T21:24:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/661877976' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:24:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:11 smithi152 ceph-mon[156819]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:24:12.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:12.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:12.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:12.395 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:12.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: osdmap e161: 8 total, 8 up, 7 in 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[180297]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: osdmap e161: 8 total, 8 up, 7 in 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:12 smithi064 ceph-mon[186706]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:12.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:12.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:12.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:12.661 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:24:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: osdmap e161: 8 total, 8 up, 7 in 2023-06-10T21:24:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:12 smithi152 ceph-mon[156819]: failed to encode map e161 with expected crc 2023-06-10T21:24:12.716 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:12.801 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:12.802 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:13.038 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:12] "GET /metrics HTTP/1.1" 200 223541 "" "Prometheus/2.33.4" 2023-06-10T21:24:13.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:13.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:13.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: pgmap v646: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:24:13.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: osdmap e162: 8 total, 8 up, 7 in 2023-06-10T21:24:13.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:13.316 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:24:13.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:13.420 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:13.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: pgmap v646: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: osdmap e162: 8 total, 8 up, 7 in 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:13.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: pgmap v646: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 659 op/s 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: osdmap e162: 8 total, 8 up, 7 in 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: failed to encode map e162 with expected crc 2023-06-10T21:24:13.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:14.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:14.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:14.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:14.280 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:14.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:24:14.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: osdmap e163: 8 total, 8 up, 7 in 2023-06-10T21:24:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.588 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 160'11574 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 160'11522 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'11399 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 160'11247 2023-06-10T21:24:14.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[180297]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: osdmap e163: 8 total, 8 up, 7 in 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.595 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 160'11574 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 160'11522 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'11399 2023-06-10T21:24:14.596 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 160'11247 2023-06-10T21:24:14.597 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:14 smithi064 ceph-mon[186706]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:14.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1011847091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: osdmap e163: 8 total, 8 up, 7 in 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 160'11574 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 160'11522 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 160'11399 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 160'11247 2023-06-10T21:24:14.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:14 smithi152 ceph-mon[156819]: failed to encode map e163 with expected crc 2023-06-10T21:24:14.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:14.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:14.810 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:14.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:15.316 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: pgmap v649: 89 pgs: 7 peering, 82 active+clean; 2.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-06-10T21:24:15.317 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:24:15.317 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.317 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: osdmap e164: 8 total, 8 up, 7 in 2023-06-10T21:24:15.317 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.317 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[180297]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: pgmap v649: 89 pgs: 7 peering, 82 active+clean; 2.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-06-10T21:24:15.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:24:15.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: osdmap e164: 8 total, 8 up, 7 in 2023-06-10T21:24:15.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:15 smithi064 ceph-mon[186706]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.582 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:15.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: pgmap v649: 89 pgs: 7 peering, 82 active+clean; 2.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 530 op/s 2023-06-10T21:24:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:24:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: osdmap e164: 8 total, 8 up, 7 in 2023-06-10T21:24:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:15 smithi152 ceph-mon[156819]: failed to encode map e164 with expected crc 2023-06-10T21:24:15.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:15.748 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:15.792 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:15.828 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:15.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:15.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:16.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:16.136 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:16.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:16.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:16.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:16.335 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:16.336 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:16 smithi152 ceph-mon[156819]: osdmap e165: 8 total, 8 up, 7 in 2023-06-10T21:24:16.336 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:16 smithi152 ceph-mon[156819]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.337 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:16 smithi152 ceph-mon[156819]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.337 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:16 smithi152 ceph-mon[156819]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.337 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/480108222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:16.365 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:16.394 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:16.437 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:16.471 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:16.511 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:16.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:16.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[186706]: osdmap e165: 8 total, 8 up, 7 in 2023-06-10T21:24:16.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[186706]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[186706]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[186706]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/480108222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:16.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[180297]: osdmap e165: 8 total, 8 up, 7 in 2023-06-10T21:24:16.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[180297]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[180297]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[180297]: failed to encode map e165 with expected crc 2023-06-10T21:24:16.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/480108222' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:16.599 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:16.612 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:16.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:16.637 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:16.672 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.672 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.672 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.673 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:24:16.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:16.807 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:16.912 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:16.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:16.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:17.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:17.097 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:17.282 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:17.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:17.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:24:17.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:17.458 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:24:17.458 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:24:17.459 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-06-10T21:24:17.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:24:17.514 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:17.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:17 smithi064 ceph-mon[180297]: pgmap v652: 89 pgs: 3 active+remapped+backfilling, 8 peering, 78 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6115/101994 objects misplaced (5.995%); 3.0 MiB/s, 48 objects/s recovering 2023-06-10T21:24:17.562 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:17 smithi064 ceph-mon[186706]: pgmap v652: 89 pgs: 3 active+remapped+backfilling, 8 peering, 78 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6115/101994 objects misplaced (5.995%); 3.0 MiB/s, 48 objects/s recovering 2023-06-10T21:24:17.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:17.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:17.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:17 smithi152 ceph-mon[156819]: pgmap v652: 89 pgs: 3 active+remapped+backfilling, 8 peering, 78 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6115/101994 objects misplaced (5.995%); 3.0 MiB/s, 48 objects/s recovering 2023-06-10T21:24:17.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:17.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:18.471 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:18.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:18.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:18.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:18.835 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:18.859 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:18.979 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:18.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:19.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:19.139 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:19.236 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:19.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:19.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:19.324 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:24:19.324 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:24:19.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:187026 9.1 9:8cea6d08:::benchmark_data_smithi064_247471_object78038:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.297310+0000", 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.013997651, 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014030621, 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:24:19.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 187026 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297310+0000", 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297310+0000", 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297308+0000", 2023-06-10T21:24:19.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297316+0000", 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6869999999999994e-06 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297337+0000", 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0934e-05 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297348+0000", 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0885e-05 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297413+0000", 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5201000000000001e-05 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297564+0000", 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015060799999999999 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-06-10T21:24:19.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.297703+0000", 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139355 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.301861+0000", 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041580719999999996 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.310156+0000", 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082951080000000007 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:187030 9.7 e165/127 9:e527acc4:::benchmark_data_smithi064_247471_object78042:head v 165'11603, mlcod=165'11601)", 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.302826+0000", 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0084818169999999991, 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086190539999999993, 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302826+0000", 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302826+0000", 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302825+0000", 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302830+0000", 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4349999999999996e-06 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302842+0000", 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1342000000000001e-05 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302849+0000", 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8920000000000003e-06 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302904+0000", 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5050000000000003e-05 2023-06-10T21:24:19.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.302926+0000", 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1905000000000001e-05 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:187031 9.8 9:1c04c187:::benchmark_data_smithi064_247471_object78043:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.303003+0000", 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0083046790000000006, 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085119480000000001, 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.342 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 187031 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303003+0000", 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303003+0000", 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303002+0000", 2023-06-10T21:24:19.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:24:19.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303008+0000", 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.443e-06 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303021+0000", 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2615e-05 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303031+0000", 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0101999999999999e-05 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303132+0000", 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101087 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303264+0000", 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013206200000000001 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303417+0000", 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015305200000000001 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307439+0000", 2023-06-10T21:24:19.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040216169999999999 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:187032 9.c 9:3e669629:::benchmark_data_smithi064_247471_object78044:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-06-10T21:24:19.351 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.303052+0000", 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0082550149999999992, 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085583199999999995, 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:19.352 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 187032 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303052+0000", 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303052+0000", 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303052+0000", 2023-06-10T21:24:19.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303055+0000", 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5099999999999999e-06 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303067+0000", 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1919000000000001e-05 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303074+0000", 2023-06-10T21:24:19.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7490000000000001e-06 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303129+0000", 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4799000000000002e-05 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303255+0000", 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125656 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303382+0000", 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012795600000000001 2023-06-10T21:24:19.358 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:187034 9.c 9:3a8abe3e:::benchmark_data_smithi064_247471_object78046:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.303088+0000", 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0082189740000000004, 2023-06-10T21:24:19.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00859913, 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 187034 2023-06-10T21:24:19.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303088+0000", 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303088+0000", 2023-06-10T21:24:19.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303088+0000", 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.303091+0000", 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1860000000000001e-06 2023-06-10T21:24:19.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307213+0000", 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041217529999999997 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307225+0000", 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.204e-05 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307293+0000", 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8199000000000002e-05 2023-06-10T21:24:19.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307511+0000", 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021772 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:24:19.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307653+0000", 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141695 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35400.0:187035 9.c 9.e6fbfc8c (undecoded) ondisk+write+known_if_redirected e165)", 2023-06-10T21:24:19.367 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.307268+0000", 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0040399329999999999, 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044931959999999996, 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35400", 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:19.368 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 187035 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307268+0000", 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307268+0000", 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307266+0000", 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307273+0000", 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8290000000000002e-06 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307284+0000", 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1124999999999999e-05 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.307291+0000", 2023-06-10T21:24:19.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3659999999999997e-06 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:187041 9.7 e165/127 9:e88c5c4d:::benchmark_data_smithi064_247471_object78053:head v 165'11604, mlcod=165'11601)", 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.309612+0000", 2023-06-10T21:24:19.373 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001695677, 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002206721, 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309612+0000", 2023-06-10T21:24:19.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309612+0000", 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309611+0000", 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309617+0000", 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2319999999999998e-06 2023-06-10T21:24:19.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309633+0000", 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.611e-05 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309641+0000", 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9319999999999996e-06 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309701+0000", 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9509999999999998e-05 2023-06-10T21:24:19.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.309722+0000", 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0808000000000001e-05 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:187039 9.4 e165/157 9:2ea00f4e:::benchmark_data_smithi064_247471_object78051:head v 165'11806, mlcod=165'11804)", 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:19.311091+0000", 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.000216317, 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00078963700000000004, 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:24:19.380 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:24:19.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311091+0000", 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:24:19.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311091+0000", 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311091+0000", 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:19.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311096+0000", 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2850000000000002e-06 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:24:19.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311111+0000", 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5255e-05 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311117+0000", 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1680000000000001e-06 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311515+0000", 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039783199999999998 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:24:19.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:19.311535+0000", 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0064e-05 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2023-06-10T21:24:19.387 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:24:19.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:19.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[186706]: pgmap v653: 89 pgs: 4 active+remapped+backfilling, 1 peering, 84 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 7788/100575 objects misplaced (7.743%); 6.0 MiB/s, 96 objects/s recovering 2023-06-10T21:24:19.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:19.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:19.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:24:19.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[180297]: pgmap v653: 89 pgs: 4 active+remapped+backfilling, 1 peering, 84 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 7788/100575 objects misplaced (7.743%); 6.0 MiB/s, 96 objects/s recovering 2023-06-10T21:24:19.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:19.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:19.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:19 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:24:19.436 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:19 smithi152 ceph-mon[156819]: pgmap v653: 89 pgs: 4 active+remapped+backfilling, 1 peering, 84 active+clean; 2.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 364 op/s; 7788/100575 objects misplaced (7.743%); 6.0 MiB/s, 96 objects/s recovering 2023-06-10T21:24:19.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:19.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:19.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:19 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:24:19.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:19.546 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:19.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:19.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:19.733 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:19.840 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:19.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:19.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:20.058 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:20.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:24:20.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:20.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:20.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:20.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:20.757 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:20.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:24:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:21 smithi152 ceph-mon[156819]: pgmap v654: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 6531/96546 objects misplaced (6.765%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:24:21.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:21 smithi064 ceph-mon[180297]: pgmap v654: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 6531/96546 objects misplaced (6.765%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:24:21.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:21 smithi064 ceph-mon[186706]: pgmap v654: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 2.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 6531/96546 objects misplaced (6.765%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:24:21.999 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:22.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:22.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:22.175 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:22.306 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:22.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:22.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:22.424 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:22.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:22.758 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:22.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:22.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:22.923 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:22.933 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:23.056 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:23.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:22] "GET /metrics HTTP/1.1" 200 223629 "" "Prometheus/2.33.4" 2023-06-10T21:24:23.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:23.216 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:23.324 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:23.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:23.386 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:24:23.415 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:24:23.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:23.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:23 smithi064 ceph-mon[186706]: pgmap v655: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6242/93618 objects misplaced (6.668%); 12 MiB/s, 195 objects/s recovering 2023-06-10T21:24:23.459 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:23 smithi064 ceph-mon[180297]: pgmap v655: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6242/93618 objects misplaced (6.668%); 12 MiB/s, 195 objects/s recovering 2023-06-10T21:24:23.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:23 smithi152 ceph-mon[156819]: pgmap v655: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 6242/93618 objects misplaced (6.668%); 12 MiB/s, 195 objects/s recovering 2023-06-10T21:24:23.484 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:23.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:23.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:23.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:23.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:23.744 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:24.510 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:24:24.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[180297]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:24.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:24.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2294499399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[186706]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:24.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:24.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2294499399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:24 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.582 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:24.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:24.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:24 smithi152 ceph-mon[156819]: from='client.45668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2294499399' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:24 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:24.727 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: pgmap v656: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 5561/91206 objects misplaced (6.097%); 14 MiB/s, 219 objects/s recovering 2023-06-10T21:24:25.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: osdmap e166: 8 total, 8 up, 7 in 2023-06-10T21:24:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[186706]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: pgmap v656: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 5561/91206 objects misplaced (6.097%); 14 MiB/s, 219 objects/s recovering 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: osdmap e166: 8 total, 8 up, 7 in 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:25 smithi064 ceph-mon[180297]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.623 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:25.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: pgmap v656: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 5561/91206 objects misplaced (6.097%); 14 MiB/s, 219 objects/s recovering 2023-06-10T21:24:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: osdmap e166: 8 total, 8 up, 7 in 2023-06-10T21:24:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:25 smithi152 ceph-mon[156819]: failed to encode map e166 with expected crc 2023-06-10T21:24:25.712 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":655,"stamp":"2023-06-10T21:24:21.534671+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2044073894,"num_objects":31206,"num_object_clones":0,"num_object_copies":93618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6242,"num_objects_unfound":0,"num_objects_dirty":31206,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":187051,"num_write_kb":6977487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1566,"num_bytes_recovered":102237280,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128091,"ondisk_log_size":128091,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":266,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7958024,"kb_used_data":5615184,"kb_used_omap":14,"kb_used_meta":2342641,"kb_avail":648258040,"statfs":{"total":671965249536,"available":663816232960,"internally_reserved":0,"allocated":5749948416,"data_stored":5746192938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15340,"internal_metadata":2398864404},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,5,6,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-277020672,"num_objects":-4227,"num_object_clones":0,"num_object_copies":-12681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6242,"num_objects_unfound":0,"num_objects_dirty":-4227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4227,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1565,"num_bytes_recovered":102237280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005706"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":148,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315433+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:24:14.315433+0000","last_peered":"2023-06-10T21:24:14.315433+0000","last_clean":"2023-06-10T21:24:14.315433+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:24:14.315433+0000","last_undegraded":"2023-06-10T21:24:14.315433+0000","last_fullsized":"2023-06-10T21:24:14.315433+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":125,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.332348+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:24:14.332348+0000","last_peered":"2023-06-10T21:24:14.332348+0000","last_clean":"2023-06-10T21:24:14.332348+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:24:14.332348+0000","last_undegraded":"2023-06-10T21:24:14.332348+0000","last_fullsized":"2023-06-10T21:24:14.332348+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":110,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.331493+0000","last_change":"2023-06-10T21:23:46.309763+0000","last_active":"2023-06-10T21:24:14.331493+0000","last_peered":"2023-06-10T21:24:14.331493+0000","last_clean":"2023-06-10T21:24:14.331493+0000","last_became_active":"2023-06-10T21:23:46.309449+0000","last_became_peered":"2023-06-10T21:23:46.309449+0000","last_unstale":"2023-06-10T21:24:14.331493+0000","last_undegraded":"2023-06-10T21:24:14.331493+0000","last_fullsized":"2023-06-10T21:24:14.331493+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:57:16.635893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":86,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.319252+0000","last_change":"2023-06-10T21:24:12.318443+0000","last_active":"2023-06-10T21:24:14.319252+0000","last_peered":"2023-06-10T21:24:14.319252+0000","last_clean":"2023-06-10T21:24:14.319252+0000","last_became_active":"2023-06-10T21:24:12.318033+0000","last_became_peered":"2023-06-10T21:24:12.318033+0000","last_unstale":"2023-06-10T21:24:14.319252+0000","last_undegraded":"2023-06-10T21:24:14.319252+0000","last_fullsized":"2023-06-10T21:24:14.319252+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":151,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:15.343089+0000","last_change":"2023-06-10T21:24:12.315323+0000","last_active":"2023-06-10T21:24:15.343089+0000","last_peered":"2023-06-10T21:24:15.343089+0000","last_clean":"2023-06-10T21:24:15.343089+0000","last_became_active":"2023-06-10T21:24:12.314142+0000","last_became_peered":"2023-06-10T21:24:12.314142+0000","last_unstale":"2023-06-10T21:24:15.343089+0000","last_undegraded":"2023-06-10T21:24:15.343089+0000","last_fullsized":"2023-06-10T21:24:15.343089+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:04.430893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320952+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:24:14.320952+0000","last_peered":"2023-06-10T21:24:14.320952+0000","last_clean":"2023-06-10T21:24:14.320952+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:24:14.320952+0000","last_undegraded":"2023-06-10T21:24:14.320952+0000","last_fullsized":"2023-06-10T21:24:14.320952+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":85,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335368+0000","last_change":"2023-06-10T21:24:12.318827+0000","last_active":"2023-06-10T21:24:14.335368+0000","last_peered":"2023-06-10T21:24:14.335368+0000","last_clean":"2023-06-10T21:24:14.335368+0000","last_became_active":"2023-06-10T21:24:12.318530+0000","last_became_peered":"2023-06-10T21:24:12.318530+0000","last_unstale":"2023-06-10T21:24:14.335368+0000","last_undegraded":"2023-06-10T21:24:14.335368+0000","last_fullsized":"2023-06-10T21:24:14.335368+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":126,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.331931+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:24:14.331931+0000","last_peered":"2023-06-10T21:24:14.331931+0000","last_clean":"2023-06-10T21:24:14.331931+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:24:14.331931+0000","last_undegraded":"2023-06-10T21:24:14.331931+0000","last_fullsized":"2023-06-10T21:24:14.331931+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335063+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:24:14.335063+0000","last_peered":"2023-06-10T21:24:14.335063+0000","last_clean":"2023-06-10T21:24:14.335063+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:24:14.335063+0000","last_undegraded":"2023-06-10T21:24:14.335063+0000","last_fullsized":"2023-06-10T21:24:14.335063+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":105,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.331429+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:24:14.331429+0000","last_peered":"2023-06-10T21:24:14.331429+0000","last_clean":"2023-06-10T21:24:14.331429+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:24:14.331429+0000","last_undegraded":"2023-06-10T21:24:14.331429+0000","last_fullsized":"2023-06-10T21:24:14.331429+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318799+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:24:14.318799+0000","last_peered":"2023-06-10T21:24:14.318799+0000","last_clean":"2023-06-10T21:24:14.318799+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:24:14.318799+0000","last_undegraded":"2023-06-10T21:24:14.318799+0000","last_fullsized":"2023-06-10T21:24:14.318799+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320407+0000","last_change":"2023-06-10T21:23:46.306714+0000","last_active":"2023-06-10T21:24:14.320407+0000","last_peered":"2023-06-10T21:24:14.320407+0000","last_clean":"2023-06-10T21:24:14.320407+0000","last_became_active":"2023-06-10T21:23:46.306393+0000","last_became_peered":"2023-06-10T21:23:46.306393+0000","last_unstale":"2023-06-10T21:24:14.320407+0000","last_undegraded":"2023-06-10T21:24:14.320407+0000","last_fullsized":"2023-06-10T21:24:14.320407+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315898+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:24:14.315898+0000","last_peered":"2023-06-10T21:24:14.315898+0000","last_clean":"2023-06-10T21:24:14.315898+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:24:14.315898+0000","last_undegraded":"2023-06-10T21:24:14.315898+0000","last_fullsized":"2023-06-10T21:24:14.315898+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315449+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:24:14.315449+0000","last_peered":"2023-06-10T21:24:14.315449+0000","last_clean":"2023-06-10T21:24:14.315449+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:24:14.315449+0000","last_undegraded":"2023-06-10T21:24:14.315449+0000","last_fullsized":"2023-06-10T21:24:14.315449+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":134,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318652+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:24:14.318652+0000","last_peered":"2023-06-10T21:24:14.318652+0000","last_clean":"2023-06-10T21:24:14.318652+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:24:14.318652+0000","last_undegraded":"2023-06-10T21:24:14.318652+0000","last_fullsized":"2023-06-10T21:24:14.318652+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":126,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.338051+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:24:14.338051+0000","last_peered":"2023-06-10T21:24:14.338051+0000","last_clean":"2023-06-10T21:24:14.338051+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:24:14.338051+0000","last_undegraded":"2023-06-10T21:24:14.338051+0000","last_fullsized":"2023-06-10T21:24:14.338051+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":85,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318419+0000","last_change":"2023-06-10T21:24:12.317940+0000","last_active":"2023-06-10T21:24:14.318419+0000","last_peered":"2023-06-10T21:24:14.318419+0000","last_clean":"2023-06-10T21:24:14.318419+0000","last_became_active":"2023-06-10T21:24:12.317592+0000","last_became_peered":"2023-06-10T21:24:12.317592+0000","last_unstale":"2023-06-10T21:24:14.318419+0000","last_undegraded":"2023-06-10T21:24:14.318419+0000","last_fullsized":"2023-06-10T21:24:14.318419+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335012+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:24:14.335012+0000","last_peered":"2023-06-10T21:24:14.335012+0000","last_clean":"2023-06-10T21:24:14.335012+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:24:14.335012+0000","last_undegraded":"2023-06-10T21:24:14.335012+0000","last_fullsized":"2023-06-10T21:24:14.335012+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318514+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:24:14.318514+0000","last_peered":"2023-06-10T21:24:14.318514+0000","last_clean":"2023-06-10T21:24:14.318514+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:24:14.318514+0000","last_undegraded":"2023-06-10T21:24:14.318514+0000","last_fullsized":"2023-06-10T21:24:14.318514+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":85,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.332451+0000","last_change":"2023-06-10T21:24:12.316959+0000","last_active":"2023-06-10T21:24:14.332451+0000","last_peered":"2023-06-10T21:24:14.332451+0000","last_clean":"2023-06-10T21:24:14.332451+0000","last_became_active":"2023-06-10T21:24:12.316593+0000","last_became_peered":"2023-06-10T21:24:12.316593+0000","last_unstale":"2023-06-10T21:24:14.332451+0000","last_undegraded":"2023-06-10T21:24:14.332451+0000","last_fullsized":"2023-06-10T21:24:14.332451+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:52.171479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":159,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320245+0000","last_change":"2023-06-10T21:24:12.326040+0000","last_active":"2023-06-10T21:24:14.320245+0000","last_peered":"2023-06-10T21:24:14.320245+0000","last_clean":"2023-06-10T21:24:14.320245+0000","last_became_active":"2023-06-10T21:24:12.316581+0000","last_became_peered":"2023-06-10T21:24:12.316581+0000","last_unstale":"2023-06-10T21:24:14.320245+0000","last_undegraded":"2023-06-10T21:24:14.320245+0000","last_fullsized":"2023-06-10T21:24:14.320245+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":126,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:15.355551+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:24:15.355551+0000","last_peered":"2023-06-10T21:24:15.355551+0000","last_clean":"2023-06-10T21:24:15.355551+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:24:15.355551+0000","last_undegraded":"2023-06-10T21:24:15.355551+0000","last_fullsized":"2023-06-10T21:24:15.355551+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"165'11734","reported_seq":12073,"reported_epoch":165,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:24:19.658673+0000","last_change":"2023-06-10T21:24:13.542971+0000","last_active":"2023-06-10T21:24:19.658673+0000","last_peered":"2023-06-10T21:24:19.658673+0000","last_clean":"2023-06-10T21:24:11.198920+0000","last_became_active":"2023-06-10T21:24:13.477906+0000","last_became_peered":"2023-06-10T21:24:13.477906+0000","last_unstale":"2023-06-10T21:24:19.658673+0000","last_undegraded":"2023-06-10T21:24:19.658673+0000","last_fullsized":"2023-06-10T21:24:19.658673+0000","mapping_epoch":162,"log_start":"133'2632","ondisk_log_start":"133'2632","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9102,"log_dups_size":0,"ondisk_log_size":9102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123469824,"num_objects":1884,"num_object_clones":0,"num_object_copies":5652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1749,"num_objects_unfound":0,"num_objects_dirty":1884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11734,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,0],"avail_no_missing":["5","0","7"],"object_location_counts":[{"shards":"0,5,7","objects":1884}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.319415+0000","last_change":"2023-06-10T21:24:12.371913+0000","last_active":"2023-06-10T21:24:14.319415+0000","last_peered":"2023-06-10T21:24:14.319415+0000","last_clean":"2023-06-10T21:24:14.319415+0000","last_became_active":"2023-06-10T21:24:12.316118+0000","last_became_peered":"2023-06-10T21:24:12.316118+0000","last_unstale":"2023-06-10T21:24:14.319415+0000","last_undegraded":"2023-06-10T21:24:14.319415+0000","last_fullsized":"2023-06-10T21:24:14.319415+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334420+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:24:14.334420+0000","last_peered":"2023-06-10T21:24:14.334420+0000","last_clean":"2023-06-10T21:24:14.334420+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:24:14.334420+0000","last_undegraded":"2023-06-10T21:24:14.334420+0000","last_fullsized":"2023-06-10T21:24:14.334420+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"165'11805","reported_seq":11865,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.246305+0000","last_change":"2023-06-10T21:23:46.406550+0000","last_active":"2023-06-10T21:24:19.246305+0000","last_peered":"2023-06-10T21:24:19.246305+0000","last_clean":"2023-06-10T21:24:19.246305+0000","last_became_active":"2023-06-10T21:23:46.309661+0000","last_became_peered":"2023-06-10T21:23:46.309661+0000","last_unstale":"2023-06-10T21:24:19.246305+0000","last_undegraded":"2023-06-10T21:24:19.246305+0000","last_fullsized":"2023-06-10T21:24:19.246305+0000","mapping_epoch":157,"log_start":"135'3400","ondisk_log_start":"135'3400","created":127,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8405,"log_dups_size":0,"ondisk_log_size":8405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130482176,"num_objects":1991,"num_object_clones":0,"num_object_copies":5973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11805,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":134,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.319211+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:24:14.319211+0000","last_peered":"2023-06-10T21:24:14.319211+0000","last_clean":"2023-06-10T21:24:14.319211+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:24:14.319211+0000","last_undegraded":"2023-06-10T21:24:14.319211+0000","last_fullsized":"2023-06-10T21:24:14.319211+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315345+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:24:14.315345+0000","last_peered":"2023-06-10T21:24:14.315345+0000","last_clean":"2023-06-10T21:24:14.315345+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:24:14.315345+0000","last_undegraded":"2023-06-10T21:24:14.315345+0000","last_fullsized":"2023-06-10T21:24:14.315345+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"165'11619","reported_seq":11665,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.601344+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:24:19.601344+0000","last_peered":"2023-06-10T21:24:19.601344+0000","last_clean":"2023-06-10T21:24:19.601344+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:24:19.601344+0000","last_undegraded":"2023-06-10T21:24:19.601344+0000","last_fullsized":"2023-06-10T21:24:19.601344+0000","mapping_epoch":127,"log_start":"133'2487","ondisk_log_start":"133'2487","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9132,"log_dups_size":0,"ondisk_log_size":9132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124059648,"num_objects":1893,"num_object_clones":0,"num_object_copies":5679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11619,"num_write_kb":432384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":118,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318713+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:24:14.318713+0000","last_peered":"2023-06-10T21:24:14.318713+0000","last_clean":"2023-06-10T21:24:14.318713+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:24:14.318713+0000","last_undegraded":"2023-06-10T21:24:14.318713+0000","last_fullsized":"2023-06-10T21:24:14.318713+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320342+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:24:14.320342+0000","last_peered":"2023-06-10T21:24:14.320342+0000","last_clean":"2023-06-10T21:24:14.320342+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:24:14.320342+0000","last_undegraded":"2023-06-10T21:24:14.320342+0000","last_fullsized":"2023-06-10T21:24:14.320342+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"165'11642","reported_seq":11688,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.636261+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:24:19.636261+0000","last_peered":"2023-06-10T21:24:19.636261+0000","last_clean":"2023-06-10T21:24:19.636261+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:24:19.636261+0000","last_undegraded":"2023-06-10T21:24:19.636261+0000","last_fullsized":"2023-06-10T21:24:19.636261+0000","mapping_epoch":127,"log_start":"133'2535","ondisk_log_start":"133'2535","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9107,"log_dups_size":0,"ondisk_log_size":9107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128057344,"num_objects":1954,"num_object_clones":0,"num_object_copies":5862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11642,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334908+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:24:14.334908+0000","last_peered":"2023-06-10T21:24:14.334908+0000","last_clean":"2023-06-10T21:24:14.334908+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:24:14.334908+0000","last_undegraded":"2023-06-10T21:24:14.334908+0000","last_fullsized":"2023-06-10T21:24:14.334908+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315375+0000","last_change":"2023-06-10T21:24:12.317115+0000","last_active":"2023-06-10T21:24:14.315375+0000","last_peered":"2023-06-10T21:24:14.315375+0000","last_clean":"2023-06-10T21:24:14.315375+0000","last_became_active":"2023-06-10T21:24:12.316637+0000","last_became_peered":"2023-06-10T21:24:12.316637+0000","last_unstale":"2023-06-10T21:24:14.315375+0000","last_undegraded":"2023-06-10T21:24:14.315375+0000","last_fullsized":"2023-06-10T21:24:14.315375+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"165'11584","reported_seq":11632,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:16.991603+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:24:16.991603+0000","last_peered":"2023-06-10T21:24:16.991603+0000","last_clean":"2023-06-10T21:24:16.991603+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:24:16.991603+0000","last_undegraded":"2023-06-10T21:24:16.991603+0000","last_fullsized":"2023-06-10T21:24:16.991603+0000","mapping_epoch":127,"log_start":"136'4604","ondisk_log_start":"136'4604","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136445952,"num_objects":2082,"num_object_clones":0,"num_object_copies":6246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11584,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":123,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.332702+0000","last_change":"2023-06-10T21:24:12.368995+0000","last_active":"2023-06-10T21:24:14.332702+0000","last_peered":"2023-06-10T21:24:14.332702+0000","last_clean":"2023-06-10T21:24:14.332702+0000","last_became_active":"2023-06-10T21:24:12.316145+0000","last_became_peered":"2023-06-10T21:24:12.316145+0000","last_unstale":"2023-06-10T21:24:14.332702+0000","last_undegraded":"2023-06-10T21:24:14.332702+0000","last_fullsized":"2023-06-10T21:24:14.332702+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:56.329639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315767+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:24:14.315767+0000","last_peered":"2023-06-10T21:24:14.315767+0000","last_clean":"2023-06-10T21:24:14.315767+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:24:14.315767+0000","last_undegraded":"2023-06-10T21:24:14.315767+0000","last_fullsized":"2023-06-10T21:24:14.315767+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"165'11341","reported_seq":11397,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.287418+0000","last_change":"2023-06-10T21:23:46.333354+0000","last_active":"2023-06-10T21:24:19.287418+0000","last_peered":"2023-06-10T21:24:19.287418+0000","last_clean":"2023-06-10T21:24:19.287418+0000","last_became_active":"2023-06-10T21:23:46.306551+0000","last_became_peered":"2023-06-10T21:23:46.306551+0000","last_unstale":"2023-06-10T21:24:19.287418+0000","last_undegraded":"2023-06-10T21:24:19.287418+0000","last_fullsized":"2023-06-10T21:24:19.287418+0000","mapping_epoch":157,"log_start":"134'2800","ondisk_log_start":"134'2800","created":127,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8541,"log_dups_size":0,"ondisk_log_size":8541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120520704,"num_objects":1839,"num_object_clones":0,"num_object_copies":5517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11341,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320236+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:24:14.320236+0000","last_peered":"2023-06-10T21:24:14.320236+0000","last_clean":"2023-06-10T21:24:14.320236+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:24:14.320236+0000","last_undegraded":"2023-06-10T21:24:14.320236+0000","last_fullsized":"2023-06-10T21:24:14.320236+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315635+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:24:14.315635+0000","last_peered":"2023-06-10T21:24:14.315635+0000","last_clean":"2023-06-10T21:24:14.315635+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:24:14.315635+0000","last_undegraded":"2023-06-10T21:24:14.315635+0000","last_fullsized":"2023-06-10T21:24:14.315635+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"165'11847","reported_seq":11893,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.438568+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:24:19.438568+0000","last_peered":"2023-06-10T21:24:19.438568+0000","last_clean":"2023-06-10T21:24:19.438568+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:24:19.438568+0000","last_undegraded":"2023-06-10T21:24:19.438568+0000","last_fullsized":"2023-06-10T21:24:19.438568+0000","mapping_epoch":127,"log_start":"137'4863","ondisk_log_start":"137'4863","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127729664,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11847,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315258+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:24:14.315258+0000","last_peered":"2023-06-10T21:24:14.315258+0000","last_clean":"2023-06-10T21:24:14.315258+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:24:14.315258+0000","last_undegraded":"2023-06-10T21:24:14.315258+0000","last_fullsized":"2023-06-10T21:24:14.315258+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315745+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:24:14.315745+0000","last_peered":"2023-06-10T21:24:14.315745+0000","last_clean":"2023-06-10T21:24:14.315745+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:24:14.315745+0000","last_undegraded":"2023-06-10T21:24:14.315745+0000","last_fullsized":"2023-06-10T21:24:14.315745+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"165'11667","reported_seq":11715,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:17.009531+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:24:17.009531+0000","last_peered":"2023-06-10T21:24:17.009531+0000","last_clean":"2023-06-10T21:24:17.009531+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:24:17.009531+0000","last_undegraded":"2023-06-10T21:24:17.009531+0000","last_fullsized":"2023-06-10T21:24:17.009531+0000","mapping_epoch":127,"log_start":"136'4667","ondisk_log_start":"136'4667","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135856128,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11667,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":127,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.331729+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:24:14.331729+0000","last_peered":"2023-06-10T21:24:14.331729+0000","last_clean":"2023-06-10T21:24:14.331729+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:24:14.331729+0000","last_undegraded":"2023-06-10T21:24:14.331729+0000","last_fullsized":"2023-06-10T21:24:14.331729+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335152+0000","last_change":"2023-06-10T21:23:46.310102+0000","last_active":"2023-06-10T21:24:14.335152+0000","last_peered":"2023-06-10T21:24:14.335152+0000","last_clean":"2023-06-10T21:24:14.335152+0000","last_became_active":"2023-06-10T21:23:46.309636+0000","last_became_peered":"2023-06-10T21:23:46.309636+0000","last_unstale":"2023-06-10T21:24:14.335152+0000","last_undegraded":"2023-06-10T21:24:14.335152+0000","last_fullsized":"2023-06-10T21:24:14.335152+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:13.066763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"165'11620","reported_seq":11666,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.663455+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:24:19.663455+0000","last_peered":"2023-06-10T21:24:19.663455+0000","last_clean":"2023-06-10T21:24:19.663455+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:24:19.663455+0000","last_undegraded":"2023-06-10T21:24:19.663455+0000","last_fullsized":"2023-06-10T21:24:19.663455+0000","mapping_epoch":127,"log_start":"133'2492","ondisk_log_start":"133'2492","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122683392,"num_objects":1872,"num_object_clones":0,"num_object_copies":5616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11620,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":79,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318604+0000","last_change":"2023-06-10T21:24:12.313179+0000","last_active":"2023-06-10T21:24:14.318604+0000","last_peered":"2023-06-10T21:24:14.318604+0000","last_clean":"2023-06-10T21:24:14.318604+0000","last_became_active":"2023-06-10T21:24:12.312782+0000","last_became_peered":"2023-06-10T21:24:12.312782+0000","last_unstale":"2023-06-10T21:24:14.318604+0000","last_undegraded":"2023-06-10T21:24:14.318604+0000","last_fullsized":"2023-06-10T21:24:14.318604+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335094+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:24:14.335094+0000","last_peered":"2023-06-10T21:24:14.335094+0000","last_clean":"2023-06-10T21:24:14.335094+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:24:14.335094+0000","last_undegraded":"2023-06-10T21:24:14.335094+0000","last_fullsized":"2023-06-10T21:24:14.335094+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":111,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.339316+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:24:14.339316+0000","last_peered":"2023-06-10T21:24:14.339316+0000","last_clean":"2023-06-10T21:24:14.339316+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:24:14.339316+0000","last_undegraded":"2023-06-10T21:24:14.339316+0000","last_fullsized":"2023-06-10T21:24:14.339316+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"165'11608","reported_seq":12702,"reported_epoch":165,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:24:19.518065+0000","last_change":"2023-06-10T21:24:13.609062+0000","last_active":"2023-06-10T21:24:19.518065+0000","last_peered":"2023-06-10T21:24:19.518065+0000","last_clean":"2023-06-10T21:24:11.287618+0000","last_became_active":"2023-06-10T21:24:13.501052+0000","last_became_peered":"2023-06-10T21:24:13.501052+0000","last_unstale":"2023-06-10T21:24:19.518065+0000","last_undegraded":"2023-06-10T21:24:19.518065+0000","last_fullsized":"2023-06-10T21:24:19.518065+0000","mapping_epoch":162,"log_start":"136'4623","ondisk_log_start":"136'4623","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126353408,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1442,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11608,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":34013184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,0],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":1928}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334814+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:24:14.334814+0000","last_peered":"2023-06-10T21:24:14.334814+0000","last_clean":"2023-06-10T21:24:14.334814+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:24:14.334814+0000","last_undegraded":"2023-06-10T21:24:14.334814+0000","last_fullsized":"2023-06-10T21:24:14.334814+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":141,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315562+0000","last_change":"2023-06-10T21:24:12.313277+0000","last_active":"2023-06-10T21:24:14.315562+0000","last_peered":"2023-06-10T21:24:14.315562+0000","last_clean":"2023-06-10T21:24:14.315562+0000","last_became_active":"2023-06-10T21:24:12.312911+0000","last_became_peered":"2023-06-10T21:24:12.312911+0000","last_unstale":"2023-06-10T21:24:14.315562+0000","last_undegraded":"2023-06-10T21:24:14.315562+0000","last_fullsized":"2023-06-10T21:24:14.315562+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":126,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.338756+0000","last_change":"2023-06-10T21:23:46.305694+0000","last_active":"2023-06-10T21:24:14.338756+0000","last_peered":"2023-06-10T21:24:14.338756+0000","last_clean":"2023-06-10T21:24:14.338756+0000","last_became_active":"2023-06-10T21:23:46.305252+0000","last_became_peered":"2023-06-10T21:23:46.305252+0000","last_unstale":"2023-06-10T21:24:14.338756+0000","last_undegraded":"2023-06-10T21:24:14.338756+0000","last_fullsized":"2023-06-10T21:24:14.338756+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:15:57.437674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"165'11745","reported_seq":11790,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.663289+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:24:19.663289+0000","last_peered":"2023-06-10T21:24:19.663289+0000","last_clean":"2023-06-10T21:24:19.663289+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:24:19.663289+0000","last_undegraded":"2023-06-10T21:24:19.663289+0000","last_fullsized":"2023-06-10T21:24:19.663289+0000","mapping_epoch":127,"log_start":"133'2615","ondisk_log_start":"133'2615","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9130,"log_dups_size":0,"ondisk_log_size":9130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126156800,"num_objects":1925,"num_object_clones":0,"num_object_copies":5775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11745,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":141,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.338492+0000","last_change":"2023-06-10T21:24:12.314648+0000","last_active":"2023-06-10T21:24:14.338492+0000","last_peered":"2023-06-10T21:24:14.338492+0000","last_clean":"2023-06-10T21:24:14.338492+0000","last_became_active":"2023-06-10T21:24:12.313974+0000","last_became_peered":"2023-06-10T21:24:12.313974+0000","last_unstale":"2023-06-10T21:24:14.338492+0000","last_undegraded":"2023-06-10T21:24:14.338492+0000","last_fullsized":"2023-06-10T21:24:14.338492+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:40.877911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":106,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.339636+0000","last_change":"2023-06-10T21:23:46.307260+0000","last_active":"2023-06-10T21:24:14.339636+0000","last_peered":"2023-06-10T21:24:14.339636+0000","last_clean":"2023-06-10T21:24:14.339636+0000","last_became_active":"2023-06-10T21:23:46.306268+0000","last_became_peered":"2023-06-10T21:23:46.306268+0000","last_unstale":"2023-06-10T21:24:14.339636+0000","last_undegraded":"2023-06-10T21:24:14.339636+0000","last_fullsized":"2023-06-10T21:24:14.339636+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:44:18.334577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335435+0000","last_change":"2023-06-10T21:24:12.321321+0000","last_active":"2023-06-10T21:24:14.335435+0000","last_peered":"2023-06-10T21:24:14.335435+0000","last_clean":"2023-06-10T21:24:14.335435+0000","last_became_active":"2023-06-10T21:24:12.320999+0000","last_became_peered":"2023-06-10T21:24:12.320999+0000","last_unstale":"2023-06-10T21:24:14.335435+0000","last_undegraded":"2023-06-10T21:24:14.335435+0000","last_fullsized":"2023-06-10T21:24:14.335435+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"165'11370","reported_seq":12053,"reported_epoch":165,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:24:16.995290+0000","last_change":"2023-06-10T21:24:13.518017+0000","last_active":"2023-06-10T21:24:16.995290+0000","last_peered":"2023-06-10T21:24:16.995290+0000","last_clean":"2023-06-10T21:24:11.277173+0000","last_became_active":"2023-06-10T21:24:13.413716+0000","last_became_peered":"2023-06-10T21:24:13.413716+0000","last_unstale":"2023-06-10T21:24:16.995290+0000","last_undegraded":"2023-06-10T21:24:16.995290+0000","last_fullsized":"2023-06-10T21:24:16.995290+0000","mapping_epoch":162,"log_start":"136'4371","ondisk_log_start":"136'4371","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:17:38.704900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129499136,"num_objects":1976,"num_object_clones":0,"num_object_copies":5928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1673,"num_objects_unfound":0,"num_objects_dirty":1976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11370,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":20185088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,0],"avail_no_missing":["3","0","5"],"object_location_counts":[{"shards":"0,3,5","objects":1976}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334840+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:24:14.334840+0000","last_peered":"2023-06-10T21:24:14.334840+0000","last_clean":"2023-06-10T21:24:14.334840+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:24:14.334840+0000","last_undegraded":"2023-06-10T21:24:14.334840+0000","last_fullsized":"2023-06-10T21:24:14.334840+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":391,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:15.376027+0000","last_change":"2023-06-10T21:24:15.376027+0000","last_active":"2023-06-10T21:24:15.376027+0000","last_peered":"2023-06-10T21:24:15.376027+0000","last_clean":"2023-06-10T21:24:15.376027+0000","last_became_active":"2023-06-10T21:24:15.375602+0000","last_became_peered":"2023-06-10T21:24:15.375602+0000","last_unstale":"2023-06-10T21:24:15.376027+0000","last_undegraded":"2023-06-10T21:24:15.376027+0000","last_fullsized":"2023-06-10T21:24:15.376027+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":105,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334971+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:24:14.334971+0000","last_peered":"2023-06-10T21:24:14.334971+0000","last_clean":"2023-06-10T21:24:14.334971+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:24:14.334971+0000","last_undegraded":"2023-06-10T21:24:14.334971+0000","last_fullsized":"2023-06-10T21:24:14.334971+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320553+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:24:14.320553+0000","last_peered":"2023-06-10T21:24:14.320553+0000","last_clean":"2023-06-10T21:24:14.320553+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:24:14.320553+0000","last_undegraded":"2023-06-10T21:24:14.320553+0000","last_fullsized":"2023-06-10T21:24:14.320553+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"165'11792","reported_seq":13015,"reported_epoch":165,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:24:19.310078+0000","last_change":"2023-06-10T21:24:13.792453+0000","last_active":"2023-06-10T21:24:19.310078+0000","last_peered":"2023-06-10T21:24:19.310078+0000","last_clean":"2023-06-10T21:24:11.262159+0000","last_became_active":"2023-06-10T21:24:13.722342+0000","last_became_peered":"2023-06-10T21:24:13.722342+0000","last_unstale":"2023-06-10T21:24:19.310078+0000","last_undegraded":"2023-06-10T21:24:19.310078+0000","last_fullsized":"2023-06-10T21:24:19.310078+0000","mapping_epoch":162,"log_start":"133'2399","ondisk_log_start":"133'2399","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9393,"log_dups_size":0,"ondisk_log_size":9393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127008768,"num_objects":1938,"num_object_clones":0,"num_object_copies":5814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1378,"num_objects_unfound":0,"num_objects_dirty":1938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11792,"num_write_kb":439360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":583,"num_bytes_recovered":38207488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,0],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":1938}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":160,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318285+0000","last_change":"2023-06-10T21:24:12.318185+0000","last_active":"2023-06-10T21:24:14.318285+0000","last_peered":"2023-06-10T21:24:14.318285+0000","last_clean":"2023-06-10T21:24:14.318285+0000","last_became_active":"2023-06-10T21:24:12.317818+0000","last_became_peered":"2023-06-10T21:24:12.317818+0000","last_unstale":"2023-06-10T21:24:14.318285+0000","last_undegraded":"2023-06-10T21:24:14.318285+0000","last_fullsized":"2023-06-10T21:24:14.318285+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":86,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315494+0000","last_change":"2023-06-10T21:24:12.317279+0000","last_active":"2023-06-10T21:24:14.315494+0000","last_peered":"2023-06-10T21:24:14.315494+0000","last_clean":"2023-06-10T21:24:14.315494+0000","last_became_active":"2023-06-10T21:24:12.316409+0000","last_became_peered":"2023-06-10T21:24:12.316409+0000","last_unstale":"2023-06-10T21:24:14.315494+0000","last_undegraded":"2023-06-10T21:24:14.315494+0000","last_fullsized":"2023-06-10T21:24:14.315494+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.339381+0000","last_change":"2023-06-10T21:24:12.315336+0000","last_active":"2023-06-10T21:24:14.339381+0000","last_peered":"2023-06-10T21:24:14.339381+0000","last_clean":"2023-06-10T21:24:14.339381+0000","last_became_active":"2023-06-10T21:24:12.315051+0000","last_became_peered":"2023-06-10T21:24:12.315051+0000","last_unstale":"2023-06-10T21:24:14.339381+0000","last_undegraded":"2023-06-10T21:24:14.339381+0000","last_fullsized":"2023-06-10T21:24:14.339381+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:09:37.040761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.e","version":"165'11870","reported_seq":11916,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.496074+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:24:19.496074+0000","last_peered":"2023-06-10T21:24:19.496074+0000","last_clean":"2023-06-10T21:24:19.496074+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:24:19.496074+0000","last_undegraded":"2023-06-10T21:24:19.496074+0000","last_fullsized":"2023-06-10T21:24:19.496074+0000","mapping_epoch":127,"log_start":"136'4871","ondisk_log_start":"136'4871","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121503744,"num_objects":1854,"num_object_clones":0,"num_object_copies":5562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11870,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":145,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.318762+0000","last_change":"2023-06-10T21:24:12.315998+0000","last_active":"2023-06-10T21:24:14.318762+0000","last_peered":"2023-06-10T21:24:14.318762+0000","last_clean":"2023-06-10T21:24:14.318762+0000","last_became_active":"2023-06-10T21:24:12.315407+0000","last_became_peered":"2023-06-10T21:24:12.315407+0000","last_unstale":"2023-06-10T21:24:14.318762+0000","last_undegraded":"2023-06-10T21:24:14.318762+0000","last_fullsized":"2023-06-10T21:24:14.318762+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":106,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.338831+0000","last_change":"2023-06-10T21:23:46.307703+0000","last_active":"2023-06-10T21:24:14.338831+0000","last_peered":"2023-06-10T21:24:14.338831+0000","last_clean":"2023-06-10T21:24:14.338831+0000","last_became_active":"2023-06-10T21:23:46.307381+0000","last_became_peered":"2023-06-10T21:23:46.307381+0000","last_unstale":"2023-06-10T21:24:14.338831+0000","last_undegraded":"2023-06-10T21:24:14.338831+0000","last_fullsized":"2023-06-10T21:24:14.338831+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:01.121461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":28,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335053+0000","last_change":"2023-06-10T21:24:12.319491+0000","last_active":"2023-06-10T21:24:14.335053+0000","last_peered":"2023-06-10T21:24:14.335053+0000","last_clean":"2023-06-10T21:24:14.335053+0000","last_became_active":"2023-06-10T21:24:12.319245+0000","last_became_peered":"2023-06-10T21:24:12.319245+0000","last_unstale":"2023-06-10T21:24:14.335053+0000","last_undegraded":"2023-06-10T21:24:14.335053+0000","last_fullsized":"2023-06-10T21:24:14.335053+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"165'11993","reported_seq":12040,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:19.509544+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:24:19.509544+0000","last_peered":"2023-06-10T21:24:19.509544+0000","last_clean":"2023-06-10T21:24:19.509544+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:24:19.509544+0000","last_undegraded":"2023-06-10T21:24:19.509544+0000","last_fullsized":"2023-06-10T21:24:19.509544+0000","mapping_epoch":127,"log_start":"137'4980","ondisk_log_start":"137'4980","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7013,"log_dups_size":0,"ondisk_log_size":7013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843800,"num_objects":1967,"num_object_clones":0,"num_object_copies":5901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1967,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11993,"num_write_kb":446657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":123,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335074+0000","last_change":"2023-06-10T21:24:12.371516+0000","last_active":"2023-06-10T21:24:14.335074+0000","last_peered":"2023-06-10T21:24:14.335074+0000","last_clean":"2023-06-10T21:24:14.335074+0000","last_became_active":"2023-06-10T21:24:12.318743+0000","last_became_peered":"2023-06-10T21:24:12.318743+0000","last_unstale":"2023-06-10T21:24:14.335074+0000","last_undegraded":"2023-06-10T21:24:14.335074+0000","last_fullsized":"2023-06-10T21:24:14.335074+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:31:28.116770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":74,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320750+0000","last_change":"2023-06-10T21:23:46.310628+0000","last_active":"2023-06-10T21:24:14.320750+0000","last_peered":"2023-06-10T21:24:14.320750+0000","last_clean":"2023-06-10T21:24:14.320750+0000","last_became_active":"2023-06-10T21:23:46.310211+0000","last_became_peered":"2023-06-10T21:23:46.310211+0000","last_unstale":"2023-06-10T21:24:14.320750+0000","last_undegraded":"2023-06-10T21:24:14.320750+0000","last_fullsized":"2023-06-10T21:24:14.320750+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.331797+0000","last_change":"2023-06-10T21:24:12.315529+0000","last_active":"2023-06-10T21:24:14.331797+0000","last_peered":"2023-06-10T21:24:14.331797+0000","last_clean":"2023-06-10T21:24:14.331797+0000","last_became_active":"2023-06-10T21:24:12.314926+0000","last_became_peered":"2023-06-10T21:24:12.314926+0000","last_unstale":"2023-06-10T21:24:14.331797+0000","last_undegraded":"2023-06-10T21:24:14.331797+0000","last_fullsized":"2023-06-10T21:24:14.331797+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:14.626102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"165'11553","reported_seq":11601,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:16.985040+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:24:16.985040+0000","last_peered":"2023-06-10T21:24:16.985040+0000","last_clean":"2023-06-10T21:24:16.985040+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:24:16.985040+0000","last_undegraded":"2023-06-10T21:24:16.985040+0000","last_fullsized":"2023-06-10T21:24:16.985040+0000","mapping_epoch":127,"log_start":"136'4555","ondisk_log_start":"136'4555","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134807552,"num_objects":2057,"num_object_clones":0,"num_object_copies":6171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11553,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":136,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.320813+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:24:14.320813+0000","last_peered":"2023-06-10T21:24:14.320813+0000","last_clean":"2023-06-10T21:24:14.320813+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:24:14.320813+0000","last_undegraded":"2023-06-10T21:24:14.320813+0000","last_fullsized":"2023-06-10T21:24:14.320813+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":30,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315808+0000","last_change":"2023-06-10T21:24:12.362278+0000","last_active":"2023-06-10T21:24:14.315808+0000","last_peered":"2023-06-10T21:24:14.315808+0000","last_clean":"2023-06-10T21:24:14.315808+0000","last_became_active":"2023-06-10T21:24:12.314820+0000","last_became_peered":"2023-06-10T21:24:12.314820+0000","last_unstale":"2023-06-10T21:24:14.315808+0000","last_undegraded":"2023-06-10T21:24:14.315808+0000","last_fullsized":"2023-06-10T21:24:14.315808+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":116,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315269+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:24:14.315269+0000","last_peered":"2023-06-10T21:24:14.315269+0000","last_clean":"2023-06-10T21:24:14.315269+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:24:14.315269+0000","last_undegraded":"2023-06-10T21:24:14.315269+0000","last_fullsized":"2023-06-10T21:24:14.315269+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":24,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315649+0000","last_change":"2023-06-10T21:24:12.317141+0000","last_active":"2023-06-10T21:24:14.315649+0000","last_peered":"2023-06-10T21:24:14.315649+0000","last_clean":"2023-06-10T21:24:14.315649+0000","last_became_active":"2023-06-10T21:24:12.315654+0000","last_became_peered":"2023-06-10T21:24:12.315654+0000","last_unstale":"2023-06-10T21:24:14.315649+0000","last_undegraded":"2023-06-10T21:24:14.315649+0000","last_fullsized":"2023-06-10T21:24:14.315649+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":138,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.338687+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:24:14.338687+0000","last_peered":"2023-06-10T21:24:14.338687+0000","last_clean":"2023-06-10T21:24:14.338687+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:24:14.338687+0000","last_undegraded":"2023-06-10T21:24:14.338687+0000","last_fullsized":"2023-06-10T21:24:14.338687+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":134,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.334999+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:24:14.334999+0000","last_peered":"2023-06-10T21:24:14.334999+0000","last_clean":"2023-06-10T21:24:14.334999+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:24:14.334999+0000","last_undegraded":"2023-06-10T21:24:14.334999+0000","last_fullsized":"2023-06-10T21:24:14.334999+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":146,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315669+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:24:14.315669+0000","last_peered":"2023-06-10T21:24:14.315669+0000","last_clean":"2023-06-10T21:24:14.315669+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:24:14.315669+0000","last_undegraded":"2023-06-10T21:24:14.315669+0000","last_fullsized":"2023-06-10T21:24:14.315669+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":119,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335141+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:24:14.335141+0000","last_peered":"2023-06-10T21:24:14.335141+0000","last_clean":"2023-06-10T21:24:14.335141+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:24:14.335141+0000","last_undegraded":"2023-06-10T21:24:14.335141+0000","last_fullsized":"2023-06-10T21:24:14.335141+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":155,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315433+0000","last_change":"2023-06-10T21:24:12.316819+0000","last_active":"2023-06-10T21:24:14.315433+0000","last_peered":"2023-06-10T21:24:14.315433+0000","last_clean":"2023-06-10T21:24:14.315433+0000","last_became_active":"2023-06-10T21:24:12.316438+0000","last_became_peered":"2023-06-10T21:24:12.316438+0000","last_unstale":"2023-06-10T21:24:14.315433+0000","last_undegraded":"2023-06-10T21:24:14.315433+0000","last_fullsized":"2023-06-10T21:24:14.315433+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.332300+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:24:14.332300+0000","last_peered":"2023-06-10T21:24:14.332300+0000","last_clean":"2023-06-10T21:24:14.332300+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:24:14.332300+0000","last_undegraded":"2023-06-10T21:24:14.332300+0000","last_fullsized":"2023-06-10T21:24:14.332300+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":164,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-06-10T21:24:15.353931+0000","last_change":"2023-06-10T21:24:12.312559+0000","last_active":"2023-06-10T21:24:15.353931+0000","last_peered":"2023-06-10T21:24:15.353931+0000","last_clean":"2023-06-10T21:24:15.353931+0000","last_became_active":"2023-06-10T21:24:12.312135+0000","last_became_peered":"2023-06-10T21:24:12.312135+0000","last_unstale":"2023-06-10T21:24:15.353931+0000","last_undegraded":"2023-06-10T21:24:15.353931+0000","last_fullsized":"2023-06-10T21:24:15.353931+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:52.126088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":41,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.335303+0000","last_change":"2023-06-10T21:24:12.326721+0000","last_active":"2023-06-10T21:24:14.335303+0000","last_peered":"2023-06-10T21:24:14.335303+0000","last_clean":"2023-06-10T21:24:14.335303+0000","last_became_active":"2023-06-10T21:24:12.314800+0000","last_became_peered":"2023-06-10T21:24:12.314800+0000","last_unstale":"2023-06-10T21:24:14.335303+0000","last_undegraded":"2023-06-10T21:24:14.335303+0000","last_fullsized":"2023-06-10T21:24:14.335303+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":162,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":164,"state":"active+clean","last_fresh":"2023-06-10T21:24:14.315251+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:24:14.315251+0000","last_peered":"2023-06-10T21:24:14.315251+0000","last_clean":"2023-06-10T21:24:14.315251+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:24:14.315251+0000","last_undegraded":"2023-06-10T21:24:14.315251+0000","last_fullsized":"2023-06-10T21:24:14.315251+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":576,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2043478040,"num_objects":31182,"num_object_clones":0,"num_object_copies":93546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6242,"num_objects_unfound":0,"num_objects_dirty":31182,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186790,"num_write_kb":6975041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1554,"num_bytes_recovered":101646336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6191001600,"data_stored":6190989384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127896,"ondisk_log_size":127896,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430527,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574160,"kb_used_data":1165704,"kb_used_omap":0,"kb_used_meta":408448,"kb_avail":92170992,"statfs":{"total":95995035648,"available":94383095808,"internally_reserved":0,"allocated":1193680896,"data_stored":1193172543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":418250752},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:14 2023","interfaces":[{"interface":"back","average":{"1min":4.2190000000000003,"5min":2.5499999999999998,"15min":1.845},"min":{"1min":0.29699999999999999,"5min":0.218,"15min":0.218},"max":{"1min":13.566000000000001,"5min":53.706000000000003,"15min":53.706000000000003},"last":1.3180000000000001},{"interface":"front","average":{"1min":3.6240000000000001,"5min":2.3439999999999999,"15min":2.1429999999999998},"min":{"1min":0.253,"5min":0.252,"15min":0.245},"max":{"1min":10.102,"5min":53.777000000000001,"15min":53.777000000000001},"last":3.294}]},{"osd":1,"last update":"Sat Jun 10 21:23:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":3.9239999999999999,"15min":2.4409999999999998},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.176,"5min":130.19399999999999,"15min":130.19399999999999},"last":2.1659999999999999},{"interface":"front","average":{"1min":2.327,"5min":6.04,"15min":3.0990000000000002},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":8.1769999999999996,"5min":212.20599999999999,"15min":212.20599999999999},"last":1.9279999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":1.73,"15min":0.92100000000000004},"min":{"1min":0.32500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":6.8730000000000002,"5min":9.1609999999999996,"15min":9.1609999999999996},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.3980000000000001,"5min":3.8660000000000001,"15min":1.6439999999999999},"min":{"1min":0.32800000000000001,"5min":0.309,"15min":0.309},"max":{"1min":12.301,"5min":212.13399999999999,"15min":212.13399999999999},"last":0.69399999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:23:34 2023","interfaces":[{"interface":"back","average":{"1min":6.4340000000000002,"5min":3.5990000000000002,"15min":1.5740000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":63.406999999999996,"5min":63.406999999999996,"15min":63.406999999999996},"last":2.4399999999999999},{"interface":"front","average":{"1min":6.782,"5min":3.6819999999999999,"15min":1.6180000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":63.549999999999997,"5min":63.549999999999997,"15min":63.549999999999997},"last":0.625}]},{"osd":4,"last update":"Sat Jun 10 21:23:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.79800000000000004,"15min":1.913},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":7.0010000000000003,"5min":7.0010000000000003,"15min":219.17699999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.3069999999999999,"5min":1.198,"15min":2.206},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":6.3339999999999996,"5min":9.2129999999999992,"15min":219.13999999999999},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:23:34 2023","interfaces":[{"interface":"back","average":{"1min":12.843,"5min":3.3050000000000002,"15min":3.2959999999999998},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":211.99799999999999,"5min":211.99799999999999,"15min":311.202},"last":0.67000000000000004},{"interface":"front","average":{"1min":12.015000000000001,"5min":3.0219999999999998,"15min":3.1190000000000002},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.29699999999999999},"max":{"1min":212.23699999999999,"5min":212.23699999999999,"15min":311.875},"last":4.1420000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:23:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4359999999999999,"5min":1.129,"15min":1.1220000000000001},"min":{"1min":0.39000000000000001,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":10.159000000000001,"5min":10.159000000000001,"15min":10.159000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.6459999999999999,"5min":1.161,"15min":1.0700000000000001},"min":{"1min":0.39100000000000001,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":7.9980000000000002,"5min":7.9980000000000002,"15min":7.9980000000000002},"last":0.46999999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626755,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":686976,"kb_used_data":371384,"kb_used_omap":0,"kb_used_meta":315584,"kb_avail":93058176,"statfs":{"total":95995035648,"available":95291572224,"internally_reserved":0,"allocated":380297216,"data_stored":379805316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":323158016},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":2.0830000000000002,"15min":2.359},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":81.057000000000002,"5min":81.057000000000002,"15min":119.276},"last":5.0410000000000004},{"interface":"front","average":{"1min":4.96,"5min":2.0179999999999998,"15min":2.71},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":80.534999999999997,"5min":80.534999999999997,"15min":119.22},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:23:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.4890000000000001,"15min":1.796},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.1180000000000003,"5min":8.8840000000000003,"15min":14.897},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.641,"15min":1.893},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5899999999999999,"5min":8.0299999999999994,"15min":10.766999999999999},"last":0.498}]},{"osd":2,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":3.016,"5min":1.694,"15min":0.93899999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.3369999999999997,"5min":10.125999999999999,"15min":10.125999999999999},"last":2.4500000000000002},{"interface":"front","average":{"1min":2.9220000000000002,"5min":1.5900000000000001,"15min":0.88800000000000001},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.699999999999999,"5min":13.699999999999999,"15min":13.699999999999999},"last":2.468}]},{"osd":3,"last update":"Sat Jun 10 21:23:42 2023","interfaces":[{"interface":"back","average":{"1min":3.1160000000000001,"5min":2.2709999999999999,"15min":1.1220000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.586,"5min":53.591000000000001,"15min":53.591000000000001},"last":9.9209999999999994},{"interface":"front","average":{"1min":3.6389999999999998,"5min":2.4870000000000001,"15min":1.2090000000000001},"min":{"1min":0.33300000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":15.744999999999999,"5min":53.616,"15min":53.616},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:23:27 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":0.92000000000000004,"15min":1.8260000000000001},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":5.633,"5min":5.633,"15min":221.76900000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.669,"5min":0.98099999999999998,"15min":2.0139999999999998},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.27300000000000002},"max":{"1min":6.7110000000000003,"5min":10.221,"15min":221.905},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:23:27 2023","interfaces":[{"interface":"back","average":{"1min":15.746,"5min":4.1040000000000001,"15min":3.3149999999999999},"min":{"1min":0.32100000000000001,"5min":0.316,"15min":0.25700000000000001},"max":{"1min":198.154,"5min":198.154,"15min":313.661},"last":0.40699999999999997},{"interface":"front","average":{"1min":16.169,"5min":4.0209999999999999,"15min":3.3919999999999999},"min":{"1min":0.36599999999999999,"5min":0.33500000000000002,"15min":0.29499999999999998},"max":{"1min":198.08199999999999,"5min":198.08199999999999,"15min":314.38900000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:23:42 2023","interfaces":[{"interface":"back","average":{"1min":6.5220000000000002,"5min":2.2770000000000001,"15min":1.532},"min":{"1min":0.34000000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":100.55500000000001,"5min":100.55500000000001,"15min":100.55500000000001},"last":0.38},{"interface":"front","average":{"1min":6.859,"5min":2.282,"15min":1.575},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.308},"max":{"1min":100.66,"5min":100.66,"15min":100.66},"last":0.53300000000000003}]}]},{"osd":1,"up_from":105,"seq":450971566198,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1451788,"kb_used_data":1094724,"kb_used_omap":4,"kb_used_meta":357051,"kb_avail":92293364,"statfs":{"total":95995035648,"available":94508404736,"internally_reserved":0,"allocated":1120997376,"data_stored":1120376332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":365621134},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":12.198,"5min":4.4790000000000001,"15min":2.8239999999999998},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":80.486000000000004,"5min":86.009,"15min":86.009},"last":0.20499999999999999},{"interface":"front","average":{"1min":13.145,"5min":5.0039999999999996,"15min":3.3889999999999998},"min":{"1min":0.40200000000000002,"5min":0.27800000000000002,"15min":0.22600000000000001},"max":{"1min":80.519999999999996,"5min":85.869,"15min":85.869},"last":7.4809999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:24:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":2.29,"15min":1.0940000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":18.152000000000001,"5min":18.152000000000001,"15min":18.152000000000001},"last":7.4219999999999997},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.4139999999999999,"15min":0.78400000000000003},"min":{"1min":0.222,"5min":0.218,"15min":0.218},"max":{"1min":8.984,"5min":8.984,"15min":8.984},"last":12.081}]},{"osd":3,"last update":"Sat Jun 10 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":4.0289999999999999,"5min":3.6589999999999998,"15min":1.4590000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":14.544,"5min":118.348,"15min":118.348},"last":0.36099999999999999},{"interface":"front","average":{"1min":4.4340000000000002,"5min":4.2460000000000004,"15min":1.6919999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":18.195,"5min":117.75,"15min":117.75},"last":7.4539999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5800000000000001,"5min":1.647,"15min":1.9059999999999999},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.507999999999999,"5min":12.507999999999999,"15min":12.94},"last":7.9880000000000004},{"interface":"front","average":{"1min":3.1280000000000001,"5min":1.49,"15min":2.3479999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.460000000000001,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.443}]},{"osd":5,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":4.6760000000000002,"5min":2.0960000000000001,"15min":2.2400000000000002},"min":{"1min":0.435,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":14.811999999999999,"5min":14.811999999999999,"15min":14.811999999999999},"last":6.7430000000000003},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.76,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5199999999999996,"5min":8.5199999999999996,"15min":13.602},"last":5.8339999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":4.3460000000000001,"5min":1.9399999999999999,"15min":2.2120000000000002},"min":{"1min":0.46899999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":14.858000000000001,"5min":14.858000000000001,"15min":14.858000000000001},"last":6.7030000000000003},{"interface":"front","average":{"1min":2.8719999999999999,"5min":1.5449999999999999,"15min":2.2730000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.5510000000000002,"5min":8.5510000000000002,"15min":12.970000000000001},"last":5.8170000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":1.6910000000000001,"15min":2.7890000000000001},"min":{"1min":0.45000000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.4480000000000004,"5min":8.4480000000000004,"15min":13.025},"last":5.7240000000000002},{"interface":"front","average":{"1min":3.609,"5min":1.621,"15min":2.0790000000000002},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":14.481,"5min":14.481,"15min":19.286999999999999},"last":0.47299999999999998}]}]},{"osd":0,"up_from":0,"seq":369367187620,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075586,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":610524,"kb_used_data":409556,"kb_used_omap":7,"kb_used_meta":200952,"kb_avail":93134628,"statfs":{"total":95995035648,"available":95369859072,"internally_reserved":0,"allocated":419385344,"data_stored":418807013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":205774929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":2.1579999999999999,"15min":0.97099999999999997},"min":{"1min":0.27900000000000003,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":25.452999999999999,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.246},{"interface":"front","average":{"1min":3.879,"5min":2.633,"15min":1.121},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.053000000000001,"5min":62.451999999999998,"15min":62.451999999999998},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":4.282,"5min":1.9710000000000001,"15min":0.91400000000000003},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.718,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.929,"15min":0.91800000000000004},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":16.641999999999999,"5min":28.013999999999999,"15min":28.013999999999999},"last":0.51500000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:23:31 2023","interfaces":[{"interface":"back","average":{"1min":7.907,"5min":3.6499999999999999,"15min":1.482},"min":{"1min":0.29599999999999999,"5min":0.247,"15min":0.247},"max":{"1min":57.802,"5min":96.295000000000002,"15min":96.295000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":8.3640000000000008,"5min":4.2210000000000001,"15min":1.6910000000000001},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.252},"max":{"1min":60.436999999999998,"5min":99.504999999999995,"15min":99.504999999999995},"last":0.26100000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.4610000000000001,"15min":0.92400000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.8510000000000009,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.86199999999999999},{"interface":"front","average":{"1min":3.375,"5min":1.587,"15min":0.97299999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":17.295000000000002,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.623,"15min":0.998},"min":{"1min":0.38200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.7880000000000003,"5min":10.254,"15min":10.254},"last":3.4790000000000001},{"interface":"front","average":{"1min":2.8900000000000001,"5min":1.7390000000000001,"15min":1.028},"min":{"1min":0.40000000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.5449999999999999,"5min":9.5449999999999999,"15min":9.5449999999999999},"last":2.274}]},{"osd":6,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.597,"5min":1.6459999999999999,"15min":1.0149999999999999},"min":{"1min":0.436,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4400000000000004,"5min":8.048,"15min":8.048},"last":9.4949999999999992},{"interface":"front","average":{"1min":3.8900000000000001,"5min":1.857,"15min":1.099},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":17.244,"5min":17.244,"15min":17.244},"last":5.2569999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":1.3080000000000001,"15min":0.92600000000000005},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.8350000000000009,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":2.1509999999999998},{"interface":"front","average":{"1min":2.952,"5min":1.6879999999999999,"15min":1.0469999999999999},"min":{"1min":0.47599999999999998,"5min":0.434,"15min":0.434},"max":{"1min":8.8140000000000001,"5min":8.8140000000000001,"15min":8.8140000000000001},"last":2.133}]}]},{"osd":3,"up_from":125,"seq":536870912060,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1272732,"kb_used_data":1072340,"kb_used_omap":2,"kb_used_meta":200381,"kb_avail":92472420,"statfs":{"total":95995035648,"available":94691758080,"internally_reserved":0,"allocated":1098076160,"data_stored":1097498080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":205190197},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6050000000000004,"5min":1.7869999999999999,"15min":0.83699999999999997},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":46.677999999999997,"5min":46.677999999999997,"15min":46.677999999999997},"last":2.8530000000000002},{"interface":"front","average":{"1min":4.9100000000000001,"5min":1.927,"15min":0.878},"min":{"1min":0.28699999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":44.301000000000002,"5min":44.301000000000002,"15min":44.301000000000002},"last":0.59799999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":4.7080000000000002,"5min":2.2989999999999999,"15min":1.0349999999999999},"min":{"1min":0.27600000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":30.161999999999999,"5min":54.112000000000002,"15min":54.112000000000002},"last":2.4289999999999998},{"interface":"front","average":{"1min":5.2350000000000003,"5min":2.6030000000000002,"15min":1.1439999999999999},"min":{"1min":0.29499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":34.216000000000001,"5min":57.814,"15min":57.814},"last":2.4500000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":2.6240000000000001,"5min":1.4079999999999999,"15min":0.72899999999999998},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":10.013,"5min":12.726000000000001,"15min":12.726000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.345,"15min":0.70999999999999996},"min":{"1min":0.30299999999999999,"5min":0.249,"15min":0.249},"max":{"1min":7.577,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.4019999999999999,"15min":0.84999999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.9770000000000003,"5min":10.92,"15min":10.92},"last":0.69199999999999995},{"interface":"front","average":{"1min":2.4900000000000002,"5min":1.4139999999999999,"15min":0.85399999999999998},"min":{"1min":0.40100000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.1809999999999992,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":0.63200000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":10.432,"5min":3.0289999999999999,"15min":1.407},"min":{"1min":0.34499999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":170.119,"5min":170.119,"15min":170.119},"last":0.64700000000000002},{"interface":"front","average":{"1min":11.305,"5min":3.302,"15min":1.5069999999999999},"min":{"1min":0.36599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":170.18799999999999,"5min":170.18799999999999,"15min":170.18799999999999},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":3.246,"5min":1.7190000000000001,"15min":0.96699999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.9619999999999997,"5min":9.9619999999999997,"15min":9.9619999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":3.544,"5min":1.6279999999999999,"15min":0.98599999999999999},"min":{"1min":0.47799999999999998,"5min":0.313,"15min":0.313},"max":{"1min":12.835000000000001,"5min":12.835000000000001,"15min":12.835000000000001},"last":0.80800000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:23:39 2023","interfaces":[{"interface":"back","average":{"1min":11.159000000000001,"5min":3.2890000000000001,"15min":1.504},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":174.44800000000001,"5min":174.44800000000001,"15min":174.44800000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":11.598000000000001,"5min":3.27,"15min":1.538},"min":{"1min":0.504,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":173.18100000000001,"5min":173.18100000000001,"15min":173.18100000000001},"last":0.75900000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019211,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750784,"kb_used_data":381816,"kb_used_omap":0,"kb_used_meta":368960,"kb_avail":92994368,"statfs":{"total":95995035648,"available":95226232832,"internally_reserved":0,"allocated":390979584,"data_stored":390486662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":377815040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":2.0310000000000001,"15min":1.978},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.22},"max":{"1min":6.4880000000000004,"5min":40.186999999999998,"15min":40.186999999999998},"last":7.1639999999999997},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.351,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23100000000000001},"max":{"1min":9.8170000000000002,"5min":39.395000000000003,"15min":39.395000000000003},"last":2.5840000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:24:03 2023","interfaces":[{"interface":"back","average":{"1min":6.1970000000000001,"5min":3.5369999999999999,"15min":2.4609999999999999},"min":{"1min":0.33900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":45.857999999999997,"5min":79.731999999999999,"15min":79.731999999999999},"last":3.8290000000000002},{"interface":"front","average":{"1min":5.6150000000000002,"5min":3.0880000000000001,"15min":2.6539999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":45.822000000000003,"5min":79.146000000000001,"15min":79.146000000000001},"last":7.2279999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":1.147,"15min":0.73299999999999998},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.9610000000000003,"5min":9.9610000000000003,"15min":9.9610000000000003},"last":7.25},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.417,"15min":0.89100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.0299999999999994,"5min":11.099,"15min":11.099},"last":2.4700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":7.0700000000000003,"5min":4.2640000000000002,"15min":1.792},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":53.776000000000003,"5min":92.936000000000007,"15min":92.936000000000007},"last":6.593},{"interface":"front","average":{"1min":6.3680000000000003,"5min":3.8119999999999998,"15min":1.6839999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":53.851999999999997,"5min":92.912000000000006,"15min":92.912000000000006},"last":7.2770000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.4350000000000001,"15min":1.4430000000000001},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":6.3650000000000002,"5min":10.984999999999999,"15min":92.653000000000006},"last":5.6020000000000003},{"interface":"front","average":{"1min":1.768,"5min":1.2909999999999999,"15min":1.542},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":6.6239999999999997,"5min":8.3659999999999997,"15min":93.281999999999996},"last":7.2030000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:23:33 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.2430000000000001,"15min":1.1339999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":6.9269999999999996,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":2.98},{"interface":"front","average":{"1min":2.097,"5min":1.1910000000000001,"15min":1.073},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.30399999999999999},"max":{"1min":11.055,"5min":11.055,"15min":11.055},"last":2.9470000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:23:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.4590000000000001,"15min":1.393},"min":{"1min":0.34100000000000003,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":11.077,"5min":11.077,"15min":11.077},"last":2.4049999999999998},{"interface":"front","average":{"1min":2.9199999999999999,"5min":1.46,"15min":1.228},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.312},"max":{"1min":9.5060000000000002,"5min":9.5060000000000002,"15min":9.5060000000000002},"last":2.7080000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822983,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1611060,"kb_used_data":1119660,"kb_used_omap":0,"kb_used_meta":491264,"kb_avail":92134092,"statfs":{"total":95995035648,"available":94345310208,"internally_reserved":0,"allocated":1146531840,"data_stored":1146046992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":503054336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":10.728,"5min":3.8780000000000001,"15min":2.1360000000000001},"min":{"1min":0.28399999999999997,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":148.447,"5min":148.447,"15min":148.447},"last":0.70899999999999996},{"interface":"front","average":{"1min":10.789999999999999,"5min":3.827,"15min":2.7829999999999999},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":148.42099999999999,"5min":148.42099999999999,"15min":148.42099999999999},"last":4.0629999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:24:00 2023","interfaces":[{"interface":"back","average":{"1min":22.015999999999998,"5min":6.3099999999999996,"15min":3.5430000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24299999999999999},"max":{"1min":149.786,"5min":149.786,"15min":149.786},"last":3.5960000000000001},{"interface":"front","average":{"1min":22.021000000000001,"5min":5.9699999999999998,"15min":3.8500000000000001},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":149.52500000000001,"5min":149.52500000000001,"15min":149.52500000000001},"last":4.4909999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.2569999999999999,"15min":0.82299999999999995},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.465999999999999,"5min":11.465999999999999,"15min":11.465999999999999},"last":4.4610000000000003},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.4730000000000001,"15min":0.89500000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.7850000000000001,"5min":10.895,"15min":10.895},"last":4.0339999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:23:32 2023","interfaces":[{"interface":"back","average":{"1min":14.323,"5min":5.6699999999999999,"15min":2.2999999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":179.10300000000001,"5min":179.10300000000001,"15min":179.10300000000001},"last":9.0830000000000002},{"interface":"front","average":{"1min":14.298999999999999,"5min":5.423,"15min":2.165},"min":{"1min":0.39700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":181.44300000000001,"5min":181.44300000000001,"15min":181.44300000000001},"last":10.195}]},{"osd":4,"last update":"Sat Jun 10 21:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1100000000000001,"15min":1.0249999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":3.2229999999999999,"5min":11.039999999999999,"15min":11.039999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80700000000000005,"15min":1.117},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":3.0739999999999998,"5min":5.016,"15min":12.901},"last":1.1799999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:24:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.381,"15min":1.198},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8759999999999994,"5min":10.991,"15min":10.991},"last":4.9109999999999996},{"interface":"front","average":{"1min":1.089,"5min":1.141,"15min":1.198},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3540000000000001,"5min":8.0429999999999993,"15min":8.0429999999999993},"last":4.9169999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.163,"15min":1.325},"min":{"1min":0.33900000000000002,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":1.3029999999999999},{"interface":"front","average":{"1min":3.4409999999999998,"5min":1.5629999999999999,"15min":1.325},"min":{"1min":0.38700000000000001,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":13.047000000000001,"5min":13.047000000000001,"15min":13.047000000000001},"last":0.626}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":515899392,"data_stored":515899392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1109135360,"data_stored":1109131288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":418840576,"data_stored":418840576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1085734912,"data_stored":1085734912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":382533632,"data_stored":382533632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1126436864,"data_stored":1126432792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":377749504,"data_stored":377749504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1174671360,"data_stored":1174667288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:24:25.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_4 pgp_num 16 2023-06-10T21:24:25.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:25.799 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:25.915 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:25.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:26.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:26.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:26.499 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:26.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:26.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:26 smithi064 ceph-mon[186706]: failed to encode map e166 with expected crc 2023-06-10T21:24:26.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:26 smithi064 ceph-mon[180297]: failed to encode map e166 with expected crc 2023-06-10T21:24:26.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:26.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:26 smithi152 ceph-mon[156819]: failed to encode map e166 with expected crc 2023-06-10T21:24:26.653 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:26.748 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:26.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:26.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:26.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:27.174 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:27.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:27.213 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:27.218 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:24:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:24:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:24:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:24:27.219 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:24:27.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:27.348 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:27.351 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:27.384 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:27.421 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:27.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:27 smithi064 ceph-mon[186706]: pgmap v658: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 3965/84273 objects misplaced (4.705%); 19 MiB/s, 309 objects/s recovering 2023-06-10T21:24:27.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:27 smithi064 ceph-mon[180297]: pgmap v658: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 3965/84273 objects misplaced (4.705%); 19 MiB/s, 309 objects/s recovering 2023-06-10T21:24:27.457 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:27.468 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:27.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:27.525 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:27.553 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:27.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:27.587 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:27.639 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:27.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.653 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:24:27.669 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:27 smithi152 ceph-mon[156819]: pgmap v658: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 3965/84273 objects misplaced (4.705%); 19 MiB/s, 309 objects/s recovering 2023-06-10T21:24:27.669 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:27.770 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:27.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:27.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:27.958 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:28.078 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:28.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:28.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:28.266 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:28.277 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4285668871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:28.278 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:24:28.334 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:28.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:28.440 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:28.479 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:28.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4285668871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:28.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:24:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4285668871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:24:28.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:24:28.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:24:29.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:29.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:29.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:29.177 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:29.268 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:29.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[180297]: pgmap v659: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 3393/81786 objects misplaced (4.149%); 21 MiB/s, 329 objects/s recovering 2023-06-10T21:24:29.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:29.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:29.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:29.420 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:29 smithi152 ceph-mon[156819]: pgmap v659: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 3393/81786 objects misplaced (4.149%); 21 MiB/s, 329 objects/s recovering 2023-06-10T21:24:29.421 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.421 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:29.525 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:29.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:29.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:29.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[186706]: pgmap v659: 89 pgs: 4 active+remapped+backfilling, 85 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 3393/81786 objects misplaced (4.149%); 21 MiB/s, 329 objects/s recovering 2023-06-10T21:24:29.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:29.693 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:30.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:30.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:30.374 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:30.376 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:30.424 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:30.437 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:30.503 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:30.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:30.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:30.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[186706]: osdmap e167: 8 total, 8 up, 7 in 2023-06-10T21:24:30.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[186706]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[186706]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[180297]: osdmap e167: 8 total, 8 up, 7 in 2023-06-10T21:24:30.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[180297]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:30 smithi064 ceph-mon[180297]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:30 smithi152 ceph-mon[156819]: osdmap e167: 8 total, 8 up, 7 in 2023-06-10T21:24:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:30 smithi152 ceph-mon[156819]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:30 smithi152 ceph-mon[156819]: failed to encode map e167 with expected crc 2023-06-10T21:24:30.779 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:30.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:30.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:30.912 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:31.012 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:31.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:31.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:31.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:31.147 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:31.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:24:31.195 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:31.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:31.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:31.342 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:31.576 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:31.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:31 smithi152 ceph-mon[156819]: pgmap v661: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 2075/77925 objects misplaced (2.663%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:24:31.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:31 smithi152 ceph-mon[156819]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:31 smithi152 ceph-mon[156819]: osdmap e168: 8 total, 8 up, 7 in 2023-06-10T21:24:31.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:31 smithi152 ceph-mon[156819]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:31 smithi152 ceph-mon[156819]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:31.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[186706]: pgmap v661: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 2075/77925 objects misplaced (2.663%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:24:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[186706]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[186706]: osdmap e168: 8 total, 8 up, 7 in 2023-06-10T21:24:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[186706]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[186706]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[180297]: pgmap v661: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 2075/77925 objects misplaced (2.663%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:24:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[180297]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[180297]: osdmap e168: 8 total, 8 up, 7 in 2023-06-10T21:24:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[180297]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:31 smithi064 ceph-mon[180297]: failed to encode map e168 with expected crc 2023-06-10T21:24:31.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:31.761 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:31.884 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:31.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:31.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:32.075 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:32.308 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:32.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:32.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:32.495 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:32.613 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:32.641 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.641 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: osdmap e169: 8 total, 8 up, 7 in 2023-06-10T21:24:32.641 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.642 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.642 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.642 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:32 smithi152 ceph-mon[156819]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: osdmap e169: 8 total, 8 up, 7 in 2023-06-10T21:24:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[186706]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: osdmap e169: 8 total, 8 up, 7 in 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:32 smithi064 ceph-mon[180297]: failed to encode map e169 with expected crc 2023-06-10T21:24:32.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:32.778 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:32.984 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:32.985 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:33.009 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:33.013 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:32] "GET /metrics HTTP/1.1" 200 223629 "" "Prometheus/2.33.4" 2023-06-10T21:24:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:33.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:33.121 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:33.214 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:33.314 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:33.323 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 9 pgp_num to 16 2023-06-10T21:24:33.323 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: pgmap v664: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 1838/75360 objects misplaced (2.439%); 19 MiB/s, 303 objects/s recovering 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/311380941' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:33.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:33.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:33.497 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:33.634 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: pgmap v664: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 1838/75360 objects misplaced (2.439%); 19 MiB/s, 303 objects/s recovering 2023-06-10T21:24:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/311380941' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: pgmap v664: 89 pgs: 1 peering, 3 active+remapped+backfilling, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 1838/75360 objects misplaced (2.439%); 19 MiB/s, 303 objects/s recovering 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/311380941' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:34.565 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:34.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:34.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:34.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[186706]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:24:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[186706]: osdmap e170: 8 total, 8 up, 7 in 2023-06-10T21:24:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[186706]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[180297]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:24:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[180297]: osdmap e170: 8 total, 8 up, 7 in 2023-06-10T21:24:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:34 smithi064 ceph-mon[180297]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:34 smithi152 ceph-mon[156819]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:24:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:34 smithi152 ceph-mon[156819]: osdmap e170: 8 total, 8 up, 7 in 2023-06-10T21:24:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:34 smithi152 ceph-mon[156819]: failed to encode map e170 with expected crc 2023-06-10T21:24:34.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:34.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:24:35.015 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:35.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:35.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:35.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:35.461 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:35 smithi064 ceph-mon[186706]: pgmap v666: 89 pgs: 1 peering, 2 active+remapped+backfilling, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 1418/73857 objects misplaced (1.920%); 25 MiB/s, 405 objects/s recovering 2023-06-10T21:24:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:35 smithi064 ceph-mon[180297]: pgmap v666: 89 pgs: 1 peering, 2 active+remapped+backfilling, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 1418/73857 objects misplaced (1.920%); 25 MiB/s, 405 objects/s recovering 2023-06-10T21:24:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:35 smithi152 ceph-mon[156819]: pgmap v666: 89 pgs: 1 peering, 2 active+remapped+backfilling, 86 active+clean; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 1418/73857 objects misplaced (1.920%); 25 MiB/s, 405 objects/s recovering 2023-06-10T21:24:36.049 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:36.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:36.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:36.222 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:36.325 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:36.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:36.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:36.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:36.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:36.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:36 smithi064 ceph-mon[180297]: pgmap v667: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 1009/68499 objects misplaced (1.473%); 8.5 MiB/s, 136 objects/s recovering 2023-06-10T21:24:36.602 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:24:36.639 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:24:36.700 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:36.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:36.806 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:36.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:36.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:36.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:36 smithi064 ceph-mon[186706]: pgmap v667: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 1009/68499 objects misplaced (1.473%); 8.5 MiB/s, 136 objects/s recovering 2023-06-10T21:24:36.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:24:36.869 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:36.899 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:36.926 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:36 smithi152 ceph-mon[156819]: pgmap v667: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 1009/68499 objects misplaced (1.473%); 8.5 MiB/s, 136 objects/s recovering 2023-06-10T21:24:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:24:36.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:36.972 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:36.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:36.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:36.996 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:37.014 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:37.032 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:37.033 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.033 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.033 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.033 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.033 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.034 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.034 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.034 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:24:37.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:37.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:37.622 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:37.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:37.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:37.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:37.783 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:37.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:24:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: osdmap e171: 8 total, 8 up, 7 in 2023-06-10T21:24:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2569035576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:37.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: osdmap e171: 8 total, 8 up, 7 in 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2569035576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:37.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:37.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/110916181' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:24:37.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: osdmap e171: 8 total, 8 up, 7 in 2023-06-10T21:24:37.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: failed to encode map e171 with expected crc 2023-06-10T21:24:37.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2569035576' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:37.951 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:37.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:38.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:38.119 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:38.330 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:38.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:38.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:38.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:38.593 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:38.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:38.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:38 smithi152 ceph-mon[156819]: pgmap v669: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 1009/66093 objects misplaced (1.527%); 7.1 MiB/s, 114 objects/s recovering 2023-06-10T21:24:38.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:38.735 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:24:38.736 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:24:38.736 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:24:38.736 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:24:38.747 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:38.847 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:38.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:38 smithi064 ceph-mon[180297]: pgmap v669: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 1009/66093 objects misplaced (1.527%); 7.1 MiB/s, 114 objects/s recovering 2023-06-10T21:24:38.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:38 smithi064 ceph-mon[186706]: pgmap v669: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 1009/66093 objects misplaced (1.527%); 7.1 MiB/s, 114 objects/s recovering 2023-06-10T21:24:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:38.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:38.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:39.028 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:39.103 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:39.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:39.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:39.233 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:39.275 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:39.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:39.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:39.462 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:39.666 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:39.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:39.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:24:39.734 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:24:39.734 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:24:39.734 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-06-10T21:24:39.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 0 2023-06-10T21:24:39.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:39.846 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:39.983 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:40.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:40.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:24:40.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:40.171 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:40.270 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:40.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:40.332 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:24:40.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:40.404 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:40.508 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:40.569 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:40.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:40.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:40.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:40.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:40.830 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:41.028 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:41.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:41.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:41.168 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: pgmap v670: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 1009/62169 objects misplaced (1.623%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:24:41.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:24:41.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:24:41.169 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.173 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: pgmap v670: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 1009/62169 objects misplaced (1.623%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:24:41.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:24:41.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:24:41.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.190 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:41.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:41.292 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:41.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:41.297 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: pgmap v670: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 1009/62169 objects misplaced (1.623%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:24:41.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:24:41.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:24:41.298 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:41.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:41.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:42.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:42.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:42.232 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:42.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:42.337 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:42.341 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:24:42.341 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:24:42.341 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:24:42.342 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:24:42.342 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:24:42.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:42.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:42.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:42.775 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:42.948 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:42.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:42.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:43.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:43.105 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:42] "GET /metrics HTTP/1.1" 200 223663 "" "Prometheus/2.33.4" 2023-06-10T21:24:43.105 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:43 smithi064 ceph-mon[180297]: pgmap v671: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 778/59418 objects misplaced (1.309%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:24:43.107 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:43 smithi152 ceph-mon[156819]: pgmap v671: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 778/59418 objects misplaced (1.309%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:24:43.116 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:43.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:24:43.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:43 smithi064 ceph-mon[186706]: pgmap v671: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 778/59418 objects misplaced (1.309%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:24:43.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:43 smithi064 conmon[180272]: 2023-06-10T21:24:43.248+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e171 definitely_dead 0 2023-06-10T21:24:43.683 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:43.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:43.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:43.890 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:44.035 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:44.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:44.105 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:44.117 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:24:44.143 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.0. 2023-06-10T21:24:44.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:44.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:24:44.200 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:44.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:24:45.045 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:45.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:45.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:45.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:45.150 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-06-10T21:24:45.218 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:45.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:45.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:45.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:45.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:45.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 778/57231 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:24:45.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:24:45.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: osdmap e172: 8 total, 8 up, 8 in 2023-06-10T21:24:45.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:45 smithi152 ceph-mon[156819]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.401 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 778/57231 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:24:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:24:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: osdmap e172: 8 total, 8 up, 8 in 2023-06-10T21:24:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[180297]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: pgmap v672: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 778/57231 objects misplaced (1.359%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:24:45.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/960823128' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: osdmap e172: 8 total, 8 up, 8 in 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:45 smithi064 ceph-mon[186706]: failed to encode map e172 with expected crc 2023-06-10T21:24:45.453 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:45.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:45.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:45.562 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:45.592 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:45.692 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:45.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:45.727 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:45.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:45.764 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:45.782 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:45.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:45.801 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:45.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:45.826 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:45.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:45.857 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:45.858 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:24:45.872 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:45.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:45.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:45.885 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:45.885 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:45.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:46.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:46.097 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:46.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: osdmap e173: 8 total, 8 up, 8 in 2023-06-10T21:24:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2837207323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2099910751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: osdmap e173: 8 total, 8 up, 8 in 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2837207323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2099910751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:46.161 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:24:46.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:46.266 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:46.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:46.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:46.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: osdmap e173: 8 total, 8 up, 8 in 2023-06-10T21:24:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: failed to encode map e173 with expected crc 2023-06-10T21:24:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2837207323' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:24:46.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2099910751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:46.505 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:46.566 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:46.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:24:46.611 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:46.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:46.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:46.805 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:46.984 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:46.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:46.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:47.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:47.157 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:47.220 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:47.357 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:47.360 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: pgmap v675: 89 pgs: 1 remapped+peering, 8 peering, 80 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:24:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:24:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: 9.b starting backfill to osd.0 from (133'2699,166'12153] MIN to 171'12699 2023-06-10T21:24:47.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:47.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: osdmap e174: 8 total, 8 up, 8 in 2023-06-10T21:24:47.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: 9.9 starting backfill to osd.0 from (137'4923,167'11999] MIN to 171'12506 2023-06-10T21:24:47.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: 9.8 starting backfill to osd.0 from (137'4771,167'11843] MIN to 171'12326 2023-06-10T21:24:47.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[186706]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: pgmap v675: 89 pgs: 1 remapped+peering, 8 peering, 80 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: 9.b starting backfill to osd.0 from (133'2699,166'12153] MIN to 171'12699 2023-06-10T21:24:47.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:47.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: osdmap e174: 8 total, 8 up, 8 in 2023-06-10T21:24:47.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: 9.9 starting backfill to osd.0 from (137'4923,167'11999] MIN to 171'12506 2023-06-10T21:24:47.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: 9.8 starting backfill to osd.0 from (137'4771,167'11843] MIN to 171'12326 2023-06-10T21:24:47.364 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:47 smithi064 ceph-mon[180297]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:47.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: pgmap v675: 89 pgs: 1 remapped+peering, 8 peering, 80 active+clean; 1.1 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:24:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:24:47.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:24:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: 9.b starting backfill to osd.0 from (133'2699,166'12153] MIN to 171'12699 2023-06-10T21:24:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:24:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: osdmap e174: 8 total, 8 up, 8 in 2023-06-10T21:24:47.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: 9.9 starting backfill to osd.0 from (137'4923,167'11999] MIN to 171'12506 2023-06-10T21:24:47.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: 9.8 starting backfill to osd.0 from (137'4771,167'11843] MIN to 171'12326 2023-06-10T21:24:47.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:47 smithi152 ceph-mon[156819]: failed to encode map e174 with expected crc 2023-06-10T21:24:47.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:47.567 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:47.682 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:47.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:47.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:47.859 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:47.958 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:47.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:48.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:48.097 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:48.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:48.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:48.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:48.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[180297]: osdmap e175: 8 total, 8 up, 8 in 2023-06-10T21:24:48.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[180297]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[180297]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:48.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:48.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[186706]: osdmap e175: 8 total, 8 up, 8 in 2023-06-10T21:24:48.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[186706]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[186706]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:48.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:48.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:48.368 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:48.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:48.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:48 smithi152 ceph-mon[156819]: osdmap e175: 8 total, 8 up, 8 in 2023-06-10T21:24:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:48 smithi152 ceph-mon[156819]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:48 smithi152 ceph-mon[156819]: failed to encode map e175 with expected crc 2023-06-10T21:24:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:24:48.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:24:48.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:48.514 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:48.562 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:48.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:48.668 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:48.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:48.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:48.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:48.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:48.960 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:49.046 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 8 peering, 78 active+clean; 1.0 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 1903/49854 objects misplaced (3.817%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: osdmap e176: 8 total, 8 up, 8 in 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[180297]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.160 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 8 peering, 78 active+clean; 1.0 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 1903/49854 objects misplaced (3.817%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:24:49.160 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: osdmap e176: 8 total, 8 up, 8 in 2023-06-10T21:24:49.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.161 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:49 smithi152 ceph-mon[156819]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.177 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:49.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:49.195 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:49.195 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:49.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:49.344 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:49.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: pgmap v678: 89 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 remapped+peering, 8 peering, 78 active+clean; 1.0 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 1903/49854 objects misplaced (3.817%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: osdmap e176: 8 total, 8 up, 8 in 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:49 smithi064 ceph-mon[186706]: failed to encode map e176 with expected crc 2023-06-10T21:24:49.429 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:49.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:49.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:24:49.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:24:49.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:49.598 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:49.661 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:49.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:49.770 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:49.830 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:49.957 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:50.000 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:50.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:24:50.109 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:50.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:50.150 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:24:50.151 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:24:50.151 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-06-10T21:24:50.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:24:50.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:50.295 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:50 smithi152 ceph-mon[156819]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:50 smithi152 ceph-mon[156819]: osdmap e177: 8 total, 8 up, 8 in 2023-06-10T21:24:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:50 smithi152 ceph-mon[156819]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:50 smithi152 ceph-mon[156819]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[186706]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[186706]: osdmap e177: 8 total, 8 up, 8 in 2023-06-10T21:24:50.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[186706]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[186706]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[180297]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[180297]: osdmap e177: 8 total, 8 up, 8 in 2023-06-10T21:24:50.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[180297]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:50 smithi064 ceph-mon[180297]: failed to encode map e177 with expected crc 2023-06-10T21:24:50.523 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:50.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:50.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:50.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:50.670 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:50.687 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:50.770 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:50.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:50.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:50.905 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:50.963 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:50.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:51.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:51.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 85 active+clean; 977 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 2849/46956 objects misplaced (6.067%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:24:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: osdmap e178: 8 total, 8 up, 8 in 2023-06-10T21:24:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: osdmap e179: 8 total, 8 up, 8 in 2023-06-10T21:24:51.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:51 smithi152 ceph-mon[156819]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.337 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:51.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 85 active+clean; 977 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 2849/46956 objects misplaced (6.067%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:24:51.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: osdmap e178: 8 total, 8 up, 8 in 2023-06-10T21:24:51.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: osdmap e179: 8 total, 8 up, 8 in 2023-06-10T21:24:51.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[186706]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: pgmap v681: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 1 peering, 85 active+clean; 977 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 2849/46956 objects misplaced (6.067%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:24:51.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: osdmap e178: 8 total, 8 up, 8 in 2023-06-10T21:24:51.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e178 with expected crc 2023-06-10T21:24:51.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: osdmap e179: 8 total, 8 up, 8 in 2023-06-10T21:24:51.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:51 smithi064 ceph-mon[180297]: failed to encode map e179 with expected crc 2023-06-10T21:24:51.443 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:51.471 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:51.512 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:51.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:51.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:51.645 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:51.952 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:51.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:52.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:52.092 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:52.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: failed to encode map e179 with expected crc 2023-06-10T21:24:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: osdmap e180: 8 total, 8 up, 8 in 2023-06-10T21:24:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:52 smithi152 ceph-mon[156819]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.239 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:52.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:52.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:52.375 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:52.395 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:24:52.396 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:24:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: failed to encode map e179 with expected crc 2023-06-10T21:24:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: osdmap e180: 8 total, 8 up, 8 in 2023-06-10T21:24:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[186706]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: failed to encode map e179 with expected crc 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: osdmap e180: 8 total, 8 up, 8 in 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:52 smithi064 ceph-mon[180297]: failed to encode map e180 with expected crc 2023-06-10T21:24:52.422 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:52.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:52.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:52.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:52.557 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:52.602 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:52.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:52.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:52.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:52.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:52.872 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:52.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:52.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:53.053 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:53.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:24:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:24:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:24:52] "GET /metrics HTTP/1.1" 200 223674 "" "Prometheus/2.33.4" 2023-06-10T21:24:53.161 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:53.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:53.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:53.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:53.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: pgmap v684: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 947 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 2849/45513 objects misplaced (6.260%); 144 KiB/s, 1 objects/s recovering 2023-06-10T21:24:53.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: failed to encode map e180 with expected crc 2023-06-10T21:24:53.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: osdmap e181: 8 total, 8 up, 8 in 2023-06-10T21:24:53.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.336 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:53 smithi152 ceph-mon[156819]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.338 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:53.460 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:53.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:53.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:53.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:24:53.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: pgmap v684: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 947 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 2849/45513 objects misplaced (6.260%); 144 KiB/s, 1 objects/s recovering 2023-06-10T21:24:53.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: failed to encode map e180 with expected crc 2023-06-10T21:24:53.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: osdmap e181: 8 total, 8 up, 8 in 2023-06-10T21:24:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[186706]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: pgmap v684: 89 pgs: 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 86 active+clean; 947 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 361 op/s; 2849/45513 objects misplaced (6.260%); 144 KiB/s, 1 objects/s recovering 2023-06-10T21:24:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: failed to encode map e180 with expected crc 2023-06-10T21:24:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: osdmap e181: 8 total, 8 up, 8 in 2023-06-10T21:24:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:53 smithi064 ceph-mon[180297]: failed to encode map e181 with expected crc 2023-06-10T21:24:53.671 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:53.773 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:53.796 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:24:53.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:53.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:24:53.902 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:53.948 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:54.099 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:54.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:54.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:24:54.336 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:54 smithi064 ceph-mon[180297]: failed to encode map e181 with expected crc 2023-06-10T21:24:54.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:54 smithi064 ceph-mon[186706]: failed to encode map e181 with expected crc 2023-06-10T21:24:54.436 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:54.461 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:54 smithi152 ceph-mon[156819]: failed to encode map e181 with expected crc 2023-06-10T21:24:54.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:24:54.540 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:24:54.571 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:24:54.604 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:24:54.641 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:24:54.666 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:24:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:54.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:24:54.682 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:24:54.720 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:24:54.760 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:24:54.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:54.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.784 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.785 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.785 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.785 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.785 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:24:54.831 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:55.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:55.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:55.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:55.228 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:24:55.252 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:55.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:55 smithi152 ceph-mon[156819]: pgmap v687: 89 pgs: 1 peering, 88 active+clean; 892 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9.9 MiB/s, 158 objects/s recovering 2023-06-10T21:24:55.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/956759968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:55.398 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:55.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[180297]: pgmap v687: 89 pgs: 1 peering, 88 active+clean; 892 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9.9 MiB/s, 158 objects/s recovering 2023-06-10T21:24:55.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/956759968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:55.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[186706]: pgmap v687: 89 pgs: 1 peering, 88 active+clean; 892 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 342 op/s; 9.9 MiB/s, 158 objects/s recovering 2023-06-10T21:24:55.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/956759968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:24:55.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:24:55.473 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:55.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:55.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:24:55.623 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:55.695 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109295 9.9 e141/127 9:9b3a09da:::benchmark_data_smithi064_247471_object307:head v 141'6783, mlcod=141'6781)", 2023-06-10T21:24:55.696 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.79023280499999, 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998, 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:24:55.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898798+0000", 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2339999999999997e-06 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898805+0000", 2023-06-10T21:24:55.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5519999999999993e-06 2023-06-10T21:24:55.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898815+0000", 2023-06-10T21:24:55.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2270000000000001e-06 2023-06-10T21:24:55.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898820+0000", 2023-06-10T21:24:55.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4044e-05 2023-06-10T21:24:55.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898864+0000", 2023-06-10T21:24:55.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9573e-05 2023-06-10T21:24:55.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898884+0000", 2023-06-10T21:24:55.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35487087899999997 2023-06-10T21:24:55.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253755+0000", 2023-06-10T21:24:55.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3262000000000001e-05 2023-06-10T21:24:55.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253788+0000", 2023-06-10T21:24:55.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998 2023-06-10T21:24:55.709 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.709 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.709 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.710 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109297 9.5 e141/127 9:a2279825:::benchmark_data_smithi064_247471_object309:head v 141'6825, mlcod=141'6823)", 2023-06-10T21:24:55.711 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:24:55.711 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78981598499999, 2023-06-10T21:24:55.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998, 2023-06-10T21:24:55.712 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.712 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.712 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899215+0000", 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8650000000000003e-06 2023-06-10T21:24:55.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899222+0000", 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899232+0000", 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6350000000000003e-06 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899237+0000", 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0815000000000001e-05 2023-06-10T21:24:55.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899278+0000", 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1216000000000002e-05 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899299+0000", 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.349632361 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248932+0000", 2023-06-10T21:24:55.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100882 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249032+0000", 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.719 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109299 9.7 9:e2e5e039:::benchmark_data_smithi064_247471_object311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78690202999999, 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998, 2023-06-10T21:24:55.720 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109299 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:24:55.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902128+0000", 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902135+0000", 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0420000000000008e-06 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902144+0000", 2023-06-10T21:24:55.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6470000000000003e-06 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902150+0000", 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6499000000000003e-05 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902197+0000", 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011322100000000001 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:24:55.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012462399999999999 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902434+0000", 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046768010000000004 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907111+0000", 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6680000000000004e-06 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907118+0000", 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34198863299999999 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249107+0000", 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102996 2023-06-10T21:24:55.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249210+0000", 2023-06-10T21:24:55.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4842000000000002e-05 2023-06-10T21:24:55.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249254+0000", 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109302 9.a 9:593e78fe:::benchmark_data_smithi064_247471_object314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78672058000001, 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999, 2023-06-10T21:24:55.732 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109302 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.733 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5360000000000003e-06 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902315+0000", 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.669e-06 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902322+0000", 2023-06-10T21:24:55.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9060000000000004e-06 2023-06-10T21:24:55.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902325+0000", 2023-06-10T21:24:55.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074886199999999999 2023-06-10T21:24:55.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903074+0000", 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010393800000000001 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903178+0000", 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8347000000000001e-05 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903247+0000", 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.345750055 2023-06-10T21:24:55.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248997+0000", 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000337282 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249334+0000", 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6821e-05 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249361+0000", 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7002000000000002e-05 2023-06-10T21:24:55.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249408+0000", 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8299e-05 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249446+0000", 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.745 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109305 9.6 9:68c28d89:::benchmark_data_smithi064_247471_object317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78597856499999, 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997, 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.746 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109305 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903052+0000", 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903058+0000", 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0450000000000006e-06 2023-06-10T21:24:55.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903067+0000", 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8790000000000002e-06 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903071+0000", 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8942e-05 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903120+0000", 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100547 2023-06-10T21:24:55.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903221+0000", 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109324 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:24:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903330+0000", 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34556609100000002 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248896+0000", 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042370840000000003 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253133+0000", 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3943000000000006e-05 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253217+0000", 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3278999999999997e-05 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253261+0000", 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3470000000000001e-05 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253314+0000", 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109296 9.9 e141/127 9:9d3096da:::benchmark_data_smithi064_247471_object308:head v 141'6784, mlcod=141'6781)", 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78426828600001, 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001, 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.757 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3180000000000003e-06 2023-06-10T21:24:55.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904769+0000", 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0399999999999993e-06 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904777+0000", 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6299999999999997e-06 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904782+0000", 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115028 2023-06-10T21:24:55.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904897+0000", 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8039000000000001e-05 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904915+0000", 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34887546699999999 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253790+0000", 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5639999999999999e-05 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253806+0000", 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109303 9.b e141/127 9:d16859ff:::benchmark_data_smithi064_247471_object315:head v 141'6884, mlcod=141'6882)", 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78423840599999, 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885, 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.765 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.766 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:24:55.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.456e-06 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904797+0000", 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9980000000000003e-06 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904802+0000", 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0400000000000001e-06 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904805+0000", 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34422613600000002 2023-06-10T21:24:55.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249031+0000", 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7886000000000001e-05 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249089+0000", 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078164589999999996 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256905+0000", 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2179999999999999e-05 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256947+0000", 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885 2023-06-10T21:24:55.773 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109304 9.5 e141/127 9:a38538a9:::benchmark_data_smithi064_247471_object316:head v 141'6826, mlcod=141'6823)", 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:24:55.774 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.784217106, 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202, 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:24:55.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904814+0000", 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8880000000000002e-06 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904820+0000", 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904825+0000", 2023-06-10T21:24:55.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.965e-06 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904828+0000", 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002189528 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907018+0000", 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3083e-05 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907031+0000", 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34204052800000001 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249071+0000", 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7423000000000003e-05 2023-06-10T21:24:55.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249119+0000", 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.782 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109300 9.9 e141/127 9:94ae4f7e:::benchmark_data_smithi064_247471_object312:head v 141'6785, mlcod=141'6781)", 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.781228072, 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001, 2023-06-10T21:24:55.783 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907803+0000", 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.772e-06 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907812+0000", 2023-06-10T21:24:55.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34109469599999998 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248906+0000", 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1025999999999997e-05 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248937+0000", 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016277199999999999 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249100+0000", 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4561999999999997e-05 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249155+0000", 2023-06-10T21:24:55.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083045289999999997 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257459+0000", 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8495000000000002e-05 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257508+0000", 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109307 9.f 9:f866cb96:::benchmark_data_smithi064_247471_object319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.78064186099999, 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092, 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.792 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109307 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908388+0000", 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4800000000000004e-06 2023-06-10T21:24:55.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908396+0000", 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34054057399999998 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248936+0000", 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1486000000000001e-05 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248958+0000", 2023-06-10T21:24:55.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032944700000000002 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249287+0000", 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101111 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249388+0000", 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123366 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249512+0000", 2023-06-10T21:24:55.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077532640000000002 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257265+0000", 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045560400000000002 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257721+0000", 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003373905 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261094+0000", 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6616000000000002e-05 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261141+0000", 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6924000000000001e-05 2023-06-10T21:24:55.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261178+0000", 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.803 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:109299 9.7 e141/127 ondisk, result = 0)", 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 152.77767800399999, 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999, 2023-06-10T21:24:55.804 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911353+0000", 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7229999999999998e-06 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911359+0000", 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33752210999999999 2023-06-10T21:24:55.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248882+0000", 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6460000000000001e-05 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248908+0000", 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015431399999999999 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249062+0000", 2023-06-10T21:24:55.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2586999999999999e-05 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249085+0000", 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017446900000000001 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249259+0000", 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.811 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.812 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137750 9.f 9:fe4ed14c:::benchmark_data_smithi064_247471_object28762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:24:55.812 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:24:55.812 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 113.260518164, 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768, 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.813 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137750 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:24:55.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428513+0000", 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.377e-06 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428516+0000", 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1229999999999997e-06 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428523+0000", 2023-06-10T21:24:55.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8249999999999998e-06 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428526+0000", 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00081994200000000004 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429346+0000", 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2603999999999996e-05 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429439+0000", 2023-06-10T21:24:55.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104776 2023-06-10T21:24:55.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:24:55.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429544+0000", 2023-06-10T21:24:55.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049570079999999997 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.434501+0000", 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018088150000000001 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436310+0000", 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30228024799999997 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738590+0000", 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2488999999999998e-05 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738642+0000", 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8994e-05 2023-06-10T21:24:55.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738681+0000", 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.824 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137758 9.7 9:e64fca9d:::benchmark_data_smithi064_247471_object28770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 113.25265392199999, 2023-06-10T21:24:55.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999, 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137758 2023-06-10T21:24:55.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436377+0000", 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2279999999999998e-06 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436382+0000", 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4789999999999996e-06 2023-06-10T21:24:55.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436390+0000", 2023-06-10T21:24:55.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6550000000000003e-06 2023-06-10T21:24:55.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436395+0000", 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5500000000000001e-05 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436441+0000", 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105098 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436546+0000", 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011236899999999999 2023-06-10T21:24:55.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436658+0000", 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30189442999999999 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738552+0000", 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000176961 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738729+0000", 2023-06-10T21:24:55.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5865999999999997e-05 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738785+0000", 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5071999999999999e-05 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738820+0000", 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1973e-05 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738852+0000", 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137759 9.7 9:e8bc7c80:::benchmark_data_smithi064_247471_object28771:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 113.252625665, 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001, 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.838 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137759 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:24:55.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:24:55.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8289999999999999e-06 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436409+0000", 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5740000000000003e-06 2023-06-10T21:24:55.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436414+0000", 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7810000000000005e-06 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436422+0000", 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004238187 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440660+0000", 2023-06-10T21:24:55.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117327 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440777+0000", 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012919200000000001 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440907+0000", 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.297783668 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738690+0000", 2023-06-10T21:24:55.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8073999999999999e-05 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738758+0000", 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000156365 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738915+0000", 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6863000000000003e-05 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738952+0000", 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8658999999999997e-05 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738990+0000", 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001 2023-06-10T21:24:55.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137770 9.7 9:e3c55b93:::benchmark_data_smithi064_247471_object28782:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:24:55.850 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 113.244026992, 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009, 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.851 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137770 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:24:55.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445004+0000", 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.569e-06 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445008+0000", 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1970000000000001e-06 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445014+0000", 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9210000000000002e-06 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445018+0000", 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28887405700000002 2023-06-10T21:24:55.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.733892+0000", 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146957 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734039+0000", 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012832300000000001 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734167+0000", 2023-06-10T21:24:55.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049619210000000002 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739129+0000", 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047693650000000002 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.743899+0000", 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011874 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744017+0000", 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8343e-05 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744076+0000", 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9201e-05 2023-06-10T21:24:55.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744115+0000", 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.862 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199845 9.f 9:ff71ad38:::benchmark_data_smithi064_247471_object90857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 14.164640606000001, 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001, 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.863 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199845 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524389+0000", 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:24:55.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524398+0000", 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2369e-05 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524410+0000", 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8360000000000003e-06 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524417+0000", 2023-06-10T21:24:55.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084813649999999994 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532898+0000", 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114187 2023-06-10T21:24:55.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533013+0000", 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118496 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533131+0000", 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050821640000000001 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:55.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.538213+0000", 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087860630000000002 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.546999+0000", 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28852924899999999 2023-06-10T21:24:55.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835529+0000", 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4122999999999998e-05 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835573+0000", 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9298999999999998e-05 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835612+0000", 2023-06-10T21:24:55.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201393 9.5 9:a576e4a0:::benchmark_data_smithi064_247471_object92405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:24:55.875 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 11.565981664000001, 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001, 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.876 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201393 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:24:55.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123049+0000", 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7429999999999999e-06 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123055+0000", 2023-06-10T21:24:55.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6000000000000007e-06 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123064+0000", 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9880000000000001e-06 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123070+0000", 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019396100000000001 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123263+0000", 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018801169999999999 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125144+0000", 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000161557 2023-06-10T21:24:55.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125305+0000", 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023599202 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148904+0000", 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6900000000000004e-06 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148914+0000", 2023-06-10T21:24:55.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593960899999999 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124854+0000", 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2537999999999999e-05 2023-06-10T21:24:55.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124866+0000", 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201395 9.5 9:a9ebfe85:::benchmark_data_smithi064_247471_object92407:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 11.565686879999999, 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999, 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.887 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201395 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123344+0000", 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0849999999999996e-06 2023-06-10T21:24:55.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123349+0000", 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.354e-06 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123357+0000", 2023-06-10T21:24:55.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4389999999999999e-06 2023-06-10T21:24:55.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123363+0000", 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011364407 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134727+0000", 2023-06-10T21:24:55.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118065 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134845+0000", 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000154458 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.135000+0000", 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013944756000000001 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148945+0000", 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9670000000000007e-06 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148954+0000", 2023-06-10T21:24:55.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593688599999995 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124890+0000", 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2764e-05 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124903+0000", 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201403 9.5 9.ddfae635 (undecoded) ondisk+write+known_if_redirected e171)", 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 11.555581689, 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997, 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.899 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201403 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9119999999999998e-06 2023-06-10T21:24:55.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133453+0000", 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5379999999999999e-06 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133459+0000", 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0280000000000002e-06 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133463+0000", 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020130169999999999 2023-06-10T21:24:55.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153593+0000", 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4200000000000004e-06 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153600+0000", 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97132188100000005 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124922+0000", 2023-06-10T21:24:55.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2189999999999998e-06 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124929+0000", 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.907 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201415 9.5 9.ce5b1865 (undecoded) ondisk+write+known_if_redirected e172)", 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 11.522529211, 2023-06-10T21:24:55.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998, 2023-06-10T21:24:55.909 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:55.909 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:24:55.909 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:55.909 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:55.909 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201415 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:55.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:55.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166501+0000", 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0360000000000003e-06 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:55.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166507+0000", 2023-06-10T21:24:55.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8352999999999999e-05 2023-06-10T21:24:55.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166526+0000", 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7720000000000002e-06 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166531+0000", 2023-06-10T21:24:55.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5688e-05 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166576+0000", 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8590000000000001e-06 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166582+0000", 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95836468200000002 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124947+0000", 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3220000000000004e-06 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124954+0000", 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998 2023-06-10T21:24:55.917 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.918 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.918 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.918 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:55.918 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:55.918 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:24:55.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:55.919 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:24:55.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:24:56.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:56.075 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:56.202 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:56.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:56.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:56.382 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:56.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: osdmap e182: 8 total, 8 up, 8 in 2023-06-10T21:24:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[186706]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: osdmap e182: 8 total, 8 up, 8 in 2023-06-10T21:24:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:56 smithi064 ceph-mon[180297]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:24:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1033101713' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:24:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: osdmap e182: 8 total, 8 up, 8 in 2023-06-10T21:24:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:56 smithi152 ceph-mon[156819]: failed to encode map e182 with expected crc 2023-06-10T21:24:56.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:24:56.854 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:56.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:56.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:57.016 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:57.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:57.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:57.256 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:24:57.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:57.345 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:57 smithi064 ceph-mon[180297]: pgmap v689: 89 pgs: 1 peering, 88 active+clean; 820 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 8.0 MiB/s, 128 objects/s recovering 2023-06-10T21:24:57.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:57 smithi064 ceph-mon[186706]: pgmap v689: 89 pgs: 1 peering, 88 active+clean; 820 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 8.0 MiB/s, 128 objects/s recovering 2023-06-10T21:24:57.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:57.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:57 smithi152 ceph-mon[156819]: pgmap v689: 89 pgs: 1 peering, 88 active+clean; 820 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 8.0 MiB/s, 128 objects/s recovering 2023-06-10T21:24:57.556 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:57.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:57.669 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:57.754 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:57.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:57.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:57.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:58.051 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:58.096 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:58.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:24:58.116 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:24:58.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:24:58.228 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:24:58.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:58.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:24:58.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:24:58.564 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:58.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:24:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:58.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:24:58.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:24:58.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:24:58.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:59.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:24:59.125 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:24:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:59.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:24:59.143 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":690,"stamp":"2023-06-10T21:24:57.546187+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":806033318,"num_objects":12315,"num_object_clones":0,"num_object_copies":36945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12315,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":205942,"num_write_kb":6977487,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6752,"num_bytes_recovered":442042006,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133182,"ondisk_log_size":133182,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5416724,"kb_used_data":2582996,"kb_used_omap":20,"kb_used_meta":2833387,"kb_avail":744544492,"statfs":{"total":767960285184,"available":762413559808,"internally_reserved":0,"allocated":2644987904,"data_stored":2640405565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2901388942},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-187170816,"num_objects":-2856,"num_object_clones":0,"num_object_copies":-8568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2849,"num_objects_unfound":0,"num_objects_dirty":-2856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2856,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001588"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":164,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618213+0000","last_change":"2023-06-10T21:19:00.497735+0000","last_active":"2023-06-10T21:24:51.618213+0000","last_peered":"2023-06-10T21:24:51.618213+0000","last_clean":"2023-06-10T21:24:51.618213+0000","last_became_active":"2023-06-10T21:19:00.497436+0000","last_became_peered":"2023-06-10T21:19:00.497436+0000","last_unstale":"2023-06-10T21:24:51.618213+0000","last_undegraded":"2023-06-10T21:24:51.618213+0000","last_fullsized":"2023-06-10T21:24:51.618213+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620081+0000","last_change":"2023-06-10T21:19:28.478832+0000","last_active":"2023-06-10T21:24:52.620081+0000","last_peered":"2023-06-10T21:24:52.620081+0000","last_clean":"2023-06-10T21:24:52.620081+0000","last_became_active":"2023-06-10T21:19:28.478412+0000","last_became_peered":"2023-06-10T21:19:28.478412+0000","last_unstale":"2023-06-10T21:24:52.620081+0000","last_undegraded":"2023-06-10T21:24:52.620081+0000","last_fullsized":"2023-06-10T21:24:52.620081+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:49.294677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619865+0000","last_change":"2023-06-10T21:23:46.309763+0000","last_active":"2023-06-10T21:24:52.619865+0000","last_peered":"2023-06-10T21:24:52.619865+0000","last_clean":"2023-06-10T21:24:52.619865+0000","last_became_active":"2023-06-10T21:23:46.309449+0000","last_became_peered":"2023-06-10T21:23:46.309449+0000","last_unstale":"2023-06-10T21:24:52.619865+0000","last_undegraded":"2023-06-10T21:24:52.619865+0000","last_fullsized":"2023-06-10T21:24:52.619865+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:57:16.635893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617362+0000","last_change":"2023-06-10T21:24:45.151792+0000","last_active":"2023-06-10T21:24:51.617362+0000","last_peered":"2023-06-10T21:24:51.617362+0000","last_clean":"2023-06-10T21:24:51.617362+0000","last_became_active":"2023-06-10T21:24:45.151049+0000","last_became_peered":"2023-06-10T21:24:45.151049+0000","last_unstale":"2023-06-10T21:24:51.617362+0000","last_undegraded":"2023-06-10T21:24:51.617362+0000","last_fullsized":"2023-06-10T21:24:51.617362+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:36:00.495165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":177,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619707+0000","last_change":"2023-06-10T21:24:45.144448+0000","last_active":"2023-06-10T21:24:52.619707+0000","last_peered":"2023-06-10T21:24:52.619707+0000","last_clean":"2023-06-10T21:24:52.619707+0000","last_became_active":"2023-06-10T21:24:45.144058+0000","last_became_peered":"2023-06-10T21:24:45.144058+0000","last_unstale":"2023-06-10T21:24:52.619707+0000","last_undegraded":"2023-06-10T21:24:52.619707+0000","last_fullsized":"2023-06-10T21:24:52.619707+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:54:27.983016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621331+0000","last_change":"2023-06-10T21:14:42.434602+0000","last_active":"2023-06-10T21:24:51.621331+0000","last_peered":"2023-06-10T21:24:51.621331+0000","last_clean":"2023-06-10T21:24:51.621331+0000","last_became_active":"2023-06-10T21:14:42.434293+0000","last_became_peered":"2023-06-10T21:14:42.434293+0000","last_unstale":"2023-06-10T21:24:51.621331+0000","last_undegraded":"2023-06-10T21:24:51.621331+0000","last_fullsized":"2023-06-10T21:24:51.621331+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617157+0000","last_change":"2023-06-10T21:24:45.152017+0000","last_active":"2023-06-10T21:24:51.617157+0000","last_peered":"2023-06-10T21:24:51.617157+0000","last_clean":"2023-06-10T21:24:51.617157+0000","last_became_active":"2023-06-10T21:24:45.151446+0000","last_became_peered":"2023-06-10T21:24:45.151446+0000","last_unstale":"2023-06-10T21:24:51.617157+0000","last_undegraded":"2023-06-10T21:24:51.617157+0000","last_fullsized":"2023-06-10T21:24:51.617157+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:04:06.319281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619705+0000","last_change":"2023-06-10T21:19:28.480260+0000","last_active":"2023-06-10T21:24:52.619705+0000","last_peered":"2023-06-10T21:24:52.619705+0000","last_clean":"2023-06-10T21:24:52.619705+0000","last_became_active":"2023-06-10T21:19:28.479855+0000","last_became_peered":"2023-06-10T21:19:28.479855+0000","last_unstale":"2023-06-10T21:24:52.619705+0000","last_undegraded":"2023-06-10T21:24:52.619705+0000","last_fullsized":"2023-06-10T21:24:52.619705+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:51.523733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622147+0000","last_change":"2023-06-10T21:14:42.438045+0000","last_active":"2023-06-10T21:24:51.622147+0000","last_peered":"2023-06-10T21:24:51.622147+0000","last_clean":"2023-06-10T21:24:51.622147+0000","last_became_active":"2023-06-10T21:14:42.437704+0000","last_became_peered":"2023-06-10T21:14:42.437704+0000","last_unstale":"2023-06-10T21:24:51.622147+0000","last_undegraded":"2023-06-10T21:24:51.622147+0000","last_fullsized":"2023-06-10T21:24:51.622147+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619805+0000","last_change":"2023-06-10T21:19:28.483795+0000","last_active":"2023-06-10T21:24:52.619805+0000","last_peered":"2023-06-10T21:24:52.619805+0000","last_clean":"2023-06-10T21:24:52.619805+0000","last_became_active":"2023-06-10T21:19:28.483446+0000","last_became_peered":"2023-06-10T21:19:28.483446+0000","last_unstale":"2023-06-10T21:24:52.619805+0000","last_undegraded":"2023-06-10T21:24:52.619805+0000","last_fullsized":"2023-06-10T21:24:52.619805+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:28:52.762759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":151,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622828+0000","last_change":"2023-06-10T21:19:28.483994+0000","last_active":"2023-06-10T21:24:51.622828+0000","last_peered":"2023-06-10T21:24:51.622828+0000","last_clean":"2023-06-10T21:24:51.622828+0000","last_became_active":"2023-06-10T21:19:28.483563+0000","last_became_peered":"2023-06-10T21:19:28.483563+0000","last_unstale":"2023-06-10T21:24:51.622828+0000","last_undegraded":"2023-06-10T21:24:51.622828+0000","last_fullsized":"2023-06-10T21:24:51.622828+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621097+0000","last_change":"2023-06-10T21:23:46.306714+0000","last_active":"2023-06-10T21:24:51.621097+0000","last_peered":"2023-06-10T21:24:51.621097+0000","last_clean":"2023-06-10T21:24:51.621097+0000","last_became_active":"2023-06-10T21:23:46.306393+0000","last_became_peered":"2023-06-10T21:23:46.306393+0000","last_unstale":"2023-06-10T21:24:51.621097+0000","last_undegraded":"2023-06-10T21:24:51.621097+0000","last_fullsized":"2023-06-10T21:24:51.621097+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":156,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618553+0000","last_change":"2023-06-10T20:59:25.407103+0000","last_active":"2023-06-10T21:24:51.618553+0000","last_peered":"2023-06-10T21:24:51.618553+0000","last_clean":"2023-06-10T21:24:51.618553+0000","last_became_active":"2023-06-10T20:59:25.405861+0000","last_became_peered":"2023-06-10T20:59:25.405861+0000","last_unstale":"2023-06-10T21:24:51.618553+0000","last_undegraded":"2023-06-10T21:24:51.618553+0000","last_fullsized":"2023-06-10T21:24:51.618553+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618316+0000","last_change":"2023-06-10T21:14:42.429785+0000","last_active":"2023-06-10T21:24:51.618316+0000","last_peered":"2023-06-10T21:24:51.618316+0000","last_clean":"2023-06-10T21:24:51.618316+0000","last_became_active":"2023-06-10T21:14:42.429448+0000","last_became_peered":"2023-06-10T21:14:42.429448+0000","last_unstale":"2023-06-10T21:24:51.618316+0000","last_undegraded":"2023-06-10T21:24:51.618316+0000","last_fullsized":"2023-06-10T21:24:51.618316+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622830+0000","last_change":"2023-06-10T21:19:00.492279+0000","last_active":"2023-06-10T21:24:51.622830+0000","last_peered":"2023-06-10T21:24:51.622830+0000","last_clean":"2023-06-10T21:24:51.622830+0000","last_became_active":"2023-06-10T21:19:00.491839+0000","last_became_peered":"2023-06-10T21:19:00.491839+0000","last_unstale":"2023-06-10T21:24:51.622830+0000","last_undegraded":"2023-06-10T21:24:51.622830+0000","last_fullsized":"2023-06-10T21:24:51.622830+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620130+0000","last_change":"2023-06-10T21:19:28.483638+0000","last_active":"2023-06-10T21:24:52.620130+0000","last_peered":"2023-06-10T21:24:52.620130+0000","last_clean":"2023-06-10T21:24:52.620130+0000","last_became_active":"2023-06-10T21:19:28.483281+0000","last_became_peered":"2023-06-10T21:19:28.483281+0000","last_unstale":"2023-06-10T21:24:52.620130+0000","last_undegraded":"2023-06-10T21:24:52.620130+0000","last_fullsized":"2023-06-10T21:24:52.620130+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:16:24.175233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.616691+0000","last_change":"2023-06-10T21:24:45.151415+0000","last_active":"2023-06-10T21:24:51.616691+0000","last_peered":"2023-06-10T21:24:51.616691+0000","last_clean":"2023-06-10T21:24:51.616691+0000","last_became_active":"2023-06-10T21:24:45.151097+0000","last_became_peered":"2023-06-10T21:24:45.151097+0000","last_unstale":"2023-06-10T21:24:51.616691+0000","last_undegraded":"2023-06-10T21:24:51.616691+0000","last_fullsized":"2023-06-10T21:24:51.616691+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:40:14.687278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622232+0000","last_change":"2023-06-10T21:19:28.481386+0000","last_active":"2023-06-10T21:24:51.622232+0000","last_peered":"2023-06-10T21:24:51.622232+0000","last_clean":"2023-06-10T21:24:51.622232+0000","last_became_active":"2023-06-10T21:19:28.481053+0000","last_became_peered":"2023-06-10T21:19:28.481053+0000","last_unstale":"2023-06-10T21:24:51.622232+0000","last_undegraded":"2023-06-10T21:24:51.622232+0000","last_fullsized":"2023-06-10T21:24:51.622232+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":133,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622628+0000","last_change":"2023-06-10T20:59:25.409728+0000","last_active":"2023-06-10T21:24:51.622628+0000","last_peered":"2023-06-10T21:24:51.622628+0000","last_clean":"2023-06-10T21:24:51.622628+0000","last_became_active":"2023-06-10T20:59:25.409345+0000","last_became_peered":"2023-06-10T20:59:25.409345+0000","last_unstale":"2023-06-10T21:24:51.622628+0000","last_undegraded":"2023-06-10T21:24:51.622628+0000","last_fullsized":"2023-06-10T21:24:51.622628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.616917+0000","last_change":"2023-06-10T21:24:45.151123+0000","last_active":"2023-06-10T21:24:51.616917+0000","last_peered":"2023-06-10T21:24:51.616917+0000","last_clean":"2023-06-10T21:24:51.616917+0000","last_became_active":"2023-06-10T21:24:45.150815+0000","last_became_peered":"2023-06-10T21:24:45.150815+0000","last_unstale":"2023-06-10T21:24:51.616917+0000","last_undegraded":"2023-06-10T21:24:51.616917+0000","last_fullsized":"2023-06-10T21:24:51.616917+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:30:28.350444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":194,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.620502+0000","last_change":"2023-06-10T21:24:45.160590+0000","last_active":"2023-06-10T21:24:51.620502+0000","last_peered":"2023-06-10T21:24:51.620502+0000","last_clean":"2023-06-10T21:24:51.620502+0000","last_became_active":"2023-06-10T21:24:45.131915+0000","last_became_peered":"2023-06-10T21:24:45.131915+0000","last_unstale":"2023-06-10T21:24:51.620502+0000","last_undegraded":"2023-06-10T21:24:51.620502+0000","last_fullsized":"2023-06-10T21:24:51.620502+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620145+0000","last_change":"2023-06-10T21:19:28.484436+0000","last_active":"2023-06-10T21:24:52.620145+0000","last_peered":"2023-06-10T21:24:52.620145+0000","last_clean":"2023-06-10T21:24:52.620145+0000","last_became_active":"2023-06-10T21:19:28.484084+0000","last_became_peered":"2023-06-10T21:19:28.484084+0000","last_unstale":"2023-06-10T21:24:52.620145+0000","last_undegraded":"2023-06-10T21:24:52.620145+0000","last_fullsized":"2023-06-10T21:24:52.620145+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:18.235487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"181'12755","reported_seq":14179,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.898647+0000","last_change":"2023-06-10T21:24:46.229248+0000","last_active":"2023-06-10T21:24:52.898647+0000","last_peered":"2023-06-10T21:24:52.898647+0000","last_clean":"2023-06-10T21:24:52.898647+0000","last_became_active":"2023-06-10T21:24:46.138852+0000","last_became_peered":"2023-06-10T21:24:46.138852+0000","last_unstale":"2023-06-10T21:24:52.898647+0000","last_undegraded":"2023-06-10T21:24:52.898647+0000","last_fullsized":"2023-06-10T21:24:52.898647+0000","mapping_epoch":173,"log_start":"135'3432","ondisk_log_start":"135'3432","created":127,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9323,"log_dups_size":0,"ondisk_log_size":9323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:23:11.601121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12755,"num_write_kb":435776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":671,"num_bytes_recovered":43909120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":175,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.623769+0000","last_change":"2023-06-10T21:24:45.162493+0000","last_active":"2023-06-10T21:24:51.623769+0000","last_peered":"2023-06-10T21:24:51.623769+0000","last_clean":"2023-06-10T21:24:51.623769+0000","last_became_active":"2023-06-10T21:24:45.139001+0000","last_became_peered":"2023-06-10T21:24:45.139001+0000","last_unstale":"2023-06-10T21:24:51.623769+0000","last_undegraded":"2023-06-10T21:24:51.623769+0000","last_fullsized":"2023-06-10T21:24:51.623769+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622283+0000","last_change":"2023-06-10T20:59:26.106092+0000","last_active":"2023-06-10T21:24:51.622283+0000","last_peered":"2023-06-10T21:24:51.622283+0000","last_clean":"2023-06-10T21:24:51.622283+0000","last_became_active":"2023-06-10T20:59:26.105697+0000","last_became_peered":"2023-06-10T20:59:26.105697+0000","last_unstale":"2023-06-10T21:24:51.622283+0000","last_undegraded":"2023-06-10T21:24:51.622283+0000","last_fullsized":"2023-06-10T21:24:51.622283+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"181'12985","reported_seq":13061,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.320420+0000","last_change":"2023-06-10T21:23:46.406550+0000","last_active":"2023-06-10T21:24:54.320420+0000","last_peered":"2023-06-10T21:24:54.320420+0000","last_clean":"2023-06-10T21:24:54.320420+0000","last_became_active":"2023-06-10T21:23:46.309661+0000","last_became_peered":"2023-06-10T21:23:46.309661+0000","last_unstale":"2023-06-10T21:24:54.320420+0000","last_undegraded":"2023-06-10T21:24:54.320420+0000","last_fullsized":"2023-06-10T21:24:54.320420+0000","mapping_epoch":157,"log_start":"136'4600","ondisk_log_start":"136'4600","created":127,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":8385,"log_dups_size":0,"ondisk_log_size":8385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53149696,"num_objects":811,"num_object_clones":0,"num_object_copies":2433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12985,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.623272+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:24:51.623272+0000","last_peered":"2023-06-10T21:24:51.623272+0000","last_clean":"2023-06-10T21:24:51.623272+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:24:51.623272+0000","last_undegraded":"2023-06-10T21:24:51.623272+0000","last_fullsized":"2023-06-10T21:24:51.623272+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":162,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618133+0000","last_change":"2023-06-10T21:19:28.487906+0000","last_active":"2023-06-10T21:24:51.618133+0000","last_peered":"2023-06-10T21:24:51.618133+0000","last_clean":"2023-06-10T21:24:51.618133+0000","last_became_active":"2023-06-10T21:19:28.487632+0000","last_became_peered":"2023-06-10T21:19:28.487632+0000","last_unstale":"2023-06-10T21:24:51.618133+0000","last_undegraded":"2023-06-10T21:24:51.618133+0000","last_fullsized":"2023-06-10T21:24:51.618133+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"181'12751","reported_seq":12813,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.650236+0000","last_change":"2023-06-10T21:20:45.268507+0000","last_active":"2023-06-10T21:24:54.650236+0000","last_peered":"2023-06-10T21:24:54.650236+0000","last_clean":"2023-06-10T21:24:54.650236+0000","last_became_active":"2023-06-10T21:20:45.267933+0000","last_became_peered":"2023-06-10T21:20:45.267933+0000","last_unstale":"2023-06-10T21:24:54.650236+0000","last_undegraded":"2023-06-10T21:24:54.650236+0000","last_fullsized":"2023-06-10T21:24:54.650236+0000","mapping_epoch":127,"log_start":"135'3387","ondisk_log_start":"135'3387","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9364,"log_dups_size":0,"ondisk_log_size":9364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49872896,"num_objects":761,"num_object_clones":0,"num_object_copies":2283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12751,"num_write_kb":432384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":134,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622671+0000","last_change":"2023-06-10T20:59:25.412086+0000","last_active":"2023-06-10T21:24:51.622671+0000","last_peered":"2023-06-10T21:24:51.622671+0000","last_clean":"2023-06-10T21:24:51.622671+0000","last_became_active":"2023-06-10T20:59:25.411693+0000","last_became_peered":"2023-06-10T20:59:25.411693+0000","last_unstale":"2023-06-10T21:24:51.622671+0000","last_undegraded":"2023-06-10T21:24:51.622671+0000","last_fullsized":"2023-06-10T21:24:51.622671+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621015+0000","last_change":"2023-06-10T21:14:42.440888+0000","last_active":"2023-06-10T21:24:51.621015+0000","last_peered":"2023-06-10T21:24:51.621015+0000","last_clean":"2023-06-10T21:24:51.621015+0000","last_became_active":"2023-06-10T21:14:42.440547+0000","last_became_peered":"2023-06-10T21:14:42.440547+0000","last_unstale":"2023-06-10T21:24:51.621015+0000","last_undegraded":"2023-06-10T21:24:51.621015+0000","last_fullsized":"2023-06-10T21:24:51.621015+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"181'12804","reported_seq":12866,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.684814+0000","last_change":"2023-06-10T21:20:45.292019+0000","last_active":"2023-06-10T21:24:54.684814+0000","last_peered":"2023-06-10T21:24:54.684814+0000","last_clean":"2023-06-10T21:24:54.684814+0000","last_became_active":"2023-06-10T21:20:45.291698+0000","last_became_peered":"2023-06-10T21:20:45.291698+0000","last_unstale":"2023-06-10T21:24:54.684814+0000","last_undegraded":"2023-06-10T21:24:54.684814+0000","last_fullsized":"2023-06-10T21:24:54.684814+0000","mapping_epoch":127,"log_start":"135'3435","ondisk_log_start":"135'3435","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9369,"log_dups_size":0,"ondisk_log_size":9369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51904512,"num_objects":792,"num_object_clones":0,"num_object_copies":2376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12804,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621804+0000","last_change":"2023-06-10T21:19:28.480945+0000","last_active":"2023-06-10T21:24:51.621804+0000","last_peered":"2023-06-10T21:24:51.621804+0000","last_clean":"2023-06-10T21:24:51.621804+0000","last_became_active":"2023-06-10T21:19:28.480634+0000","last_became_peered":"2023-06-10T21:19:28.480634+0000","last_unstale":"2023-06-10T21:24:51.621804+0000","last_undegraded":"2023-06-10T21:24:51.621804+0000","last_fullsized":"2023-06-10T21:24:51.621804+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617290+0000","last_change":"2023-06-10T21:24:45.144751+0000","last_active":"2023-06-10T21:24:51.617290+0000","last_peered":"2023-06-10T21:24:51.617290+0000","last_clean":"2023-06-10T21:24:51.617290+0000","last_became_active":"2023-06-10T21:24:45.143921+0000","last_became_peered":"2023-06-10T21:24:45.143921+0000","last_unstale":"2023-06-10T21:24:51.617290+0000","last_undegraded":"2023-06-10T21:24:51.617290+0000","last_fullsized":"2023-06-10T21:24:51.617290+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:21:57.739161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"182'12977","reported_seq":13043,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-06-10T21:24:57.186184+0000","last_change":"2023-06-10T21:20:45.268245+0000","last_active":"2023-06-10T21:24:57.186184+0000","last_peered":"2023-06-10T21:24:57.186184+0000","last_clean":"2023-06-10T21:24:57.186184+0000","last_became_active":"2023-06-10T21:20:45.268000+0000","last_became_peered":"2023-06-10T21:20:45.268000+0000","last_unstale":"2023-06-10T21:24:57.186184+0000","last_undegraded":"2023-06-10T21:24:57.186184+0000","last_fullsized":"2023-06-10T21:24:57.186184+0000","mapping_epoch":127,"log_start":"138'5604","ondisk_log_start":"138'5604","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:30:56.393113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45154304,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12977,"num_write_kb":437312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":153,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619983+0000","last_change":"2023-06-10T21:24:45.164405+0000","last_active":"2023-06-10T21:24:52.619983+0000","last_peered":"2023-06-10T21:24:52.619983+0000","last_clean":"2023-06-10T21:24:52.619983+0000","last_became_active":"2023-06-10T21:24:45.151890+0000","last_became_peered":"2023-06-10T21:24:45.151890+0000","last_unstale":"2023-06-10T21:24:52.619983+0000","last_undegraded":"2023-06-10T21:24:52.619983+0000","last_fullsized":"2023-06-10T21:24:52.619983+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:37:16.228629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618462+0000","last_change":"2023-06-10T21:19:28.488064+0000","last_active":"2023-06-10T21:24:51.618462+0000","last_peered":"2023-06-10T21:24:51.618462+0000","last_clean":"2023-06-10T21:24:51.618462+0000","last_became_active":"2023-06-10T21:19:28.487668+0000","last_became_peered":"2023-06-10T21:19:28.487668+0000","last_unstale":"2023-06-10T21:24:51.618462+0000","last_undegraded":"2023-06-10T21:24:51.618462+0000","last_fullsized":"2023-06-10T21:24:51.618462+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"181'12417","reported_seq":12489,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.286133+0000","last_change":"2023-06-10T21:23:46.333354+0000","last_active":"2023-06-10T21:24:54.286133+0000","last_peered":"2023-06-10T21:24:54.286133+0000","last_clean":"2023-06-10T21:24:54.286133+0000","last_became_active":"2023-06-10T21:23:46.306551+0000","last_became_peered":"2023-06-10T21:23:46.306551+0000","last_unstale":"2023-06-10T21:24:54.286133+0000","last_undegraded":"2023-06-10T21:24:54.286133+0000","last_fullsized":"2023-06-10T21:24:54.286133+0000","mapping_epoch":157,"log_start":"134'2800","ondisk_log_start":"134'2800","created":127,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9617,"log_dups_size":0,"ondisk_log_size":9617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50003968,"num_objects":763,"num_object_clones":0,"num_object_copies":2289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12417,"num_write_kb":421760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.620735+0000","last_change":"2023-06-10T21:14:42.440840+0000","last_active":"2023-06-10T21:24:51.620735+0000","last_peered":"2023-06-10T21:24:51.620735+0000","last_clean":"2023-06-10T21:24:51.620735+0000","last_became_active":"2023-06-10T21:14:42.440529+0000","last_became_peered":"2023-06-10T21:14:42.440529+0000","last_unstale":"2023-06-10T21:24:51.620735+0000","last_undegraded":"2023-06-10T21:24:51.620735+0000","last_fullsized":"2023-06-10T21:24:51.620735+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618400+0000","last_change":"2023-06-10T21:19:00.494042+0000","last_active":"2023-06-10T21:24:51.618400+0000","last_peered":"2023-06-10T21:24:51.618400+0000","last_clean":"2023-06-10T21:24:51.618400+0000","last_became_active":"2023-06-10T21:19:00.493705+0000","last_became_peered":"2023-06-10T21:19:00.493705+0000","last_unstale":"2023-06-10T21:24:51.618400+0000","last_undegraded":"2023-06-10T21:24:51.618400+0000","last_fullsized":"2023-06-10T21:24:51.618400+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"181'13014","reported_seq":13076,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.515774+0000","last_change":"2023-06-10T21:20:45.269929+0000","last_active":"2023-06-10T21:24:54.515774+0000","last_peered":"2023-06-10T21:24:54.515774+0000","last_clean":"2023-06-10T21:24:54.515774+0000","last_became_active":"2023-06-10T21:20:45.269497+0000","last_became_peered":"2023-06-10T21:20:45.269497+0000","last_unstale":"2023-06-10T21:24:54.515774+0000","last_undegraded":"2023-06-10T21:24:54.515774+0000","last_fullsized":"2023-06-10T21:24:54.515774+0000","mapping_epoch":127,"log_start":"138'5763","ondisk_log_start":"138'5763","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7251,"log_dups_size":0,"ondisk_log_size":7251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51249152,"num_objects":782,"num_object_clones":0,"num_object_copies":2346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13014,"num_write_kb":441472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":148,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617836+0000","last_change":"2023-06-10T21:19:00.492837+0000","last_active":"2023-06-10T21:24:51.617836+0000","last_peered":"2023-06-10T21:24:51.617836+0000","last_clean":"2023-06-10T21:24:51.617836+0000","last_became_active":"2023-06-10T21:19:00.492507+0000","last_became_peered":"2023-06-10T21:19:00.492507+0000","last_unstale":"2023-06-10T21:24:51.617836+0000","last_undegraded":"2023-06-10T21:24:51.617836+0000","last_fullsized":"2023-06-10T21:24:51.617836+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":147,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618818+0000","last_change":"2023-06-10T21:14:42.430919+0000","last_active":"2023-06-10T21:24:51.618818+0000","last_peered":"2023-06-10T21:24:51.618818+0000","last_clean":"2023-06-10T21:24:51.618818+0000","last_became_active":"2023-06-10T21:14:42.430543+0000","last_became_peered":"2023-06-10T21:14:42.430543+0000","last_unstale":"2023-06-10T21:24:51.618818+0000","last_undegraded":"2023-06-10T21:24:51.618818+0000","last_fullsized":"2023-06-10T21:24:51.618818+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"182'13053","reported_seq":13119,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-06-10T21:24:57.174919+0000","last_change":"2023-06-10T21:20:45.292026+0000","last_active":"2023-06-10T21:24:57.174919+0000","last_peered":"2023-06-10T21:24:57.174919+0000","last_clean":"2023-06-10T21:24:57.174919+0000","last_became_active":"2023-06-10T21:20:45.291704+0000","last_became_peered":"2023-06-10T21:20:45.291704+0000","last_unstale":"2023-06-10T21:24:57.174919+0000","last_undegraded":"2023-06-10T21:24:57.174919+0000","last_fullsized":"2023-06-10T21:24:57.174919+0000","mapping_epoch":127,"log_start":"138'5667","ondisk_log_start":"138'5667","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7386,"log_dups_size":0,"ondisk_log_size":7386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:33:18.329714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45023232,"num_objects":687,"num_object_clones":0,"num_object_copies":2061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13053,"num_write_kb":439680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":144,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619592+0000","last_change":"2023-06-10T21:19:28.483541+0000","last_active":"2023-06-10T21:24:52.619592+0000","last_peered":"2023-06-10T21:24:52.619592+0000","last_clean":"2023-06-10T21:24:52.619592+0000","last_became_active":"2023-06-10T21:19:28.483188+0000","last_became_peered":"2023-06-10T21:19:28.483188+0000","last_unstale":"2023-06-10T21:24:52.619592+0000","last_undegraded":"2023-06-10T21:24:52.619592+0000","last_fullsized":"2023-06-10T21:24:52.619592+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:41:19.340547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":128,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620229+0000","last_change":"2023-06-10T21:23:46.310102+0000","last_active":"2023-06-10T21:24:52.620229+0000","last_peered":"2023-06-10T21:24:52.620229+0000","last_clean":"2023-06-10T21:24:52.620229+0000","last_became_active":"2023-06-10T21:23:46.309636+0000","last_became_peered":"2023-06-10T21:23:46.309636+0000","last_unstale":"2023-06-10T21:24:52.620229+0000","last_undegraded":"2023-06-10T21:24:52.620229+0000","last_fullsized":"2023-06-10T21:24:52.620229+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:13.066763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.f","version":"181'12706","reported_seq":12768,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.673340+0000","last_change":"2023-06-10T21:20:45.263336+0000","last_active":"2023-06-10T21:24:54.673340+0000","last_peered":"2023-06-10T21:24:54.673340+0000","last_clean":"2023-06-10T21:24:54.673340+0000","last_became_active":"2023-06-10T21:20:45.263011+0000","last_became_peered":"2023-06-10T21:20:45.263011+0000","last_unstale":"2023-06-10T21:24:54.673340+0000","last_undegraded":"2023-06-10T21:24:54.673340+0000","last_fullsized":"2023-06-10T21:24:54.673340+0000","mapping_epoch":127,"log_start":"135'3292","ondisk_log_start":"135'3292","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9414,"log_dups_size":0,"ondisk_log_size":9414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12706,"num_write_kb":431744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.616783+0000","last_change":"2023-06-10T21:24:45.143914+0000","last_active":"2023-06-10T21:24:51.616783+0000","last_peered":"2023-06-10T21:24:51.616783+0000","last_clean":"2023-06-10T21:24:51.616783+0000","last_became_active":"2023-06-10T21:24:45.143600+0000","last_became_peered":"2023-06-10T21:24:45.143600+0000","last_unstale":"2023-06-10T21:24:51.616783+0000","last_undegraded":"2023-06-10T21:24:51.616783+0000","last_fullsized":"2023-06-10T21:24:51.616783+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:50.288121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622293+0000","last_change":"2023-06-10T21:19:28.482192+0000","last_active":"2023-06-10T21:24:51.622293+0000","last_peered":"2023-06-10T21:24:51.622293+0000","last_clean":"2023-06-10T21:24:51.622293+0000","last_became_active":"2023-06-10T21:19:28.481824+0000","last_became_peered":"2023-06-10T21:19:28.481824+0000","last_unstale":"2023-06-10T21:24:51.622293+0000","last_undegraded":"2023-06-10T21:24:51.622293+0000","last_fullsized":"2023-06-10T21:24:51.622293+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615311+0000","last_change":"2023-06-10T21:19:00.492398+0000","last_active":"2023-06-10T21:24:51.615311+0000","last_peered":"2023-06-10T21:24:51.615311+0000","last_clean":"2023-06-10T21:24:51.615311+0000","last_became_active":"2023-06-10T21:19:00.492070+0000","last_became_peered":"2023-06-10T21:19:00.492070+0000","last_unstale":"2023-06-10T21:24:51.615311+0000","last_undegraded":"2023-06-10T21:24:51.615311+0000","last_fullsized":"2023-06-10T21:24:51.615311+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:23:11.811768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"181'12705","reported_seq":16804,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.898012+0000","last_change":"2023-06-10T21:24:50.215574+0000","last_active":"2023-06-10T21:24:52.898012+0000","last_peered":"2023-06-10T21:24:52.898012+0000","last_clean":"2023-06-10T21:24:52.898012+0000","last_became_active":"2023-06-10T21:24:50.182193+0000","last_became_peered":"2023-06-10T21:24:50.182193+0000","last_unstale":"2023-06-10T21:24:52.898012+0000","last_undegraded":"2023-06-10T21:24:52.898012+0000","last_fullsized":"2023-06-10T21:24:52.898012+0000","mapping_epoch":177,"log_start":"138'5523","ondisk_log_start":"138'5523","created":127,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:43:10.415123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12705,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1998,"num_bytes_recovered":130940928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":152,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621962+0000","last_change":"2023-06-10T21:14:42.436167+0000","last_active":"2023-06-10T21:24:51.621962+0000","last_peered":"2023-06-10T21:24:51.621962+0000","last_clean":"2023-06-10T21:24:51.621962+0000","last_became_active":"2023-06-10T21:14:42.435624+0000","last_became_peered":"2023-06-10T21:14:42.435624+0000","last_unstale":"2023-06-10T21:24:51.621962+0000","last_undegraded":"2023-06-10T21:24:51.621962+0000","last_fullsized":"2023-06-10T21:24:51.621962+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618430+0000","last_change":"2023-06-10T21:24:45.145544+0000","last_active":"2023-06-10T21:24:51.618430+0000","last_peered":"2023-06-10T21:24:51.618430+0000","last_clean":"2023-06-10T21:24:51.618430+0000","last_became_active":"2023-06-10T21:24:45.145239+0000","last_became_peered":"2023-06-10T21:24:45.145239+0000","last_unstale":"2023-06-10T21:24:51.618430+0000","last_undegraded":"2023-06-10T21:24:51.618430+0000","last_fullsized":"2023-06-10T21:24:51.618430+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":142,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615629+0000","last_change":"2023-06-10T21:23:46.305694+0000","last_active":"2023-06-10T21:24:51.615629+0000","last_peered":"2023-06-10T21:24:51.615629+0000","last_clean":"2023-06-10T21:24:51.615629+0000","last_became_active":"2023-06-10T21:23:46.305252+0000","last_became_peered":"2023-06-10T21:23:46.305252+0000","last_unstale":"2023-06-10T21:24:51.615629+0000","last_undegraded":"2023-06-10T21:24:51.615629+0000","last_fullsized":"2023-06-10T21:24:51.615629+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:15:57.437674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"181'12870","reported_seq":12932,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.650965+0000","last_change":"2023-06-10T21:20:45.268279+0000","last_active":"2023-06-10T21:24:54.650965+0000","last_peered":"2023-06-10T21:24:54.650965+0000","last_clean":"2023-06-10T21:24:54.650965+0000","last_became_active":"2023-06-10T21:20:45.267886+0000","last_became_peered":"2023-06-10T21:20:45.267886+0000","last_unstale":"2023-06-10T21:24:54.650965+0000","last_undegraded":"2023-06-10T21:24:54.650965+0000","last_fullsized":"2023-06-10T21:24:54.650965+0000","mapping_epoch":127,"log_start":"135'3515","ondisk_log_start":"135'3515","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9355,"log_dups_size":0,"ondisk_log_size":9355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52428800,"num_objects":800,"num_object_clones":0,"num_object_copies":2400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12870,"num_write_kb":437440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":166,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615269+0000","last_change":"2023-06-10T21:24:45.139619+0000","last_active":"2023-06-10T21:24:51.615269+0000","last_peered":"2023-06-10T21:24:51.615269+0000","last_clean":"2023-06-10T21:24:51.615269+0000","last_became_active":"2023-06-10T21:24:45.139285+0000","last_became_peered":"2023-06-10T21:24:45.139285+0000","last_unstale":"2023-06-10T21:24:51.615269+0000","last_undegraded":"2023-06-10T21:24:51.615269+0000","last_fullsized":"2023-06-10T21:24:51.615269+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:02.209551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615212+0000","last_change":"2023-06-10T21:23:46.307260+0000","last_active":"2023-06-10T21:24:51.615212+0000","last_peered":"2023-06-10T21:24:51.615212+0000","last_clean":"2023-06-10T21:24:51.615212+0000","last_became_active":"2023-06-10T21:23:46.306268+0000","last_became_peered":"2023-06-10T21:23:46.306268+0000","last_unstale":"2023-06-10T21:24:51.615212+0000","last_undegraded":"2023-06-10T21:24:51.615212+0000","last_fullsized":"2023-06-10T21:24:51.615212+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:44:18.334577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":110,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622286+0000","last_change":"2023-06-10T21:24:45.138749+0000","last_active":"2023-06-10T21:24:51.622286+0000","last_peered":"2023-06-10T21:24:51.622286+0000","last_clean":"2023-06-10T21:24:51.622286+0000","last_became_active":"2023-06-10T21:24:45.138391+0000","last_became_peered":"2023-06-10T21:24:45.138391+0000","last_unstale":"2023-06-10T21:24:51.622286+0000","last_undegraded":"2023-06-10T21:24:51.622286+0000","last_fullsized":"2023-06-10T21:24:51.622286+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"182'12672","reported_seq":16787,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-06-10T21:24:57.167434+0000","last_change":"2023-06-10T21:24:52.654054+0000","last_active":"2023-06-10T21:24:57.167434+0000","last_peered":"2023-06-10T21:24:57.167434+0000","last_clean":"2023-06-10T21:24:57.167434+0000","last_became_active":"2023-06-10T21:24:52.624287+0000","last_became_peered":"2023-06-10T21:24:52.624287+0000","last_unstale":"2023-06-10T21:24:57.167434+0000","last_undegraded":"2023-06-10T21:24:57.167434+0000","last_fullsized":"2023-06-10T21:24:57.167434+0000","mapping_epoch":180,"log_start":"137'5271","ondisk_log_start":"137'5271","created":127,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7401,"log_dups_size":0,"ondisk_log_size":7401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:37:59.591489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44171264,"num_objects":674,"num_object_clones":0,"num_object_copies":2022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12672,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1998,"num_bytes_recovered":130809856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":152,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622030+0000","last_change":"2023-06-10T21:19:00.490798+0000","last_active":"2023-06-10T21:24:51.622030+0000","last_peered":"2023-06-10T21:24:51.622030+0000","last_clean":"2023-06-10T21:24:51.622030+0000","last_became_active":"2023-06-10T21:19:00.490432+0000","last_became_peered":"2023-06-10T21:19:00.490432+0000","last_unstale":"2023-06-10T21:24:51.622030+0000","last_undegraded":"2023-06-10T21:24:51.622030+0000","last_fullsized":"2023-06-10T21:24:51.622030+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":430,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.623753+0000","last_change":"2023-06-10T21:24:48.232584+0000","last_active":"2023-06-10T21:24:51.623753+0000","last_peered":"2023-06-10T21:24:51.623753+0000","last_clean":"2023-06-10T21:24:51.623753+0000","last_became_active":"2023-06-10T21:24:48.232170+0000","last_became_peered":"2023-06-10T21:24:48.232170+0000","last_unstale":"2023-06-10T21:24:51.623753+0000","last_undegraded":"2023-06-10T21:24:51.623753+0000","last_fullsized":"2023-06-10T21:24:51.623753+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":122,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619807+0000","last_change":"2023-06-10T21:19:28.481502+0000","last_active":"2023-06-10T21:24:52.619807+0000","last_peered":"2023-06-10T21:24:52.619807+0000","last_clean":"2023-06-10T21:24:52.619807+0000","last_became_active":"2023-06-10T21:19:28.480113+0000","last_became_peered":"2023-06-10T21:19:28.480113+0000","last_unstale":"2023-06-10T21:24:52.619807+0000","last_undegraded":"2023-06-10T21:24:52.619807+0000","last_fullsized":"2023-06-10T21:24:52.619807+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:24.889679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":132,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621392+0000","last_change":"2023-06-10T20:59:26.657337+0000","last_active":"2023-06-10T21:24:51.621392+0000","last_peered":"2023-06-10T21:24:51.621392+0000","last_clean":"2023-06-10T21:24:51.621392+0000","last_became_active":"2023-06-10T20:59:26.656944+0000","last_became_peered":"2023-06-10T20:59:26.656944+0000","last_unstale":"2023-06-10T21:24:51.621392+0000","last_undegraded":"2023-06-10T21:24:51.621392+0000","last_fullsized":"2023-06-10T21:24:51.621392+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"181'12881","reported_seq":17105,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.870020+0000","last_change":"2023-06-10T21:24:51.654756+0000","last_active":"2023-06-10T21:24:52.870020+0000","last_peered":"2023-06-10T21:24:52.870020+0000","last_clean":"2023-06-10T21:24:52.870020+0000","last_became_active":"2023-06-10T21:24:51.622949+0000","last_became_peered":"2023-06-10T21:24:51.622949+0000","last_unstale":"2023-06-10T21:24:52.870020+0000","last_undegraded":"2023-06-10T21:24:52.870020+0000","last_fullsized":"2023-06-10T21:24:52.870020+0000","mapping_epoch":179,"log_start":"134'3299","ondisk_log_start":"134'3299","created":127,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":9582,"log_dups_size":0,"ondisk_log_size":9582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:50.665691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55640064,"num_objects":849,"num_object_clones":0,"num_object_copies":2547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12881,"num_write_kb":439360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2063,"num_bytes_recovered":135200768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":185,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.623648+0000","last_change":"2023-06-10T21:24:45.142686+0000","last_active":"2023-06-10T21:24:51.623648+0000","last_peered":"2023-06-10T21:24:51.623648+0000","last_clean":"2023-06-10T21:24:51.623648+0000","last_became_active":"2023-06-10T21:24:45.142396+0000","last_became_peered":"2023-06-10T21:24:45.142396+0000","last_unstale":"2023-06-10T21:24:51.623648+0000","last_undegraded":"2023-06-10T21:24:51.623648+0000","last_fullsized":"2023-06-10T21:24:51.623648+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.616998+0000","last_change":"2023-06-10T21:24:45.142808+0000","last_active":"2023-06-10T21:24:51.616998+0000","last_peered":"2023-06-10T21:24:51.616998+0000","last_clean":"2023-06-10T21:24:51.616998+0000","last_became_active":"2023-06-10T21:24:45.139551+0000","last_became_peered":"2023-06-10T21:24:45.139551+0000","last_unstale":"2023-06-10T21:24:51.616998+0000","last_undegraded":"2023-06-10T21:24:51.616998+0000","last_fullsized":"2023-06-10T21:24:51.616998+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:29:44.420744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.616641+0000","last_change":"2023-06-10T21:24:45.145593+0000","last_active":"2023-06-10T21:24:51.616641+0000","last_peered":"2023-06-10T21:24:51.616641+0000","last_clean":"2023-06-10T21:24:51.616641+0000","last_became_active":"2023-06-10T21:24:45.145175+0000","last_became_peered":"2023-06-10T21:24:45.145175+0000","last_unstale":"2023-06-10T21:24:51.616641+0000","last_undegraded":"2023-06-10T21:24:51.616641+0000","last_fullsized":"2023-06-10T21:24:51.616641+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:21:05.941104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"181'12972","reported_seq":13034,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.413242+0000","last_change":"2023-06-10T21:20:45.292306+0000","last_active":"2023-06-10T21:24:54.413242+0000","last_peered":"2023-06-10T21:24:54.413242+0000","last_clean":"2023-06-10T21:24:54.413242+0000","last_became_active":"2023-06-10T21:20:45.291923+0000","last_became_peered":"2023-06-10T21:20:45.291923+0000","last_unstale":"2023-06-10T21:24:54.413242+0000","last_undegraded":"2023-06-10T21:24:54.413242+0000","last_fullsized":"2023-06-10T21:24:54.413242+0000","mapping_epoch":127,"log_start":"138'5671","ondisk_log_start":"138'5671","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7301,"log_dups_size":0,"ondisk_log_size":7301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12972,"num_write_kb":439168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":170,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622745+0000","last_change":"2023-06-10T21:24:45.139255+0000","last_active":"2023-06-10T21:24:51.622745+0000","last_peered":"2023-06-10T21:24:51.622745+0000","last_clean":"2023-06-10T21:24:51.622745+0000","last_became_active":"2023-06-10T21:24:45.138895+0000","last_became_peered":"2023-06-10T21:24:45.138895+0000","last_unstale":"2023-06-10T21:24:51.622745+0000","last_undegraded":"2023-06-10T21:24:51.622745+0000","last_fullsized":"2023-06-10T21:24:51.622745+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615710+0000","last_change":"2023-06-10T21:23:46.307703+0000","last_active":"2023-06-10T21:24:51.615710+0000","last_peered":"2023-06-10T21:24:51.615710+0000","last_clean":"2023-06-10T21:24:51.615710+0000","last_became_active":"2023-06-10T21:23:46.307381+0000","last_became_peered":"2023-06-10T21:23:46.307381+0000","last_unstale":"2023-06-10T21:24:51.615710+0000","last_undegraded":"2023-06-10T21:24:51.615710+0000","last_fullsized":"2023-06-10T21:24:51.615710+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:01.121461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617206+0000","last_change":"2023-06-10T21:24:45.144827+0000","last_active":"2023-06-10T21:24:51.617206+0000","last_peered":"2023-06-10T21:24:51.617206+0000","last_clean":"2023-06-10T21:24:51.617206+0000","last_became_active":"2023-06-10T21:24:45.144430+0000","last_became_peered":"2023-06-10T21:24:45.144430+0000","last_unstale":"2023-06-10T21:24:51.617206+0000","last_undegraded":"2023-06-10T21:24:51.617206+0000","last_fullsized":"2023-06-10T21:24:51.617206+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:40:09.286655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"181'13176","reported_seq":13239,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:54.499631+0000","last_change":"2023-06-10T21:20:45.270670+0000","last_active":"2023-06-10T21:24:54.499631+0000","last_peered":"2023-06-10T21:24:54.499631+0000","last_clean":"2023-06-10T21:24:54.499631+0000","last_became_active":"2023-06-10T21:20:45.270328+0000","last_became_peered":"2023-06-10T21:20:45.270328+0000","last_unstale":"2023-06-10T21:24:54.499631+0000","last_undegraded":"2023-06-10T21:24:54.499631+0000","last_fullsized":"2023-06-10T21:24:54.499631+0000","mapping_epoch":127,"log_start":"138'5880","ondisk_log_start":"138'5880","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7296,"log_dups_size":0,"ondisk_log_size":7296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51314712,"num_objects":784,"num_object_clones":0,"num_object_copies":2352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":784,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13176,"num_write_kb":446657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":153,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620183+0000","last_change":"2023-06-10T21:24:45.166580+0000","last_active":"2023-06-10T21:24:52.620183+0000","last_peered":"2023-06-10T21:24:52.620183+0000","last_clean":"2023-06-10T21:24:52.620183+0000","last_became_active":"2023-06-10T21:24:45.152465+0000","last_became_peered":"2023-06-10T21:24:45.152465+0000","last_unstale":"2023-06-10T21:24:52.620183+0000","last_undegraded":"2023-06-10T21:24:52.620183+0000","last_fullsized":"2023-06-10T21:24:52.620183+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:40:33.455228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621702+0000","last_change":"2023-06-10T21:23:46.310628+0000","last_active":"2023-06-10T21:24:51.621702+0000","last_peered":"2023-06-10T21:24:51.621702+0000","last_clean":"2023-06-10T21:24:51.621702+0000","last_became_active":"2023-06-10T21:23:46.310211+0000","last_became_peered":"2023-06-10T21:23:46.310211+0000","last_unstale":"2023-06-10T21:24:51.621702+0000","last_undegraded":"2023-06-10T21:24:51.621702+0000","last_fullsized":"2023-06-10T21:24:51.621702+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619598+0000","last_change":"2023-06-10T21:24:45.143922+0000","last_active":"2023-06-10T21:24:52.619598+0000","last_peered":"2023-06-10T21:24:52.619598+0000","last_clean":"2023-06-10T21:24:52.619598+0000","last_became_active":"2023-06-10T21:24:45.143595+0000","last_became_peered":"2023-06-10T21:24:45.143595+0000","last_unstale":"2023-06-10T21:24:52.619598+0000","last_undegraded":"2023-06-10T21:24:52.619598+0000","last_fullsized":"2023-06-10T21:24:52.619598+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:52:39.571959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"182'12943","reported_seq":13008,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-06-10T21:24:57.135594+0000","last_change":"2023-06-10T21:20:45.292522+0000","last_active":"2023-06-10T21:24:57.135594+0000","last_peered":"2023-06-10T21:24:57.135594+0000","last_clean":"2023-06-10T21:24:57.135594+0000","last_became_active":"2023-06-10T21:20:45.292219+0000","last_became_peered":"2023-06-10T21:20:45.292219+0000","last_unstale":"2023-06-10T21:24:57.135594+0000","last_undegraded":"2023-06-10T21:24:57.135594+0000","last_fullsized":"2023-06-10T21:24:57.135594+0000","mapping_epoch":127,"log_start":"138'5555","ondisk_log_start":"138'5555","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:20:44.235522+0000","last_clean_scrub_stamp":"2023-06-10T21:20:44.235522+0000","objects_scrubbed":0,"log_size":7388,"log_dups_size":0,"ondisk_log_size":7388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:07.877590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12943,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":152,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.621256+0000","last_change":"2023-06-10T21:19:00.494592+0000","last_active":"2023-06-10T21:24:51.621256+0000","last_peered":"2023-06-10T21:24:51.621256+0000","last_clean":"2023-06-10T21:24:51.621256+0000","last_became_active":"2023-06-10T21:19:00.494252+0000","last_became_peered":"2023-06-10T21:19:00.494252+0000","last_unstale":"2023-06-10T21:24:51.621256+0000","last_undegraded":"2023-06-10T21:24:51.621256+0000","last_fullsized":"2023-06-10T21:24:51.621256+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":23,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617308+0000","last_change":"2023-06-10T21:24:45.166159+0000","last_active":"2023-06-10T21:24:51.617308+0000","last_peered":"2023-06-10T21:24:51.617308+0000","last_clean":"2023-06-10T21:24:51.617308+0000","last_became_active":"2023-06-10T21:24:45.147781+0000","last_became_peered":"2023-06-10T21:24:45.147781+0000","last_unstale":"2023-06-10T21:24:51.617308+0000","last_undegraded":"2023-06-10T21:24:51.617308+0000","last_fullsized":"2023-06-10T21:24:51.617308+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:25:35.150006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":132,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618075+0000","last_change":"2023-06-10T20:59:26.107275+0000","last_active":"2023-06-10T21:24:51.618075+0000","last_peered":"2023-06-10T21:24:51.618075+0000","last_clean":"2023-06-10T21:24:51.618075+0000","last_became_active":"2023-06-10T20:59:26.106929+0000","last_became_peered":"2023-06-10T20:59:26.106929+0000","last_unstale":"2023-06-10T21:24:51.618075+0000","last_undegraded":"2023-06-10T21:24:51.618075+0000","last_fullsized":"2023-06-10T21:24:51.618075+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":19,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617074+0000","last_change":"2023-06-10T21:24:45.144952+0000","last_active":"2023-06-10T21:24:51.617074+0000","last_peered":"2023-06-10T21:24:51.617074+0000","last_clean":"2023-06-10T21:24:51.617074+0000","last_became_active":"2023-06-10T21:24:45.144474+0000","last_became_peered":"2023-06-10T21:24:45.144474+0000","last_unstale":"2023-06-10T21:24:51.617074+0000","last_undegraded":"2023-06-10T21:24:51.617074+0000","last_fullsized":"2023-06-10T21:24:51.617074+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:25:09.157193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.615546+0000","last_change":"2023-06-10T21:19:28.482329+0000","last_active":"2023-06-10T21:24:51.615546+0000","last_peered":"2023-06-10T21:24:51.615546+0000","last_clean":"2023-06-10T21:24:51.615546+0000","last_became_active":"2023-06-10T21:19:28.481933+0000","last_became_peered":"2023-06-10T21:19:28.481933+0000","last_unstale":"2023-06-10T21:24:51.615546+0000","last_undegraded":"2023-06-10T21:24:51.615546+0000","last_fullsized":"2023-06-10T21:24:51.615546+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:44.828605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":150,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622093+0000","last_change":"2023-06-10T21:14:42.436081+0000","last_active":"2023-06-10T21:24:51.622093+0000","last_peered":"2023-06-10T21:24:51.622093+0000","last_clean":"2023-06-10T21:24:51.622093+0000","last_became_active":"2023-06-10T21:14:42.435634+0000","last_became_peered":"2023-06-10T21:14:42.435634+0000","last_unstale":"2023-06-10T21:24:51.622093+0000","last_undegraded":"2023-06-10T21:24:51.622093+0000","last_fullsized":"2023-06-10T21:24:51.622093+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618472+0000","last_change":"2023-06-10T21:19:28.487998+0000","last_active":"2023-06-10T21:24:51.618472+0000","last_peered":"2023-06-10T21:24:51.618472+0000","last_clean":"2023-06-10T21:24:51.618472+0000","last_became_active":"2023-06-10T21:19:28.487641+0000","last_became_peered":"2023-06-10T21:19:28.487641+0000","last_unstale":"2023-06-10T21:24:51.618472+0000","last_undegraded":"2023-06-10T21:24:51.618472+0000","last_fullsized":"2023-06-10T21:24:51.618472+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":135,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.622372+0000","last_change":"2023-06-10T20:59:25.409286+0000","last_active":"2023-06-10T21:24:51.622372+0000","last_peered":"2023-06-10T21:24:51.622372+0000","last_clean":"2023-06-10T21:24:51.622372+0000","last_became_active":"2023-06-10T20:59:25.408921+0000","last_became_peered":"2023-06-10T20:59:25.408921+0000","last_unstale":"2023-06-10T21:24:51.622372+0000","last_undegraded":"2023-06-10T21:24:51.622372+0000","last_fullsized":"2023-06-10T21:24:51.622372+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.618357+0000","last_change":"2023-06-10T21:24:45.142125+0000","last_active":"2023-06-10T21:24:51.618357+0000","last_peered":"2023-06-10T21:24:51.618357+0000","last_clean":"2023-06-10T21:24:51.618357+0000","last_became_active":"2023-06-10T21:24:45.141798+0000","last_became_peered":"2023-06-10T21:24:45.141798+0000","last_unstale":"2023-06-10T21:24:51.618357+0000","last_undegraded":"2023-06-10T21:24:51.618357+0000","last_fullsized":"2023-06-10T21:24:51.618357+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.620024+0000","last_change":"2023-06-10T21:19:28.482461+0000","last_active":"2023-06-10T21:24:52.620024+0000","last_peered":"2023-06-10T21:24:52.620024+0000","last_clean":"2023-06-10T21:24:52.620024+0000","last_became_active":"2023-06-10T21:19:28.482042+0000","last_became_peered":"2023-06-10T21:19:28.482042+0000","last_unstale":"2023-06-10T21:24:52.620024+0000","last_undegraded":"2023-06-10T21:24:52.620024+0000","last_fullsized":"2023-06-10T21:24:52.620024+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:37:00.277625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-06-10T21:24:52.619766+0000","last_change":"2023-06-10T21:24:45.152411+0000","last_active":"2023-06-10T21:24:52.619766+0000","last_peered":"2023-06-10T21:24:52.619766+0000","last_clean":"2023-06-10T21:24:52.619766+0000","last_became_active":"2023-06-10T21:24:45.151910+0000","last_became_peered":"2023-06-10T21:24:45.151910+0000","last_unstale":"2023-06-10T21:24:52.619766+0000","last_undegraded":"2023-06-10T21:24:52.619766+0000","last_fullsized":"2023-06-10T21:24:52.619766+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:50:15.382296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":25,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617013+0000","last_change":"2023-06-10T21:24:45.163443+0000","last_active":"2023-06-10T21:24:51.617013+0000","last_peered":"2023-06-10T21:24:51.617013+0000","last_clean":"2023-06-10T21:24:51.617013+0000","last_became_active":"2023-06-10T21:24:45.139444+0000","last_became_peered":"2023-06-10T21:24:45.139444+0000","last_unstale":"2023-06-10T21:24:51.617013+0000","last_undegraded":"2023-06-10T21:24:51.617013+0000","last_fullsized":"2023-06-10T21:24:51.617013+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:28:30.022573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":180,"state":"active+clean","last_fresh":"2023-06-10T21:24:51.617809+0000","last_change":"2023-06-10T21:19:00.492504+0000","last_active":"2023-06-10T21:24:51.617809+0000","last_peered":"2023-06-10T21:24:51.617809+0000","last_clean":"2023-06-10T21:24:51.617809+0000","last_became_active":"2023-06-10T21:19:00.492209+0000","last_became_peered":"2023-06-10T21:19:00.492209+0000","last_unstale":"2023-06-10T21:24:51.617809+0000","last_undegraded":"2023-06-10T21:24:51.617809+0000","last_fullsized":"2023-06-10T21:24:51.617809+0000","mapping_epoch":120,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":121,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":598,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":805437464,"num_objects":12291,"num_object_clones":0,"num_object_copies":36873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12291,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205681,"num_write_kb":6975041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6731,"num_bytes_recovered":440860672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2552115200,"data_stored":2552102984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132987,"ondisk_log_size":132987,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430534,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":965264,"kb_used_data":523784,"kb_used_omap":0,"kb_used_meta":441472,"kb_avail":92779888,"statfs":{"total":95995035648,"available":95006605312,"internally_reserved":0,"allocated":536354816,"data_stored":535840918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":452067328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:14 2023","interfaces":[{"interface":"back","average":{"1min":4.2190000000000003,"5min":2.5499999999999998,"15min":1.845},"min":{"1min":0.29699999999999999,"5min":0.218,"15min":0.218},"max":{"1min":13.566000000000001,"5min":53.706000000000003,"15min":53.706000000000003},"last":3.4470000000000001},{"interface":"front","average":{"1min":3.6240000000000001,"5min":2.3439999999999999,"15min":2.1429999999999998},"min":{"1min":0.253,"5min":0.252,"15min":0.245},"max":{"1min":10.102,"5min":53.777000000000001,"15min":53.777000000000001},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:23:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9660000000000002,"5min":3.9239999999999999,"15min":2.4409999999999998},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.176,"5min":130.19399999999999,"15min":130.19399999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.327,"5min":6.04,"15min":3.0990000000000002},"min":{"1min":0.29199999999999998,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":8.1769999999999996,"5min":212.20599999999999,"15min":212.20599999999999},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":1.73,"15min":0.92100000000000004},"min":{"1min":0.32500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":6.8730000000000002,"5min":9.1609999999999996,"15min":9.1609999999999996},"last":1.321},{"interface":"front","average":{"1min":3.3980000000000001,"5min":3.8660000000000001,"15min":1.6439999999999999},"min":{"1min":0.32800000000000001,"5min":0.309,"15min":0.309},"max":{"1min":12.301,"5min":212.13399999999999,"15min":212.13399999999999},"last":0.53800000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:24:38 2023","interfaces":[{"interface":"back","average":{"1min":6.4139999999999997,"5min":4.7699999999999996,"15min":1.964},"min":{"1min":0.46800000000000003,"5min":0.308,"15min":0.308},"max":{"1min":61.441000000000003,"5min":63.406999999999996,"15min":63.406999999999996},"last":1.3440000000000001},{"interface":"front","average":{"1min":7.1550000000000002,"5min":4.9960000000000004,"15min":2.056},"min":{"1min":0.33600000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":61.956000000000003,"5min":63.549999999999997,"15min":63.549999999999997},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:24:38 2023","interfaces":[{"interface":"back","average":{"1min":1.625,"5min":1.0189999999999999,"15min":1.897},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":8.1419999999999995,"5min":8.1419999999999995,"15min":219.17699999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":3.137,"5min":1.714,"15min":2.2469999999999999},"min":{"1min":0.34799999999999998,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":7.1619999999999999,"5min":9.2129999999999992,"15min":219.13999999999999},"last":1.288}]},{"osd":5,"last update":"Sat Jun 10 21:24:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2770000000000001,"5min":3.8450000000000002,"15min":3.419},"min":{"1min":0.40600000000000003,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":12.988,"5min":211.99799999999999,"15min":311.202},"last":0.63},{"interface":"front","average":{"1min":2.8410000000000002,"5min":3.4780000000000002,"15min":3.2229999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":8.2449999999999992,"5min":212.23699999999999,"15min":311.875},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:24:38 2023","interfaces":[{"interface":"back","average":{"1min":3.1190000000000002,"5min":1.627,"15min":1.226},"min":{"1min":0.374,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":8.4510000000000005,"5min":10.159000000000001,"15min":10.159000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.8399999999999999,"5min":1.6080000000000001,"15min":1.173},"min":{"1min":0.41499999999999998,"5min":0.33100000000000002,"15min":0.32800000000000001},"max":{"1min":10.874000000000001,"5min":10.874000000000001,"15min":10.874000000000001},"last":3.0550000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626762,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469244,"kb_used_data":154932,"kb_used_omap":0,"kb_used_meta":314304,"kb_avail":93275908,"statfs":{"total":95995035648,"available":95514529792,"internally_reserved":0,"allocated":158650368,"data_stored":158156465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":321847296},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":2.0830000000000002,"15min":2.359},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.23599999999999999},"max":{"1min":81.057000000000002,"5min":81.057000000000002,"15min":119.276},"last":0.26500000000000001},{"interface":"front","average":{"1min":4.96,"5min":2.0179999999999998,"15min":2.71},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.252},"max":{"1min":80.534999999999997,"5min":80.534999999999997,"15min":119.22},"last":0.52300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:23:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.4890000000000001,"15min":1.796},"min":{"1min":0.312,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.1180000000000003,"5min":8.8840000000000003,"15min":14.897},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.641,"15min":1.893},"min":{"1min":0.313,"5min":0.26000000000000001,"15min":0.251},"max":{"1min":7.5899999999999999,"5min":8.0299999999999994,"15min":10.766999999999999},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":3.016,"5min":1.694,"15min":0.93899999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.3369999999999997,"5min":10.125999999999999,"15min":10.125999999999999},"last":1.3380000000000001},{"interface":"front","average":{"1min":2.9220000000000002,"5min":1.5900000000000001,"15min":0.88800000000000001},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.699999999999999,"5min":13.699999999999999,"15min":13.699999999999999},"last":1.343}]},{"osd":3,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":14.476000000000001,"5min":5.056,"15min":2.0499999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":134.274,"5min":134.274,"15min":134.274},"last":0.434},{"interface":"front","average":{"1min":13.204000000000001,"5min":5.0140000000000002,"15min":2.0510000000000002},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":134.56999999999999,"5min":134.56999999999999,"15min":134.56999999999999},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:24:31 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.2,"15min":1.853},"min":{"1min":0.32100000000000001,"5min":0.27600000000000002,"15min":0.254},"max":{"1min":10.962999999999999,"5min":10.962999999999999,"15min":221.76900000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.302,"15min":2.0219999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.0609999999999999,"5min":10.221,"15min":221.905},"last":0.35699999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":4.4290000000000003,"15min":3.3650000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.25700000000000001},"max":{"1min":9.6069999999999993,"5min":198.154,"15min":313.661},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.8500000000000001,"5min":4.2779999999999996,"15min":3.387},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.6559999999999997,"5min":198.08199999999999,"15min":314.38900000000001},"last":1.5269999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":2.6600000000000001,"15min":1.617},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":9.6470000000000002,"5min":100.55500000000001,"15min":100.55500000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.4790000000000001,"5min":2.8570000000000002,"15min":1.675},"min":{"1min":0.436,"5min":0.34599999999999997,"15min":0.308},"max":{"1min":13.202999999999999,"5min":100.66,"15min":100.66},"last":0.39600000000000002}]}]},{"osd":1,"up_from":105,"seq":450971566205,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":810356,"kb_used_data":443372,"kb_used_omap":4,"kb_used_meta":366907,"kb_avail":92934796,"statfs":{"total":95995035648,"available":95165231104,"internally_reserved":0,"allocated":454012928,"data_stored":453356994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":375713678},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":12.198,"5min":4.4790000000000001,"15min":2.8239999999999998},"min":{"1min":0.29799999999999999,"5min":0.218,"15min":0.21299999999999999},"max":{"1min":80.486000000000004,"5min":86.009,"15min":86.009},"last":0.316},{"interface":"front","average":{"1min":13.145,"5min":5.0039999999999996,"15min":3.3889999999999998},"min":{"1min":0.40200000000000002,"5min":0.27800000000000002,"15min":0.22600000000000001},"max":{"1min":80.519999999999996,"5min":85.869,"15min":85.869},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:24:13 2023","interfaces":[{"interface":"back","average":{"1min":5.1749999999999998,"5min":2.29,"15min":1.0940000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":18.152000000000001,"5min":18.152000000000001,"15min":18.152000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":2.5470000000000002,"5min":1.4139999999999999,"15min":0.78400000000000003},"min":{"1min":0.222,"5min":0.218,"15min":0.218},"max":{"1min":8.984,"5min":8.984,"15min":8.984},"last":0.498}]},{"osd":3,"last update":"Sat Jun 10 21:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0129999999999999,"5min":4.1900000000000004,"15min":1.6359999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":16.553000000000001,"5min":118.348,"15min":118.348},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.9769999999999999,"5min":4.758,"15min":1.863},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":11.683,"5min":117.75,"15min":117.75},"last":0.43099999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":3.5800000000000001,"5min":1.647,"15min":1.9059999999999999},"min":{"1min":0.32200000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.507999999999999,"5min":12.507999999999999,"15min":12.94},"last":0.45300000000000001},{"interface":"front","average":{"1min":3.1280000000000001,"5min":1.49,"15min":2.3479999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.460000000000001,"5min":14.460000000000001,"15min":14.460000000000001},"last":3.3940000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":4.6760000000000002,"5min":2.0960000000000001,"15min":2.2400000000000002},"min":{"1min":0.435,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":14.811999999999999,"5min":14.811999999999999,"15min":14.811999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":3.0169999999999999,"5min":1.76,"15min":2.3399999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.5199999999999996,"5min":8.5199999999999996,"15min":13.602},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":4.3460000000000001,"5min":1.9399999999999999,"15min":2.2120000000000002},"min":{"1min":0.46899999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":14.858000000000001,"5min":14.858000000000001,"15min":14.858000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.8719999999999999,"5min":1.5449999999999999,"15min":2.2730000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.5510000000000002,"5min":8.5510000000000002,"15min":12.970000000000001},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":1.6910000000000001,"15min":2.7890000000000001},"min":{"1min":0.45000000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":8.4480000000000004,"5min":8.4480000000000004,"15min":13.025},"last":16.315999999999999},{"interface":"front","average":{"1min":3.609,"5min":1.621,"15min":2.0790000000000002},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":14.481,"5min":14.481,"15min":19.286999999999999},"last":2.431}]}]},{"osd":0,"up_from":86,"seq":369367187627,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":644232,"kb_used_data":220544,"kb_used_omap":5,"kb_used_meta":423674,"kb_avail":93100920,"statfs":{"total":95995035648,"available":95335342080,"internally_reserved":0,"allocated":225837056,"data_stored":225155912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":433842810},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:23:57 2023","interfaces":[{"interface":"back","average":{"1min":8.8870000000000005,"5min":4.7599999999999998,"15min":3.0710000000000002},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.19700000000000001},"max":{"1min":95.748000000000005,"5min":137.761,"15min":137.761},"last":0.39800000000000002},{"interface":"front","average":{"1min":7.6280000000000001,"5min":4.3300000000000001,"15min":3.6760000000000002},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":96.463999999999999,"5min":137.88999999999999,"15min":137.88999999999999},"last":1.4710000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9830000000000001,"5min":1.7969999999999999,"15min":0.82499999999999996},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.3109999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.552,"5min":1.6719999999999999,"15min":0.79500000000000004},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":10.391,"5min":16.782,"15min":16.782},"last":1.619}]},{"osd":3,"last update":"Sat Jun 10 21:24:35 2023","interfaces":[{"interface":"back","average":{"1min":14.766,"5min":8.7210000000000001,"15min":3.1749999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":181.886,"5min":181.886,"15min":181.886},"last":0.625},{"interface":"front","average":{"1min":16.021000000000001,"5min":9.4809999999999999,"15min":3.4180000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":180.04900000000001,"5min":180.04900000000001,"15min":180.04900000000001},"last":2.4830000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.6819999999999999,"15min":2.0800000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.6890000000000001,"5min":10.733000000000001,"15min":10.948},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.5289999999999999,"5min":1.613,"15min":1.8520000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":25.491,"5min":25.491,"15min":25.491},"last":0.71999999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":3.52,"5min":1.7769999999999999,"15min":2.0979999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":25.545000000000002,"5min":25.545000000000002,"15min":25.545000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.5310000000000001,"5min":1.8089999999999999,"15min":2.4199999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.32400000000000001},"max":{"1min":8.5879999999999992,"5min":8.5879999999999992,"15min":17.460000000000001},"last":1.5900000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":1.8200000000000001,"15min":2.1819999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":25.608000000000001,"5min":25.608000000000001,"15min":25.608000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":3.3620000000000001,"5min":2.0270000000000001,"15min":2.5539999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":10.231999999999999,"5min":10.231999999999999,"15min":17.411000000000001},"last":1.5269999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":1.9370000000000001,"15min":2.504},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.269,"5min":10.269,"15min":17.486999999999998},"last":1.5580000000000001},{"interface":"front","average":{"1min":2.214,"5min":1.857,"15min":2.1869999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":9.734,"5min":10.691000000000001,"15min":10.867000000000001},"last":0.502}]}]},{"osd":2,"up_from":120,"seq":515396075593,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":410752,"kb_used_data":217976,"kb_used_omap":7,"kb_used_meta":192760,"kb_avail":93334400,"statfs":{"total":95995035648,"available":95574425600,"internally_reserved":0,"allocated":223207424,"data_stored":222590678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":197386321},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":4.6440000000000001,"5min":2.1579999999999999,"15min":0.97099999999999997},"min":{"1min":0.27900000000000003,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":25.452999999999999,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.314},{"interface":"front","average":{"1min":3.879,"5min":2.633,"15min":1.121},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":25.053000000000001,"5min":62.451999999999998,"15min":62.451999999999998},"last":0.379}]},{"osd":1,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":4.282,"5min":1.9710000000000001,"15min":0.91400000000000003},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":17.718,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.9420000000000002,"5min":1.929,"15min":0.91800000000000004},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":16.641999999999999,"5min":28.013999999999999,"15min":28.013999999999999},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:24:31 2023","interfaces":[{"interface":"back","average":{"1min":2.7370000000000001,"5min":4.117,"15min":1.6379999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":14.644,"5min":96.295000000000002,"15min":96.295000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.931,"5min":4.7220000000000004,"15min":1.8580000000000001},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":14.792,"5min":99.504999999999995,"15min":99.504999999999995},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.4610000000000001,"15min":0.92400000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.8510000000000009,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.52700000000000002},{"interface":"front","average":{"1min":3.375,"5min":1.587,"15min":0.97299999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":17.295000000000002,"5min":17.295000000000002,"15min":17.295000000000002},"last":0.62}]},{"osd":5,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":1.623,"15min":0.998},"min":{"1min":0.38200000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":8.7880000000000003,"5min":10.254,"15min":10.254},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.8900000000000001,"5min":1.7390000000000001,"15min":1.028},"min":{"1min":0.40000000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.5449999999999999,"5min":9.5449999999999999,"15min":9.5449999999999999},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":2.597,"5min":1.6459999999999999,"15min":1.0149999999999999},"min":{"1min":0.436,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":7.4400000000000004,"5min":8.048,"15min":8.048},"last":0.58799999999999997},{"interface":"front","average":{"1min":3.8900000000000001,"5min":1.857,"15min":1.099},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":17.244,"5min":17.244,"15min":17.244},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":1.3080000000000001,"15min":0.92600000000000005},"min":{"1min":0.40999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.8350000000000009,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.952,"5min":1.6879999999999999,"15min":1.0469999999999999},"min":{"1min":0.47599999999999998,"5min":0.434,"15min":0.434},"max":{"1min":8.8140000000000001,"5min":8.8140000000000001,"15min":8.8140000000000001},"last":0.68000000000000005}]}]},{"osd":3,"up_from":125,"seq":536870912068,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617816,"kb_used_data":395280,"kb_used_omap":2,"kb_used_meta":222525,"kb_avail":93127336,"statfs":{"total":95995035648,"available":95362392064,"internally_reserved":0,"allocated":404766720,"data_stored":404134866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":227865653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.956,"15min":0.89300000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":9.7159999999999993,"5min":46.677999999999997,"15min":46.677999999999997},"last":1.8049999999999999},{"interface":"front","average":{"1min":9.6199999999999992,"5min":3.7799999999999998,"15min":1.496},"min":{"1min":0.21199999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":130.893,"5min":130.893,"15min":130.893},"last":6.7830000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":2.4340000000000002,"15min":1.0800000000000001},"min":{"1min":0.248,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.2439999999999998,"5min":54.112000000000002,"15min":54.112000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.9470000000000001,"5min":2.9100000000000001,"15min":1.246},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.0020000000000007,"5min":57.814,"15min":57.814},"last":6.843}]},{"osd":2,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":1.861,"15min":0.88},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":10.007,"5min":12.726000000000001,"15min":12.726000000000001},"last":6.9299999999999997},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.5740000000000001,"15min":0.78600000000000003},"min":{"1min":0.32000000000000001,"5min":0.249,"15min":0.249},"max":{"1min":4.6189999999999998,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":1.7749999999999999,"15min":0.97499999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":13.656000000000001,"5min":13.656000000000001,"15min":13.656000000000001},"last":6.7510000000000003},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.6319999999999999,"15min":0.92600000000000005},"min":{"1min":0.38200000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.4829999999999997,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":6.0069999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":28.803000000000001,"5min":8.6699999999999999,"15min":3.2879999999999998},"min":{"1min":0.30199999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":349.47399999999999,"5min":349.47399999999999,"15min":349.47399999999999},"last":6.8159999999999998},{"interface":"front","average":{"1min":21.940999999999999,"5min":7.5679999999999996,"15min":2.9289999999999998},"min":{"1min":0.47799999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":349.65100000000001,"5min":349.65100000000001,"15min":349.65100000000001},"last":5.8810000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":2.0539999999999998,"15min":1.079},"min":{"1min":0.47699999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.5469999999999997,"5min":9.9619999999999997,"15min":9.9619999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.8320000000000001,"15min":1.054},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":7.9960000000000004,"5min":12.835000000000001,"15min":12.835000000000001},"last":0.40699999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:24:43 2023","interfaces":[{"interface":"back","average":{"1min":9.1609999999999996,"5min":4.9989999999999997,"15min":2.0739999999999998},"min":{"1min":0.45400000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":130.852,"5min":174.44800000000001,"15min":174.44800000000001},"last":6.0410000000000004},{"interface":"front","average":{"1min":1.6459999999999999,"5min":3.4649999999999999,"15min":1.603},"min":{"1min":0.38400000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9450000000000003,"5min":173.18100000000001,"15min":173.18100000000001},"last":4.7480000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019218,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":526272,"kb_used_data":158712,"kb_used_omap":0,"kb_used_meta":367552,"kb_avail":93218880,"statfs":{"total":95995035648,"available":95456133120,"internally_reserved":0,"allocated":162521088,"data_stored":162022621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":376373248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.5349999999999999,"5min":2.0310000000000001,"15min":1.978},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.22},"max":{"1min":6.4880000000000004,"5min":40.186999999999998,"15min":40.186999999999998},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.351,"15min":2.3090000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.23100000000000001},"max":{"1min":9.8170000000000002,"5min":39.395000000000003,"15min":39.395000000000003},"last":0.49099999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:24:03 2023","interfaces":[{"interface":"back","average":{"1min":6.1970000000000001,"5min":3.5369999999999999,"15min":2.4609999999999999},"min":{"1min":0.33900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":45.857999999999997,"5min":79.731999999999999,"15min":79.731999999999999},"last":1.821},{"interface":"front","average":{"1min":5.6150000000000002,"5min":3.0880000000000001,"15min":2.6539999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27000000000000002},"max":{"1min":45.822000000000003,"5min":79.146000000000001,"15min":79.146000000000001},"last":4.3470000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":1.147,"15min":0.73299999999999998},"min":{"1min":0.35399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":9.9610000000000003,"5min":9.9610000000000003,"15min":9.9610000000000003},"last":0.94099999999999995},{"interface":"front","average":{"1min":2.0350000000000001,"5min":1.417,"15min":0.89100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.0299999999999994,"5min":11.099,"15min":11.099},"last":1.835}]},{"osd":3,"last update":"Sat Jun 10 21:24:37 2023","interfaces":[{"interface":"back","average":{"1min":9.8599999999999994,"5min":6.1239999999999997,"15min":2.4119999999999999},"min":{"1min":0.40699999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":103.575,"5min":103.575,"15min":103.575},"last":0.67200000000000004},{"interface":"front","average":{"1min":10.09,"5min":5.7060000000000004,"15min":2.3159999999999998},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":103.565,"5min":103.565,"15min":103.565},"last":0.65100000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.4350000000000001,"15min":1.4430000000000001},"min":{"1min":0.312,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":6.3650000000000002,"5min":10.984999999999999,"15min":92.653000000000006},"last":1.7969999999999999},{"interface":"front","average":{"1min":1.768,"5min":1.2909999999999999,"15min":1.542},"min":{"1min":0.28799999999999998,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":6.6239999999999997,"5min":8.3659999999999997,"15min":93.281999999999996},"last":0.29199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.76,"5min":1.4950000000000001,"15min":1.167},"min":{"1min":0.311,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":6.4610000000000003,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.9790000000000001,"5min":1.4710000000000001,"15min":1.1519999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":7.6200000000000001,"5min":11.055,"15min":11.055},"last":1.8500000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:24:53 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":1.855,"15min":1.401},"min":{"1min":0.35799999999999998,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":12.538,"5min":12.538,"15min":12.538},"last":1.865},{"interface":"front","average":{"1min":2.105,"5min":1.77,"15min":1.2629999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.312},"max":{"1min":6.5099999999999998,"5min":9.5060000000000002,"15min":9.5060000000000002},"last":2.5920000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822990,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972788,"kb_used_data":468396,"kb_used_omap":0,"kb_used_meta":504192,"kb_avail":92772364,"statfs":{"total":95995035648,"available":94998900736,"internally_reserved":0,"allocated":479637504,"data_stored":479147111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":516292608},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":10.728,"5min":3.8780000000000001,"15min":2.1360000000000001},"min":{"1min":0.28399999999999997,"5min":0.246,"15min":0.23300000000000001},"max":{"1min":148.447,"5min":148.447,"15min":148.447},"last":0.42299999999999999},{"interface":"front","average":{"1min":10.789999999999999,"5min":3.827,"15min":2.7829999999999999},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":148.42099999999999,"5min":148.42099999999999,"15min":148.42099999999999},"last":3.9460000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:24:00 2023","interfaces":[{"interface":"back","average":{"1min":22.015999999999998,"5min":6.3099999999999996,"15min":3.5430000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24299999999999999},"max":{"1min":149.786,"5min":149.786,"15min":149.786},"last":0.46000000000000002},{"interface":"front","average":{"1min":22.021000000000001,"5min":5.9699999999999998,"15min":3.8500000000000001},"min":{"1min":0.31,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":149.52500000000001,"5min":149.52500000000001,"15min":149.52500000000001},"last":0.435}]},{"osd":2,"last update":"Sat Jun 10 21:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.2569999999999999,"15min":0.82299999999999995},"min":{"1min":0.312,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.465999999999999,"5min":11.465999999999999,"15min":11.465999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.4730000000000001,"15min":0.89500000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.7850000000000001,"5min":10.895,"15min":10.895},"last":3.9860000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:24:34 2023","interfaces":[{"interface":"back","average":{"1min":21.151,"5min":9.7769999999999992,"15min":3.669},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":109.96599999999999,"5min":179.10300000000001,"15min":179.10300000000001},"last":4.0119999999999996},{"interface":"front","average":{"1min":21.884,"5min":9.6929999999999996,"15min":3.589},"min":{"1min":0.41799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":110.3,"5min":181.44300000000001,"15min":181.44300000000001},"last":0.64300000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1100000000000001,"15min":1.0249999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25900000000000001},"max":{"1min":3.2229999999999999,"5min":11.039999999999999,"15min":11.039999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80700000000000005,"15min":1.117},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.255},"max":{"1min":3.0739999999999998,"5min":5.016,"15min":12.901},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:24:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.381,"15min":1.198},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.8759999999999994,"5min":10.991,"15min":10.991},"last":10.183},{"interface":"front","average":{"1min":1.089,"5min":1.141,"15min":1.198},"min":{"1min":0.378,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.3540000000000001,"5min":8.0429999999999993,"15min":8.0429999999999993},"last":10.119999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.274,"15min":1.3260000000000001},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":4.3890000000000002,"5min":8.7360000000000007,"15min":8.7360000000000007},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.9369999999999998,"5min":2.0270000000000001,"15min":1.4239999999999999},"min":{"1min":0.35699999999999998,"5min":0.35299999999999998,"15min":0.318},"max":{"1min":12.414,"5min":13.047000000000001,"15min":13.047000000000001},"last":0.61899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":222429184,"data_stored":222429184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":431099904,"data_stored":431095832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":217251840,"data_stored":217251840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":399638528,"data_stored":399638528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":154402816,"data_stored":154402816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":458297344,"data_stored":458293272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":154927104,"data_stored":154927104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":514068480,"data_stored":514064408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:24:59.150 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:24:59.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_4 pg_num 26 2023-06-10T21:24:59.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:24:59.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:59 smithi152 ceph-mon[156819]: pgmap v690: 89 pgs: 89 active+clean; 769 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:24:59.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:24:59 smithi152 ceph-mon[156819]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:59.279 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:24:59.280 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:24:59.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.280 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:208614 9.1 e182/127 9:86a48042:::benchmark_data_smithi064_247471_object99626:head v 182'13075, mlcod=182'13072)", 2023-06-10T21:24:59.280 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:59.266641+0000", 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0077236620000000001, 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077469319999999998, 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:59.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266641+0000", 2023-06-10T21:24:59.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:59.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266641+0000", 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:59.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266640+0000", 2023-06-10T21:24:59.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1613e-05 2023-06-10T21:24:59.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266651+0000", 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9100999999999999e-05 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266670+0000", 2023-06-10T21:24:59.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0309999999999998e-06 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266678+0000", 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6814999999999997e-05 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266735+0000", 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5494e-05 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:59.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266761+0000", 2023-06-10T21:24:59.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001198 2023-06-10T21:24:59.289 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.289 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:208617 9.1 e182/127 9:87022289:::benchmark_data_smithi064_247471_object99629:head v 182'13076, mlcod=182'13072)", 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:59.266695+0000", 2023-06-10T21:24:59.290 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.007669871, 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077660029999999996, 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:59.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266695+0000", 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266695+0000", 2023-06-10T21:24:59.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:24:59.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:59.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266694+0000", 2023-06-10T21:24:59.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8470000000000003e-06 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266699+0000", 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4620000000000001e-06 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:59.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266706+0000", 2023-06-10T21:24:59.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6720000000000003e-06 2023-06-10T21:24:59.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:59.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266710+0000", 2023-06-10T21:24:59.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053328589999999997 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272043+0000", 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0066999999999999e-05 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272073+0000", 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053786140000000003 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:59.298 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.299 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:208623 9.0 9:073137e5:::benchmark_data_smithi064_247471_object99635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-06-10T21:24:59.299 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:59.266789+0000", 2023-06-10T21:24:59.300 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0075758780000000003, 2023-06-10T21:24:59.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077176329999999998, 2023-06-10T21:24:59.300 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:59.300 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:24:59.300 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 208623 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:59.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:59.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266789+0000", 2023-06-10T21:24:59.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:59.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:59.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266789+0000", 2023-06-10T21:24:59.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:24:59.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266787+0000", 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2790000000000004e-06 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:59.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266796+0000", 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2322e-05 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266808+0000", 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0079999999999996e-06 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266815+0000", 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8866000000000003e-05 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:24:59.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266864+0000", 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116249 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.266981+0000", 2023-06-10T21:24:59.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012738800000000001 2023-06-10T21:24:59.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:24:59.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.267108+0000", 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051246699999999996 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272233+0000", 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054438869999999997 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.309 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:208618 9.1 e182/127)", 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:59.272851+0000", 2023-06-10T21:24:59.310 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0015140279999999999, 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017095260000000001, 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272851+0000", 2023-06-10T21:24:59.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272851+0000", 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272850+0000", 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.01e-05 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:59.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272860+0000", 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272872+0000", 2023-06-10T21:24:59.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8390000000000001e-06 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.272879+0000", 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.862e-05 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:59.315 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.316 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:208630 9.0 9.a47a4750 (undecoded) ondisk+write+known_if_redirected e182)", 2023-06-10T21:24:59.316 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:59.274412+0000", 2023-06-10T21:24:59.316 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9999523, 2023-06-10T21:24:59.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017787900000000001, 2023-06-10T21:24:59.317 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:24:59.317 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:24:59.317 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:24:59.318 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:24:59.318 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:24:59.318 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 208630 2023-06-10T21:24:59.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.318 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:24:59.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:24:59.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274412+0000", 2023-06-10T21:24:59.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:24:59.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:24:59.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274412+0000", 2023-06-10T21:24:59.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:24:59.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:24:59.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274410+0000", 2023-06-10T21:24:59.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3610000000000007e-06 2023-06-10T21:24:59.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:24:59.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274420+0000", 2023-06-10T21:24:59.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9451000000000001e-05 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274439+0000", 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0490000000000004e-06 2023-06-10T21:24:59.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:24:59.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:24:59.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:24:59.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:59.274447+0000", 2023-06-10T21:24:59.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5052000000000003e-05 2023-06-10T21:24:59.325 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.325 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:24:59.325 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.325 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:24:59.325 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:24:59.326 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 5 2023-06-10T21:24:59.326 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:24:59.326 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:59.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:59 smithi064 ceph-mon[180297]: pgmap v690: 89 pgs: 89 active+clean; 769 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:24:59.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:24:59 smithi064 ceph-mon[180297]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:59 smithi064 ceph-mon[186706]: pgmap v690: 89 pgs: 89 active+clean; 769 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 475 op/s; 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:24:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:24:59 smithi064 ceph-mon[186706]: from='client.35664 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:24:59.491 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:24:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:59.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:24:59.568 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:24:59.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:24:59.680 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:24:59.874 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:24:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:24:59.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:24:59.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:00.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:00.146 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:00.168 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:25:00.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:00.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:00.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:00.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:00.636 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:25:01.004 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:01.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:01.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:01.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:01.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:01.260 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:01.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:01 smithi064 ceph-mon[180297]: pgmap v691: 89 pgs: 89 active+clean; 700 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:25:01.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:01 smithi064 ceph-mon[186706]: pgmap v691: 89 pgs: 89 active+clean; 700 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:25:01.347 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:01.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:01.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:01.481 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:01.522 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:01.523 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:01 smithi152 ceph-mon[156819]: pgmap v691: 89 pgs: 89 active+clean; 700 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:25:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:01.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:01.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:01.703 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:01.917 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:01.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:02.025 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:02.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:02.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:02.122 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:02.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:02.189 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:02.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/935743551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:02.274 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:02.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:02.297 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:02.297 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:02.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:02.340 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.344 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.345 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:25:02.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:02.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:02.589 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:25:02.590 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:25:02.593 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:02.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:02.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:02.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/935743551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/935743551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:02.702 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:02.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:02] "GET /metrics HTTP/1.1" 200 223674 "" "Prometheus/2.33.4" 2023-06-10T21:25:03.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:03.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:03.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:03.464 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:03.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:03 smithi152 ceph-mon[156819]: pgmap v692: 89 pgs: 89 active+clean; 611 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:25:03.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:03.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:03.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:25:03.604 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:03.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:03.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[180297]: pgmap v692: 89 pgs: 89 active+clean; 611 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:25:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:03.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[186706]: pgmap v692: 89 pgs: 89 active+clean; 611 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:25:03.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:03.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:03.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:03.762 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:04.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:04.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:04.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:04.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:04.270 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:25:04.311 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:04.409 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:04.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:04.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:04.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:04.563 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:04 smithi152 ceph-mon[156819]: pgmap v693: 89 pgs: 89 active+clean; 525 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:25:04.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:04.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:04.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:04 smithi064 ceph-mon[180297]: pgmap v693: 89 pgs: 89 active+clean; 525 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:25:04.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:04.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:04 smithi064 ceph-mon[186706]: pgmap v693: 89 pgs: 89 active+clean; 525 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-06-10T21:25:04.785 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:04.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:04.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:04.898 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:04.949 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:05.042 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:05.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:05.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:05.200 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:05.312 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:05.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:05.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:05.510 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 9 pg_num to 26 2023-06-10T21:25:05.516 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3612660804' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.516 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:05 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.516 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:05.516 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:05.522 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:25:05.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3612660804' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:05.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3612660804' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:25:05.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:05.888 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:05.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:05.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:06.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:06.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:25:06.305 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:25:06.306 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:25:06.382 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:06.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:06.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:06.560 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:06.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:06.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:25:06.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: osdmap e183: 8 total, 8 up, 8 in 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: pgmap v695: 89 pgs: 89 active+clean; 409 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:25:06.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:25:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:25:06.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: osdmap e183: 8 total, 8 up, 8 in 2023-06-10T21:25:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: pgmap v695: 89 pgs: 89 active+clean; 409 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:25:06.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2127133538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: osdmap e183: 8 total, 8 up, 8 in 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: failed to encode map e183 with expected crc 2023-06-10T21:25:06.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: pgmap v695: 89 pgs: 89 active+clean; 409 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-10T21:25:06.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:25:07.158 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:07.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:07.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:07.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:25:07.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:07.281 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:07.324 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:07.367 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:07.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:07.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:07.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:07.523 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:07.684 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[186706]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:25:07.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[186706]: osdmap e184: 8 total, 8 up, 8 in 2023-06-10T21:25:07.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[186706]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.685 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[186706]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.685 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[180297]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:25:07.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[180297]: osdmap e184: 8 total, 8 up, 8 in 2023-06-10T21:25:07.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[180297]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.686 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:07 smithi064 ceph-mon[180297]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:07.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:07.874 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:25:07.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:07 smithi152 ceph-mon[156819]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:25:07.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:07 smithi152 ceph-mon[156819]: osdmap e184: 8 total, 8 up, 8 in 2023-06-10T21:25:07.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:07 smithi152 ceph-mon[156819]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:07 smithi152 ceph-mon[156819]: failed to encode map e184 with expected crc 2023-06-10T21:25:07.995 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:08.052 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:08.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:08.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:08.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:08.200 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:08.347 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:08.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:08.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:08.527 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:08.619 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: osdmap e185: 8 total, 8 up, 8 in 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.623 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:08 smithi152 ceph-mon[156819]: pgmap v698: 99 pgs: 3 peering, 10 unknown, 86 active+clean; 362 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:25:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:08.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:08.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:08.784 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:08.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: osdmap e185: 8 total, 8 up, 8 in 2023-06-10T21:25:08.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[186706]: pgmap v698: 99 pgs: 3 peering, 10 unknown, 86 active+clean; 362 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: osdmap e185: 8 total, 8 up, 8 in 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: failed to encode map e185 with expected crc 2023-06-10T21:25:08.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:08 smithi064 ceph-mon[180297]: pgmap v698: 99 pgs: 3 peering, 10 unknown, 86 active+clean; 362 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-10T21:25:08.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:08.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:09.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:09.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:09.068 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:09.104 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:09.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:09.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:09.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:09.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:25:09.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:09.228 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:09.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:09.266 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:09.292 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:09.304 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:09.355 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.360 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.361 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.362 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:25:09.472 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:09.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:09.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:09.562 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/821034203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:09.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:09.644 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:09.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/821034203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:09.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/821034203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:09.798 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:09.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:09.902 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:09.933 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:10.020 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:25:10.020 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:25:10.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.020 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:215179 9.d 9:bebce6e5:::benchmark_data_smithi064_247471_object106191:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.003445+0000", 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0027207899999999998, 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027540059999999998, 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:10.021 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215179 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003445+0000", 2023-06-10T21:25:10.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003445+0000", 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003443+0000", 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4789999999999996e-06 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003452+0000", 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4277000000000001e-05 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003466+0000", 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5830000000000001e-06 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003474+0000", 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1607000000000002e-05 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003525+0000", 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003641+0000", 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016223400000000001 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003803+0000", 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035837 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:215173 9.e e185/127)", 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.003508+0000", 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0026581310000000002, 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027746789999999999, 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:25:10.029 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003508+0000", 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003508+0000", 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003507+0000", 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6410000000000001e-06 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003512+0000", 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8280000000000003e-06 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003520+0000", 2023-06-10T21:25:10.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4070000000000001e-06 2023-06-10T21:25:10.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003525+0000", 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7244e-05 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215176 9.c e185/127 9:3efe3622:::benchmark_data_smithi064_247471_object106188:head v 185'13437, mlcod=185'13434)", 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.003555+0000", 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0026106810000000001, 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027807719999999999, 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.035 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003555+0000", 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003555+0000", 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003555+0000", 2023-06-10T21:25:10.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2389999999999999e-06 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003561+0000", 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8290000000000002e-06 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003568+0000", 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7430000000000002e-06 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003573+0000", 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012026799999999999 2023-06-10T21:25:10.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003693+0000", 2023-06-10T21:25:10.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8521000000000001e-05 2023-06-10T21:25:10.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003711+0000", 2023-06-10T21:25:10.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000156129 2023-06-10T21:25:10.042 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.042 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.042 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.043 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215180 9.1 e185/184 9:82384130:::benchmark_data_smithi064_247471_object106192:head v 185'13427, mlcod=185'13422)", 2023-06-10T21:25:10.043 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.003583+0000", 2023-06-10T21:25:10.044 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0025833739999999998, 2023-06-10T21:25:10.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028176009999999999, 2023-06-10T21:25:10.045 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.045 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.045 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003583+0000", 2023-06-10T21:25:10.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003583+0000", 2023-06-10T21:25:10.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003582+0000", 2023-06-10T21:25:10.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9299999999999996e-06 2023-06-10T21:25:10.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003586+0000", 2023-06-10T21:25:10.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4759999999999997e-06 2023-06-10T21:25:10.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003592+0000", 2023-06-10T21:25:10.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8800000000000001e-06 2023-06-10T21:25:10.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003595+0000", 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016786899999999999 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003763+0000", 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1573000000000002e-05 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.003785+0000", 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00020241899999999999 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215181 9.9 e185/184 9:92be5557:::benchmark_data_smithi064_247471_object106193:head v 185'13306, mlcod=185'13304)", 2023-06-10T21:25:10.054 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.004352+0000", 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0018141609999999999, 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021123299999999999, 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004352+0000", 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004352+0000", 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:10.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004351+0000", 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4290000000000001e-06 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004359+0000", 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1296e-05 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004371+0000", 2023-06-10T21:25:10.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9290000000000003e-06 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004377+0000", 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00091330000000000003 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.005290+0000", 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8881000000000001e-05 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.005309+0000", 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00095697200000000005 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.061 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:215161 9.e e185/127)", 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.004388+0000", 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001777673, 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021354210000000002, 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004388+0000", 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004388+0000", 2023-06-10T21:25:10.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004388+0000", 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.36e-06 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004391+0000", 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2580000000000001e-06 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004397+0000", 2023-06-10T21:25:10.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6559999999999998e-06 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004401+0000", 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2864000000000001e-05 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.066 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:215163 9.e e185/127)", 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:10.004420+0000", 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0017459559999999999, 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021517440000000001, 2023-06-10T21:25:10.067 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004420+0000", 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004420+0000", 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004420+0000", 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.373e-06 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004422+0000", 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9919999999999998e-06 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004427+0000", 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4350000000000001e-06 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.004430+0000", 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0521e-05 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.072 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:215161 9.e 9:786d90ac:::benchmark_data_smithi064_247471_object106173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.984030+0000", 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.022135403000000001, 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022595186999999999, 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:10.073 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:10.074 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:10.074 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215161 2023-06-10T21:25:10.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.075 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984030+0000", 2023-06-10T21:25:10.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984030+0000", 2023-06-10T21:25:10.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:25:10.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984029+0000", 2023-06-10T21:25:10.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8839999999999997e-06 2023-06-10T21:25:10.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984036+0000", 2023-06-10T21:25:10.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5296000000000001e-05 2023-06-10T21:25:10.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984051+0000", 2023-06-10T21:25:10.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4679999999999997e-06 2023-06-10T21:25:10.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984058+0000", 2023-06-10T21:25:10.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4305e-05 2023-06-10T21:25:10.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984142+0000", 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011024299999999999 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984252+0000", 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000139178 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984391+0000", 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011515193999999999 2023-06-10T21:25:10.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.995907+0000", 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.613e-06 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:10.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.995913+0000", 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011882681000000001 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.086 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:215163 9.e 9:76da8a9a:::benchmark_data_smithi064_247471_object106175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.984069+0000", 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.022096543, 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022648524, 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:10.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215163 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984069+0000", 2023-06-10T21:25:10.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984069+0000", 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984069+0000", 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0580000000000002e-06 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984072+0000", 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4530000000000002e-06 2023-06-10T21:25:10.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984080+0000", 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.109e-06 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984084+0000", 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003706127 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.987791+0000", 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123325 2023-06-10T21:25:10.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.987914+0000", 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014990000000000001 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:10.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.988064+0000", 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078544530000000008 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.995918+0000", 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2557e-05 2023-06-10T21:25:10.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.995951+0000", 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011881472000000001 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:215165 9.e 9:7397583a:::benchmark_data_smithi064_247471_object106177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.984088+0000", 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.022077505000000001, 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022718044999999999, 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:25:10.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215165 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984088+0000", 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984088+0000", 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984088+0000", 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4329999999999998e-06 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984090+0000", 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0869999999999997e-06 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984096+0000", 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3799999999999998e-06 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.984099+0000", 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077175949999999998 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.991817+0000", 2023-06-10T21:25:10.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115157 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.991932+0000", 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013029 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.992062+0000", 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074278929999999996 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999490+0000", 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052950000000000002 2023-06-10T21:25:10.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.000019+0000", 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015930988 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.104 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215168 9.1 e185/184 9:8566f874:::benchmark_data_smithi064_247471_object106180:head v 185'13424, mlcod=185'13422)", 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.994109+0000", 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.012057338000000001, 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012788351999999999, 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.105 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994109+0000", 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994109+0000", 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994108+0000", 2023-06-10T21:25:10.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5519999999999993e-06 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994117+0000", 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2463000000000002e-05 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994140+0000", 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0140000000000001e-06 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994147+0000", 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00028185299999999999 2023-06-10T21:25:10.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994429+0000", 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0074000000000001e-05 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994449+0000", 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034019399999999999 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.111 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215170 9.1 e185/184 9:80ee8b06:::benchmark_data_smithi064_247471_object106182:head v 185'13425, mlcod=185'13422)", 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.994159+0000", 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.012007122, 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012808952, 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994159+0000", 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994159+0000", 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994158+0000", 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.057e-06 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994163+0000", 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5100000000000004e-06 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994170+0000", 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2119999999999997e-06 2023-06-10T21:25:10.115 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994174+0000", 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022811680000000001 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.116 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.996455+0000", 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1611e-05 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.996477+0000", 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023181299999999998 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.117 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215172 9.1 e185/184 9:86c132ec:::benchmark_data_smithi064_247471_object106184:head v 185'13426, mlcod=185'13422)", 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.994179+0000", 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011987279, 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012859061, 2023-06-10T21:25:10.118 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994179+0000", 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994179+0000", 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994178+0000", 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6129999999999998e-06 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994186+0000", 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5149999999999997e-06 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994191+0000", 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5659999999999998e-06 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994195+0000", 2023-06-10T21:25:10.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040330000000000001 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.998228+0000", 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7788999999999999e-05 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.998246+0000", 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040672290000000003 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:215173 9.e 9:74f3ede1:::benchmark_data_smithi064_247471_object106185:head [delete] snapc 0=[] ondisk+write+known_if_redirected e185)", 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.994585+0000", 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.011581091999999999, 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012527229000000001, 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:10.125 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 215173 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994585+0000", 2023-06-10T21:25:10.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994585+0000", 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.127 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994584+0000", 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6700000000000002e-06 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994589+0000", 2023-06-10T21:25:10.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1269999999999992e-06 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994598+0000", 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3410000000000002e-06 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.994603+0000", 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013743679999999999 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.995977+0000", 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105531 2023-06-10T21:25:10.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.996083+0000", 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114838 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:10.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.996198+0000", 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016130210000000001 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.132 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215174 9.c e185/127 9:3582315c:::benchmark_data_smithi064_247471_object106186:head v 185'13436, mlcod=185'13434)", 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.999346+0000", 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0068201149999999999, 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078458710000000008, 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999346+0000", 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999346+0000", 2023-06-10T21:25:10.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999346+0000", 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7060000000000003e-06 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999350+0000", 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999355+0000", 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.303e-06 2023-06-10T21:25:10.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999359+0000", 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069065999999999995 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.000049+0000", 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7853999999999999e-05 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:10.000067+0000", 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00072127699999999999 2023-06-10T21:25:10.138 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:215175 9.a e185/127 9:5d067955:::benchmark_data_smithi064_247471_object106187:head v 185'13483, mlcod=185'13481)", 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:09.999381+0000", 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0067849429999999999, 2023-06-10T21:25:10.139 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079002440000000007, 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999381+0000", 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:10.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999381+0000", 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:10.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999381+0000", 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5740000000000003e-06 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999386+0000", 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1709999999999999e-06 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999392+0000", 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.32e-06 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999396+0000", 2023-06-10T21:25:10.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6625999999999998e-05 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999432+0000", 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0394000000000002e-05 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:10.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:09.999453+0000", 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1701999999999999e-05 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:10.145 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:25:10.146 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 16 2023-06-10T21:25:10.146 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:25:10.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:10.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:10.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:10.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:10.491 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:10.530 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:10.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:10.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:10.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:10 smithi064 ceph-mon[180297]: pgmap v699: 99 pgs: 3 peering, 6 unknown, 90 active+clean; 309 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:10 smithi064 ceph-mon[186706]: pgmap v699: 99 pgs: 3 peering, 6 unknown, 90 active+clean; 309 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:10.705 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:10.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:10 smithi152 ceph-mon[156819]: pgmap v699: 99 pgs: 3 peering, 6 unknown, 90 active+clean; 309 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:10.981 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:10.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:25:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:10.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:11.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:11.293 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:11.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:11.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:11.479 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:11.587 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:11.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:11.663 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:25:11.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:11.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:11.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:11.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:11.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:11.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:12.044 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:12.193 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:25:12.193 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:25:12.194 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2023-06-10T21:25:12.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 2 2023-06-10T21:25:12.257 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:12.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:12.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:12.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:12.394 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:25:12.421 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:12.509 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:12.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:12.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:12.641 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:12.683 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:12.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:12.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:12.758 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:12.787 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:12.811 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:12 smithi152 ceph-mon[156819]: pgmap v700: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 228 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:12.813 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:12 smithi064 ceph-mon[180297]: pgmap v700: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 228 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:12.884 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:12] "GET /metrics HTTP/1.1" 200 223756 "" "Prometheus/2.33.4" 2023-06-10T21:25:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:12 smithi064 ceph-mon[186706]: pgmap v700: 99 pgs: 3 peering, 3 unknown, 93 active+clean; 228 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 476 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:12.972 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:13.112 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:13.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:13.171 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 108985 objects 2023-06-10T21:25:13.172 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :170.588 2023-06-10T21:25:13.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:13.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:13.333 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:13.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:13.435 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:13.472 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:13.638 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:25:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:25:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:25:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:25:13.639 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:25:13.641 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:25:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:13 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:25:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:13.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:13 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:25:13.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:13.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:13 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:25:14.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:14.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:14.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:14.263 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:14.368 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:14.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:14.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:25:14.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:14.499 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:14.543 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:14.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:14.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:14.677 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:14.722 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:14.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:14.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:14.792 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-06-10T21:25:14.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-06-10T21:25:14.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:14.842 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:25:14.863 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:14.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: pgmap v701: 99 pgs: 99 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:14.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[180297]: osdmap e186: 8 total, 8 up, 8 in 2023-06-10T21:25:14.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: pgmap v701: 99 pgs: 99 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:25:14.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:14 smithi064 ceph-mon[186706]: osdmap e186: 8 total, 8 up, 8 in 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: pgmap v701: 99 pgs: 99 active+clean; 121 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: failed to encode map e186 with expected crc 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4164776335' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:25:14.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:14 smithi152 ceph-mon[156819]: osdmap e186: 8 total, 8 up, 8 in 2023-06-10T21:25:15.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:15.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:15.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:15.193 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:15.237 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:15.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:15.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:15.368 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:15.413 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:15.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:15.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:15.543 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:15.841 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:15.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: osdmap e187: 8 total, 8 up, 8 in 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: 9.8 scrub starts 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: 9.8 scrub ok 2023-06-10T21:25:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:25:15.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: osdmap e187: 8 total, 8 up, 8 in 2023-06-10T21:25:15.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: 9.8 scrub starts 2023-06-10T21:25:15.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: 9.8 scrub ok 2023-06-10T21:25:15.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:25:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: osdmap e187: 8 total, 8 up, 8 in 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: 9.8 scrub starts 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: 9.8 scrub ok 2023-06-10T21:25:15.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:25:15.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:15.974 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:16.069 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:16.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:16.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:16.197 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:16.202 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:16.245 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:16.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:16.295 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:16.307 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:25:16.307 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:17364 9.3 9:c62284ab:::benchmark_data_smithi064_247471_object17363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 257.85922912799998, 2023-06-10T21:25:16.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28749911299999997, 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17364 2023-06-10T21:25:16.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443109+0000", 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443108+0000", 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0887999999999997e-05 2023-06-10T21:25:16.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443168+0000", 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3409e-05 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443182+0000", 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0399999999999998e-06 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443188+0000", 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2876e-05 2023-06-10T21:25:16.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443241+0000", 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016888800000000001 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443410+0000", 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017182300000000001 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443582+0000", 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096170299999999995 2023-06-10T21:25:16.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.444543+0000", 2023-06-10T21:25:16.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000265596 2023-06-10T21:25:16.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.444809+0000", 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28572154900000002 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730530+0000", 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4171e-05 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730585+0000", 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3592e-05 2023-06-10T21:25:16.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730608+0000", 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28749911299999997 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.319 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:17367 9.d 9:bff3de3b:::benchmark_data_smithi064_247471_object17366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 257.85913673200002, 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28726967199999998, 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.320 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 17367 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443201+0000", 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9278999999999999e-05 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443230+0000", 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4099999999999997e-06 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443239+0000", 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2370000000000003e-06 2023-06-10T21:25:16.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443244+0000", 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3118999999999997e-05 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443297+0000", 2023-06-10T21:25:16.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012421000000000001 2023-06-10T21:25:16.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443421+0000", 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015397699999999999 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.443575+0000", 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015038860000000001 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.445079+0000", 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00056494599999999998 2023-06-10T21:25:16.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.445644+0000", 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28475432899999997 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730398+0000", 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8642e-05 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730437+0000", 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4273000000000002e-05 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:58.730471+0000", 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28726967199999998 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20066 9.d 9:b754690b:::benchmark_data_smithi064_247471_object20065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31846249699998, 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999, 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.332 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20066 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983875+0000", 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019127799999999999 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984066+0000", 2023-06-10T21:25:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2920000000000001e-06 2023-06-10T21:25:16.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984075+0000", 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9660000000000001e-06 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984081+0000", 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8265999999999999e-05 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984159+0000", 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121333 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984281+0000", 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144637 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984425+0000", 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014625199999999999 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985888+0000", 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024362569999999998 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988324+0000", 2023-06-10T21:25:16.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29050814800000002 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278832+0000", 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5896999999999998e-05 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278878+0000", 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7633999999999999e-05 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278916+0000", 2023-06-10T21:25:16.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20069 9.3 9:c33031bf:::benchmark_data_smithi064_247471_object20068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:25:16.343 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31799484599998, 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999, 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20069 2023-06-10T21:25:16.344 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984342+0000", 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012432649999999999 2023-06-10T21:25:16.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985585+0000", 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6175e-05 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985602+0000", 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8390000000000001e-06 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985609+0000", 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1567999999999998e-05 2023-06-10T21:25:16.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985660+0000", 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013685800000000001 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985797+0000", 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016983999999999999 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985967+0000", 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026175999999999999 2023-06-10T21:25:16.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988584+0000", 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012783530000000001 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989863+0000", 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28834436000000002 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278207+0000", 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278244+0000", 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8973e-05 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278293+0000", 2023-06-10T21:25:16.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20072 9.3 9:ce7b60fb:::benchmark_data_smithi064_247471_object20071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.354 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31671447000002, 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998, 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.355 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20072 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985623+0000", 2023-06-10T21:25:16.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7206e-05 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985650+0000", 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7629e-05 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985668+0000", 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985674+0000", 2023-06-10T21:25:16.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001225133 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986899+0000", 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012472500000000001 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987024+0000", 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000152232 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987176+0000", 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014119930000000001 2023-06-10T21:25:16.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988588+0000", 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017869019999999999 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990375+0000", 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28840961799999998 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278785+0000", 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9700000000000002e-05 2023-06-10T21:25:16.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278834+0000", 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8235999999999998e-05 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278872+0000", 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20074 9.3 9:cf5d96ce:::benchmark_data_smithi064_247471_object20073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.365 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31665823200001, 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001, 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.366 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20074 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985679+0000", 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9383e-05 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985709+0000", 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985716+0000", 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.048e-06 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985720+0000", 2023-06-10T21:25:16.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002102862 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987822+0000", 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4449000000000004e-05 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987907+0000", 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134979 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988042+0000", 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064599799999999995 2023-06-10T21:25:16.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988688+0000", 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001729993 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990418+0000", 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28863366400000001 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279052+0000", 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3007999999999998e-05 2023-06-10T21:25:16.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279105+0000", 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8399000000000003e-05 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279143+0000", 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20077 9.d 9:bf31cec4:::benchmark_data_smithi064_247471_object20076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.376 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31547358199998, 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002, 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.377 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20077 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986863+0000", 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1553000000000001e-05 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986895+0000", 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5759999999999995e-06 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986905+0000", 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7969999999999997e-06 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986910+0000", 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1326000000000001e-05 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986982+0000", 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012606299999999999 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987108+0000", 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014393300000000001 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987252+0000", 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001079055 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988331+0000", 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117989 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988449+0000", 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29228185899999998 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280731+0000", 2023-06-10T21:25:16.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4913000000000002e-05 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280776+0000", 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8541000000000003e-05 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280814+0000", 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20079 9.d 9:b60e4f03:::benchmark_data_smithi064_247471_object20078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31334586700001, 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267, 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.388 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20079 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:25:16.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988991+0000", 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9812000000000001e-05 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989071+0000", 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1182000000000001e-05 2023-06-10T21:25:16.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989082+0000", 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1869999999999996e-06 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989087+0000", 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6656999999999998e-05 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989124+0000", 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013583000000000001 2023-06-10T21:25:16.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989260+0000", 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9172999999999995e-05 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989349+0000", 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00098969899999999992 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990339+0000", 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017819999999999999 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990517+0000", 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29222891400000001 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282746+0000", 2023-06-10T21:25:16.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8776000000000001e-05 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282805+0000", 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4026000000000002e-05 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282849+0000", 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.398 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20080 9.e 9:7e203f42:::benchmark_data_smithi064_247471_object20079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31319481600002, 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28931462600000002, 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.399 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20080 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:25:16.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989143+0000", 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989142+0000", 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0820000000000003e-05 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989213+0000", 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0723000000000001e-05 2023-06-10T21:25:16.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989224+0000", 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3220000000000004e-06 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989231+0000", 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3353999999999998e-05 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989285+0000", 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119684 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989404+0000", 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000166086 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989570+0000", 2023-06-10T21:25:16.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00096425099999999995 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990535+0000", 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019061699999999999 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990725+0000", 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28764304899999998 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278368+0000", 2023-06-10T21:25:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4506999999999998e-05 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278423+0000", 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5105999999999999e-05 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278458+0000", 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28931462600000002 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20081 9.e 9:7495eed3:::benchmark_data_smithi064_247471_object20080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 256.31085897200001, 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29004904599999998, 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.410 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20081 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991479+0000", 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991478+0000", 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6579999999999997e-05 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991535+0000", 2023-06-10T21:25:16.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1749999999999994e-06 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991544+0000", 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0550000000000001e-06 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991548+0000", 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7662e-05 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991586+0000", 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1367999999999997e-05 2023-06-10T21:25:16.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991677+0000", 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102032 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.991779+0000", 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00060211100000000001 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.992381+0000", 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000217801 2023-06-10T21:25:16.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.992599+0000", 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.288837964 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281437+0000", 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0899000000000002e-05 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281488+0000", 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0327999999999997e-05 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.281528+0000", 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29004904599999998 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137764 9.d 9:bf6cd1ea:::benchmark_data_smithi064_247471_object28776:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 133.86088459699999, 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998, 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.421 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137764 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441452+0000", 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7470000000000002e-06 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441458+0000", 2023-06-10T21:25:16.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8309999999999996e-06 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441468+0000", 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4750000000000001e-06 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441473+0000", 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3132e-05 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441516+0000", 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114258 2023-06-10T21:25:16.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441631+0000", 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119329 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441750+0000", 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041105380000000004 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445860+0000", 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074270170000000002 2023-06-10T21:25:16.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453288+0000", 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28097103499999998 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734259+0000", 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2166000000000003e-05 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734301+0000", 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6820000000000003e-05 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734338+0000", 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:197262 9.d 9:bc8c96a1:::benchmark_data_smithi064_247471_object88274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.090667093999997, 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29079689600000003, 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197262 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1159999999999999e-06 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211674+0000", 2023-06-10T21:25:16.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6749999999999999e-06 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211679+0000", 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1860000000000002e-06 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211683+0000", 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033231910000000001 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215007+0000", 2023-06-10T21:25:16.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119659 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215126+0000", 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012273399999999999 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215249+0000", 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26155416500000001 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.476803+0000", 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8266000000000002e-05 2023-06-10T21:25:16.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.476852+0000", 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025525157 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502377+0000", 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1844e-05 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502429+0000", 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9428999999999998e-05 2023-06-10T21:25:16.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502468+0000", 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29079689600000003 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:197263 9.9 9:9dc3b6b7:::benchmark_data_smithi064_247471_object88275:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:37.211693+0000", 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 39.090645039000002, 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28683843799999997, 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.443 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197263 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211693+0000", 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211693+0000", 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211693+0000", 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8260000000000001e-06 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211696+0000", 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3700000000000003e-06 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211701+0000", 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.473e-06 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211704+0000", 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26067761099999998 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.472382+0000", 2023-06-10T21:25:16.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013628300000000001 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.472518+0000", 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146174 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.472665+0000", 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040795429999999997 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.476744+0000", 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040891180000000001 2023-06-10T21:25:16.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.480833+0000", 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017602487999999999 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.498436+0000", 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0290999999999997e-05 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.498486+0000", 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5586e-05 2023-06-10T21:25:16.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.498532+0000", 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28683843799999997 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199854 9.3 9:c5f79f4f:::benchmark_data_smithi064_247471_object90866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.765902773999997, 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362, 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.454 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199854 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:25:16.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536433+0000", 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7489999999999998e-06 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536441+0000", 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5624000000000002e-05 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536457+0000", 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5969999999999997e-06 2023-06-10T21:25:16.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536463+0000", 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107921 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536571+0000", 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011766500000000001 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536689+0000", 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5592000000000003e-05 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536774+0000", 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037939639999999999 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540568+0000", 2023-06-10T21:25:16.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012242509 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552811+0000", 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.296973082 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849784+0000", 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6795999999999997e-05 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849831+0000", 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4078e-05 2023-06-10T21:25:16.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849875+0000", 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.464 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199859 9.3 9:ca245c36:::benchmark_data_smithi064_247471_object90871:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.761430933, 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001, 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.465 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199859 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.915e-06 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540910+0000", 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9649999999999996e-06 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540916+0000", 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7190000000000001e-06 2023-06-10T21:25:16.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540920+0000", 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9440999999999997e-05 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540960+0000", 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111699 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541072+0000", 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053361700000000001 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541605+0000", 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011243814 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552849+0000", 2023-06-10T21:25:16.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039226670000000003 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556772+0000", 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29326851399999998 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850040+0000", 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7537999999999999e-05 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850078+0000", 2023-06-10T21:25:16.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4517999999999997e-05 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850112+0000", 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.475 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199860 9.d 9:bf36e690:::benchmark_data_smithi064_247471_object90872:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.761406244, 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999, 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.476 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199860 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540931+0000", 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8059999999999999e-06 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540934+0000", 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2140000000000001e-06 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540940+0000", 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7560000000000001e-06 2023-06-10T21:25:16.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540944+0000", 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003930256 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544874+0000", 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122638 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544997+0000", 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012774499999999999 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545125+0000", 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046761019999999997 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549801+0000", 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00087457099999999999 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.550675+0000", 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29424700599999998 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844922+0000", 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6332000000000002e-05 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844979+0000", 2023-06-10T21:25:16.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3044000000000003e-05 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.845022+0000", 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.486 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199861 9.3 9:c4d624c7:::benchmark_data_smithi064_247471_object90873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.761240682, 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997, 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.487 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199861 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:25:16.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9880000000000004e-06 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541102+0000", 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6309999999999992e-06 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541110+0000", 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319999999999999e-06 2023-06-10T21:25:16.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541115+0000", 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045389230000000003 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545654+0000", 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010809799999999999 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545762+0000", 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000124232 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545887+0000", 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069982400000000002 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552885+0000", 2023-06-10T21:25:16.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003894041 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556779+0000", 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29336341300000002 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850142+0000", 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3971999999999999e-05 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850176+0000", 2023-06-10T21:25:16.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3324999999999997e-05 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850210+0000", 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.497 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199863 9.3 9:c858996c:::benchmark_data_smithi064_247471_object90875:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.761180512000003, 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986, 2023-06-10T21:25:16.498 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199863 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:25:16.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541157+0000", 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0770000000000001e-06 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541161+0000", 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3150000000000003e-06 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541168+0000", 2023-06-10T21:25:16.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0239999999999997e-06 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541173+0000", 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084016230000000004 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549575+0000", 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011957599999999999 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549695+0000", 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013125600000000001 2023-06-10T21:25:16.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549826+0000", 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070024570000000001 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556828+0000", 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036556430000000001 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560484+0000", 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28975610600000001 2023-06-10T21:25:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850240+0000", 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2453999999999997e-05 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850283+0000", 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3065999999999998e-05 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850316+0000", 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.508 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199865 9.3 9:cef2937d:::benchmark_data_smithi064_247471_object90877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.752700005999998, 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124, 2023-06-10T21:25:16.509 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199865 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.510 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549637+0000", 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549642+0000", 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7441000000000001e-05 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549660+0000", 2023-06-10T21:25:16.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4330000000000001e-06 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549666+0000", 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003246578 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552913+0000", 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000139326 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553052+0000", 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147138 2023-06-10T21:25:16.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553199+0000", 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036693580000000002 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556869+0000", 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003640038 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560509+0000", 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28983566300000002 2023-06-10T21:25:16.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850344+0000", 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3887999999999998e-05 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850378+0000", 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1239999999999999e-05 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850409+0000", 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.519 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.520 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.520 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199870 9.3 9:cea33e0b:::benchmark_data_smithi064_247471_object90882:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:25:16.520 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 34.745716395999999, 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003, 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:25:16.521 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199870 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:25:16.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556620+0000", 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.57e-06 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556626+0000", 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5829999999999996e-06 2023-06-10T21:25:16.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556635+0000", 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1599999999999997e-06 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:25:16.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556641+0000", 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026981200000000002 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556910+0000", 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113881 2023-06-10T21:25:16.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557024+0000", 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121586 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:25:16.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557146+0000", 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033591770000000001 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560505+0000", 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018934659999999999 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.562399+0000", 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28803842299999999 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:25:16.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850437+0000", 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2833000000000002e-05 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850470+0000", 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1829999999999998e-05 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:25:16.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850502+0000", 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:25:16.531 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:25:16.532 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:25:16.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:16.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:16.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:16.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:16.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:16.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:16.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 conmon[180272]: 2023-06-10T21:25:16.261+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e188 definitely_dead 0 2023-06-10T21:25:16.535 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:16.535 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:16.535 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.535 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.535 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.536 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:25:16.564 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:16.606 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:16.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:16.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:16.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: failed to encode map e187 with expected crc 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: pgmap v704: 99 pgs: 99 active+clean; 24 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: osdmap e188: 8 total, 8 up, 8 in 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: 9.2 scrub starts 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: 9.2 scrub ok 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4259705978' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: Client client.admin marked osd.2 out, while it was still marked up 2023-06-10T21:25:16.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4083914137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:16.667 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.2. 2023-06-10T21:25:16.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:16.736 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: failed to encode map e187 with expected crc 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: pgmap v704: 99 pgs: 99 active+clean; 24 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: osdmap e188: 8 total, 8 up, 8 in 2023-06-10T21:25:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: 9.2 scrub starts 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: 9.2 scrub ok 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4259705978' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: Client client.admin marked osd.2 out, while it was still marked up 2023-06-10T21:25:16.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4083914137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: failed to encode map e187 with expected crc 2023-06-10T21:25:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: pgmap v704: 99 pgs: 99 active+clean; 24 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 0 B/s, 0 objects/s recovering 2023-06-10T21:25:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: failed to encode map e188 with expected crc 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: osdmap e188: 8 total, 8 up, 8 in 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13056 2023-06-10T21:25:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: 9.2 scrub starts 2023-06-10T21:25:16.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: 9.2 scrub ok 2023-06-10T21:25:16.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4259705978' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-06-10T21:25:16.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: Client client.admin marked osd.2 out, while it was still marked up 2023-06-10T21:25:16.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4083914137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:16.961 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:16.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:17.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:25:17.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:17.117 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:17.747 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:25:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-06-10T21:25:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: osdmap e189: 8 total, 8 up, 7 in 2023-06-10T21:25:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[180297]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-06-10T21:25:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: osdmap e189: 8 total, 8 up, 7 in 2023-06-10T21:25:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:17 smithi064 ceph-mon[186706]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.933 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:17.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: osdmap e189: 8 total, 8 up, 7 in 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: failed to encode map e189 with expected crc 2023-06-10T21:25:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:17 smithi152 ceph-mon[156819]: failed to encode map e189 with expected crc 2023-06-10T21:25:18.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:18.066 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:18.475 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:18.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:18.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:18.610 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:18.614 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:18.657 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:18.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:18.737 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:18.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:18.792 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:18.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: 9.4 scrub starts 2023-06-10T21:25:18.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: 9.4 scrub ok 2023-06-10T21:25:18.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: pgmap v707: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 8.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:18.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:25:18.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: osdmap e190: 8 total, 8 up, 7 in 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:18.823 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: 9.4 scrub starts 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: 9.4 scrub ok 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: pgmap v707: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 8.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.824 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: osdmap e190: 8 total, 8 up, 7 in 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:18.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.848 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:18.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:18.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: 9.4 scrub starts 2023-06-10T21:25:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: 9.4 scrub ok 2023-06-10T21:25:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: pgmap v707: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 8.2 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:25:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: osdmap e190: 8 total, 8 up, 7 in 2023-06-10T21:25:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: failed to encode map e190 with expected crc 2023-06-10T21:25:18.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:18.949 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:18.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:18.980 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:19.027 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:19.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:19.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:19.135 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:19.162 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:19.203 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:19.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:19.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:19.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:19.346 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:19.447 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:19.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:19.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:19.578 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:19.621 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:19.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:19.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:25:19.687 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_4' removed 2023-06-10T21:25:19.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:19.772 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:19.818 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:19.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:19.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.7 scrub starts 2023-06-10T21:25:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.7 scrub ok 2023-06-10T21:25:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.14 scrub starts 2023-06-10T21:25:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.14 scrub ok 2023-06-10T21:25:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: failed to encode map e191 with expected crc 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: osdmap e191: 8 total, 8 up, 7 in 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: failed to encode map e191 with expected crc 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: failed to encode map e191 with expected crc 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.12 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13594 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13587 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13053 2023-06-10T21:25:19.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13450 2023-06-10T21:25:19.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13443 2023-06-10T21:25:19.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.1 scrub starts 2023-06-10T21:25:19.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: 9.1 scrub ok 2023-06-10T21:25:19.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3772591335' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:19.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:19 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:19.949 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:20.026 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:20.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:20.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:20.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:20.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.7 scrub starts 2023-06-10T21:25:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.7 scrub ok 2023-06-10T21:25:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.14 scrub starts 2023-06-10T21:25:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.14 scrub ok 2023-06-10T21:25:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: osdmap e191: 8 total, 8 up, 7 in 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.12 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13594 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13587 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13053 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13450 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13443 2023-06-10T21:25:20.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.1 scrub starts 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: 9.1 scrub ok 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3772591335' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.7 scrub starts 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.7 scrub ok 2023-06-10T21:25:20.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.14 scrub starts 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.14 scrub ok 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: osdmap e191: 8 total, 8 up, 7 in 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: failed to encode map e191 with expected crc 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.12 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13594 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 185'13587 2023-06-10T21:25:20.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 185'13053 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.16 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13450 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.6 starting backfill to osd.6 from (0'0,0'0] MAX to 185'13443 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.1 scrub starts 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: 9.1 scrub ok 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3772591335' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:20.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:19 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:25:20.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:20.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: 9.3 scrub starts 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: 9.3 scrub ok 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: pgmap v710: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 5.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:25:20.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: osdmap e192: 8 total, 8 up, 7 in 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[186706]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: 9.3 scrub starts 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: 9.3 scrub ok 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: pgmap v710: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 5.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:25:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: osdmap e192: 8 total, 8 up, 7 in 2023-06-10T21:25:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:20 smithi064 ceph-mon[180297]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: 9.3 scrub starts 2023-06-10T21:25:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: 9.3 scrub ok 2023-06-10T21:25:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: pgmap v710: 99 pgs: 6 peering, 1 active+clean+remapped, 92 active+clean; 5.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:25:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: osdmap e192: 8 total, 8 up, 7 in 2023-06-10T21:25:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: failed to encode map e192 with expected crc 2023-06-10T21:25:20.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:20 smithi152 ceph-mon[156819]: failed to encode map e192 with expected crc 2023-06-10T21:25:21.200 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:25:21.201 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:25:21.201 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-06-10T21:25:21.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_5 16 2023-06-10T21:25:21.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:21.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:21.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:21.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:21.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:21.586 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:21.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:21.766 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:21.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:21.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:21.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:21.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:21.902 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:25:21.902 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:25:21.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:22.018 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:22.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:22.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:22.695 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:25:22.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:22.754 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:22.921 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:25:22.921 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:25:22.921 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-06-10T21:25:22.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 2 2023-06-10T21:25:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:22 smithi152 ceph-mon[156819]: pgmap v712: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-06-10T21:25:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/98260798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:22.947 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:22] "GET /metrics HTTP/1.1" 200 223725 "" "Prometheus/2.33.4" 2023-06-10T21:25:22.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[186706]: pgmap v712: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-06-10T21:25:22.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/98260798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:22.948 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:22.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[180297]: pgmap v712: 73 pgs: 4 peering, 69 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 2 keys/s, 0 objects/s recovering 2023-06-10T21:25:22.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/98260798' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:22.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:23.761 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:23.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:23.866 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:23.898 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:23.921 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: osdmap e193: 8 total, 8 up, 7 in 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:23 smithi152 ceph-mon[156819]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:25:23.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:23.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: osdmap e193: 8 total, 8 up, 7 in 2023-06-10T21:25:23.945 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[186706]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:25:23.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:25:23.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:23.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: osdmap e193: 8 total, 8 up, 7 in 2023-06-10T21:25:23.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.948 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:23 smithi064 ceph-mon[180297]: failed to encode map e193 with expected crc 2023-06-10T21:25:23.949 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:23.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:24.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:24.085 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:24.131 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:24.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:24.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:24.276 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:24.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:24.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:24.345 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:24.364 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:24.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:24.383 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:24.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:24.402 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:24.420 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:24.439 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.441 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:25:24.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:24.492 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:24.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:24.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[180297]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:25:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:25:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/450790925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[186706]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:25:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:25:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/450790925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:24 smithi152 ceph-mon[156819]: pgmap v714: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:24 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:25:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:24 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:25:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/450790925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:25.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:25:25.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:25.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:25.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:25.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:25.872 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:25.937 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:26.148 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:26.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:26.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:26.286 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:26.335 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:26.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:26.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:26.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:26.468 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:26.513 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:26.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:26.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:26.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:27.175 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' created 2023-06-10T21:25:27.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:27 smithi064 ceph-mon[186706]: pgmap v715: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:27 smithi064 ceph-mon[180297]: pgmap v715: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:27 smithi152 ceph-mon[156819]: pgmap v715: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:27.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:25:28.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:28.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:28.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:28.259 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-10T21:25:28.259 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[180297]: failed to encode map e194 with expected crc 2023-06-10T21:25:28.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-10T21:25:28.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[180297]: osdmap e194: 8 total, 8 up, 7 in 2023-06-10T21:25:28.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-10T21:25:28.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[186706]: failed to encode map e194 with expected crc 2023-06-10T21:25:28.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-10T21:25:28.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:28 smithi064 ceph-mon[186706]: osdmap e194: 8 total, 8 up, 7 in 2023-06-10T21:25:28.288 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:28.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-06-10T21:25:28.360 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:28.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-10T21:25:28.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:28 smithi152 ceph-mon[156819]: failed to encode map e194 with expected crc 2023-06-10T21:25:28.389 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3070780562' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-10T21:25:28.390 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:28 smithi152 ceph-mon[156819]: osdmap e194: 8 total, 8 up, 7 in 2023-06-10T21:25:28.392 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:28.441 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:28.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:28.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:28.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:28.575 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:28.620 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:28.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:28.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:28.756 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:28.762 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:28.798 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:28.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:28.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:28.844 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:25:28.844 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:25:28.845 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:25:28.845 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:25:28.845 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:25:28.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:28.939 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:29.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:28 smithi064 conmon[180272]: 2023-06-10T21:25:28.865+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e195 definitely_dead 0 2023-06-10T21:25:29.254 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.2. 2023-06-10T21:25:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: osdmap e195: 8 total, 8 up, 7 in 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3763916733' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:29.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:29.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: osdmap e195: 8 total, 8 up, 7 in 2023-06-10T21:25:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:29.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3763916733' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:29.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: pgmap v717: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: osdmap e195: 8 total, 8 up, 7 in 2023-06-10T21:25:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: failed to encode map e195 with expected crc 2023-06-10T21:25:29.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:29.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3763916733' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:29.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-06-10T21:25:30.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:25:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:30 smithi152 ceph-mon[156819]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:30 smithi152 ceph-mon[156819]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-06-10T21:25:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:30 smithi152 ceph-mon[156819]: osdmap e196: 8 total, 8 up, 8 in 2023-06-10T21:25:30.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[180297]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[180297]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-06-10T21:25:30.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[180297]: osdmap e196: 8 total, 8 up, 8 in 2023-06-10T21:25:30.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[186706]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[186706]: failed to encode map e196 with expected crc 2023-06-10T21:25:30.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-06-10T21:25:30.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:30 smithi064 ceph-mon[186706]: osdmap e196: 8 total, 8 up, 8 in 2023-06-10T21:25:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: pgmap v720: 89 pgs: 7 peering, 3 creating+peering, 4 unknown, 75 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: osdmap e197: 8 total, 8 up, 8 in 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:31 smithi152 ceph-mon[156819]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.531 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: pgmap v720: 89 pgs: 7 peering, 3 creating+peering, 4 unknown, 75 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: osdmap e197: 8 total, 8 up, 8 in 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[180297]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: pgmap v720: 89 pgs: 7 peering, 3 creating+peering, 4 unknown, 75 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:31.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: osdmap e197: 8 total, 8 up, 8 in 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:31 smithi064 ceph-mon[186706]: failed to encode map e197 with expected crc 2023-06-10T21:25:31.621 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:31.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:31.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:31.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:32.053 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:32.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:32.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:32.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:32.190 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:32.199 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:25:32.239 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:32.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:32.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:32.275 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-06-10T21:25:32.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:32.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:32.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:25:32.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:25:32.408 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:32.419 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:32.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:32.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:32.427 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:25:32.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:32.473 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:32.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:32.509 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:32.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:32.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:32.548 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.550 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.551 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.552 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:25:32.562 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:32.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:32.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:32.712 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:32.741 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:32.783 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:32.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:32.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:32.918 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:32] "GET /metrics HTTP/1.1" 200 223725 "" "Prometheus/2.33.4" 2023-06-10T21:25:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: pgmap v722: 89 pgs: 11 peering, 3 creating+peering, 1 unknown, 74 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: failed to encode map e198 with expected crc 2023-06-10T21:25:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:25:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: osdmap e198: 8 total, 8 up, 8 in 2023-06-10T21:25:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1407598958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:33.598 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:33.598 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:33.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: pgmap v722: 89 pgs: 11 peering, 3 creating+peering, 1 unknown, 74 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:33.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:33.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: failed to encode map e198 with expected crc 2023-06-10T21:25:33.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:25:33.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: osdmap e198: 8 total, 8 up, 8 in 2023-06-10T21:25:33.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1407598958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:33.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:33.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:33.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: pgmap v722: 89 pgs: 11 peering, 3 creating+peering, 1 unknown, 74 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-06-10T21:25:33.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:33.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: failed to encode map e198 with expected crc 2023-06-10T21:25:33.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2251995394' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:25:33.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: osdmap e198: 8 total, 8 up, 8 in 2023-06-10T21:25:33.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1407598958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:33.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:33.649 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:33.657 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:33.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:33.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:33.763 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:33.796 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:33.845 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:33.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:33.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:33.979 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:34.023 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:34.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:34.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:34.164 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:34.714 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:34.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:34.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:34.847 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:35.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:35 smithi064 ceph-mon[180297]: pgmap v724: 89 pgs: 11 peering, 1 creating+peering, 1 unknown, 76 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 102 B/s, 0 objects/s recovering 2023-06-10T21:25:35.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:35 smithi064 ceph-mon[186706]: pgmap v724: 89 pgs: 11 peering, 1 creating+peering, 1 unknown, 76 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 102 B/s, 0 objects/s recovering 2023-06-10T21:25:35.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:25:35.396 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:35.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:35 smithi152 ceph-mon[156819]: pgmap v724: 89 pgs: 11 peering, 1 creating+peering, 1 unknown, 76 active+clean; 582 KiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 102 B/s, 0 objects/s recovering 2023-06-10T21:25:35.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:35.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:35.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:35.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:35.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:25:35.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:35.766 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:35.815 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:35.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:35.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:35.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:35.986 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:25:35.986 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:25:36.015 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:36.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:36.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:36.152 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:36.202 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:36.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:36.337 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:36.381 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:36.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:36.514 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:36.557 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:36.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:36.688 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:36.730 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:36.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:36.838 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:36.864 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:36.906 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:36.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:36.935 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:36.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:25:37.008 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:37.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:37.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:37.143 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:37.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:37.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:37.213 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-06-10T21:25:37.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:37.276 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 6, 7, 0, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:25:37.276 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:25:37.276 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [1, 3, 4, 5, 6, 7, 0, 2] 2023-06-10T21:25:37.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 5 2023-06-10T21:25:37.277 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:37.317 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:37.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:37.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:37.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:37.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:37 smithi152 ceph-mon[156819]: pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1825484309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:37 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.451 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:37.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[186706]: pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:37.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1825484309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[180297]: pgmap v725: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:37.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1825484309' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.487 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:37 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:25:37.492 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:37.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:37.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:37.625 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:37.665 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:37.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:37.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:37.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:37.802 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:38.574 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[186706]: failed to encode map e199 with expected crc 2023-06-10T21:25:38.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:25:38.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[186706]: osdmap e199: 8 total, 8 up, 8 in 2023-06-10T21:25:38.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[186706]: failed to encode map e199 with expected crc 2023-06-10T21:25:38.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[180297]: failed to encode map e199 with expected crc 2023-06-10T21:25:38.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:25:38.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[180297]: osdmap e199: 8 total, 8 up, 8 in 2023-06-10T21:25:38.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:38 smithi064 ceph-mon[180297]: failed to encode map e199 with expected crc 2023-06-10T21:25:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:38 smithi152 ceph-mon[156819]: failed to encode map e199 with expected crc 2023-06-10T21:25:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:25:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:38 smithi152 ceph-mon[156819]: osdmap e199: 8 total, 8 up, 8 in 2023-06-10T21:25:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:38 smithi152 ceph-mon[156819]: failed to encode map e199 with expected crc 2023-06-10T21:25:39.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:39.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:39.349 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:39.349 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:39.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:39.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[186706]: pgmap v727: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 79 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:39.468 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[186706]: failed to encode map e199 with expected crc 2023-06-10T21:25:39.468 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:39.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[180297]: pgmap v727: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 79 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:39.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[180297]: failed to encode map e199 with expected crc 2023-06-10T21:25:39.469 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:39 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:39.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:39 smithi152 ceph-mon[156819]: pgmap v727: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 79 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:39.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:39 smithi152 ceph-mon[156819]: failed to encode map e199 with expected crc 2023-06-10T21:25:39.469 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:39 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:39.471 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:39.520 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:39.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:39.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:39.657 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:39.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:39.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:40.013 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:40.057 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:40.105 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:40.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:40.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:40.135 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:40.154 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:40.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:40.194 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:40.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:40.212 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:40.224 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:25:40.233 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:40.240 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:40.254 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:40.255 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.256 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:25:40.268 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-06-10T21:25:40.270 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:25:40.270 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:25:40.283 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:40.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:40.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:40.343 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:25:40.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:25:40.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_312713 2023-06-10T21:25:40.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:25:40.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:25:40.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:40.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:40.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4151400066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:40.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4151400066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:25:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4151400066' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:40.459 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:41.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3123 3107 194.168 194.188 0.00626639 0.0051359 2023-06-10T21:25:41.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: osdmap e200: 8 total, 8 up, 8 in 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[186706]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:41.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:41.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:41.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: osdmap e200: 8 total, 8 up, 8 in 2023-06-10T21:25:41.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:41 smithi064 ceph-mon[180297]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: pgmap v728: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 73 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2449394148' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:25:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: osdmap e200: 8 total, 8 up, 8 in 2023-06-10T21:25:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:41 smithi152 ceph-mon[156819]: failed to encode map e200 with expected crc 2023-06-10T21:25:41.816 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:41.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:41.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:41.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:41.984 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:42.103 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:42.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:42.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:42.276 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:25:42.276 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.276 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.277 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:42.344 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 5617 5601 175.015 155.875 0.00907208 0.00569965 2023-06-10T21:25:42.365 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:42.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:42.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:42.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:25:42.521 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:25:42.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:42.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:42.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:42.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:42.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:42.837 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:42] "GET /metrics HTTP/1.1" 200 223697 "" "Prometheus/2.33.4" 2023-06-10T21:25:43.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 8171 8155 169.87 159.625 0.0100157 0.00587512 2023-06-10T21:25:43.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:43 smithi064 ceph-mon[180297]: pgmap v730: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:43.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:43 smithi064 ceph-mon[186706]: pgmap v730: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:43.567 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:43 smithi152 ceph-mon[156819]: pgmap v730: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:25:43.742 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:43.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:43.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:43.916 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:44.015 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:44.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:44.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:44.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:44.179 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:44.270 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:44.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:44.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 10863 10847 169.433 168.25 0.00326405 0.00588508 2023-06-10T21:25:44.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:44.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:44.469 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:25:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:25:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:11092 10.0 e200/194 10:0edb844c:::benchmark_data_smithi064_312713_object11091:head v 200'687, mlcod=200'685)", 2023-06-10T21:25:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:44.461387+0000", 2023-06-10T21:25:44.470 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00055255699999999996, 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000585178, 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461387+0000", 2023-06-10T21:25:44.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461387+0000", 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461386+0000", 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:25:44.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461430+0000", 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3961000000000001e-05 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461453+0000", 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3484000000000001e-05 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461462+0000", 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1309999999999995e-06 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461512+0000", 2023-06-10T21:25:44.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0223999999999999e-05 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461536+0000", 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3204e-05 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.476 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:11093 10.0 e200/194)", 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:44.461481+0000", 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00045812899999999999, 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053950500000000002, 2023-06-10T21:25:44.477 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461481+0000", 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:25:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461481+0000", 2023-06-10T21:25:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461481+0000", 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:25:44.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461523+0000", 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1851000000000003e-05 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461531+0000", 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6999999999999997e-06 2023-06-10T21:25:44.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461537+0000", 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.981e-06 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:11094 10.0 e200/194)", 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:44.461543+0000", 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00039575299999999999, 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050399199999999998, 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:25:44.483 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461543+0000", 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461543+0000", 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461543+0000", 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:25:44.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461581+0000", 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7518999999999998e-05 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:25:44.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461588+0000", 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4889999999999998e-06 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:44.461593+0000", 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3059999999999997e-06 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.487 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:25:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:25:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:25:44.488 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 3 2023-06-10T21:25:44.488 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:25:44.533 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:44.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:44.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:44.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:45.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:44 smithi064 conmon[180272]: 2023-06-10T21:25:44.766+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e200 definitely_dead 0 2023-06-10T21:25:45.262 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.5. 2023-06-10T21:25:45.301 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:45.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:45.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 13276 13260 165.705 150.812 0.00269736 0.0059442 2023-06-10T21:25:45.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:45.435 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:45.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:25:45.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[186706]: pgmap v731: 89 pgs: 89 active+clean; 63 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 MiB/s wr, 125 op/s 2023-06-10T21:25:45.511 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3711890264' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[186706]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:25:45.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[180297]: pgmap v731: 89 pgs: 89 active+clean; 63 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 MiB/s wr, 125 op/s 2023-06-10T21:25:45.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3711890264' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.513 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:45 smithi064 ceph-mon[180297]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:25:45.648 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:45.655 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:45 smithi152 ceph-mon[156819]: pgmap v731: 89 pgs: 89 active+clean; 63 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 MiB/s wr, 125 op/s 2023-06-10T21:25:45.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3711890264' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:25:45.656 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:45 smithi152 ceph-mon[156819]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:25:45.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:45.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:45.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:45.786 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:45.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:25:46.345 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 13276 13260 138.09 0 - 0.0059442 2023-06-10T21:25:46.385 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:46.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:46.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:46.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: osdmap e201: 8 total, 8 up, 7 in 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: osdmap e202: 8 total, 8 up, 7 in 2023-06-10T21:25:46.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: osdmap e201: 8 total, 8 up, 7 in 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: osdmap e202: 8 total, 8 up, 7 in 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.524 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:46 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:25:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: osdmap e201: 8 total, 8 up, 7 in 2023-06-10T21:25:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e201 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: osdmap e202: 8 total, 8 up, 7 in 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:46 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:46.830 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:46.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:46.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:25:46.936 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:47.036 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:47.045 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:47.177 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:47.219 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:25:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:47.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:25:47.327 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:47.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 13990 13974 124.739 22.3125 0.00482372 0.00801163 2023-06-10T21:25:47.370 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:47.433 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:47.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:47.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: pgmap v733: 89 pgs: 89 active+clean; 651 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:25:47.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: osdmap e203: 8 total, 8 up, 7 in 2023-06-10T21:25:47.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[180297]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: pgmap v733: 89 pgs: 89 active+clean; 651 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:25:47.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: osdmap e203: 8 total, 8 up, 7 in 2023-06-10T21:25:47.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:47 smithi064 ceph-mon[186706]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: pgmap v733: 89 pgs: 89 active+clean; 651 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e202 with expected crc 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.574 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:47.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: osdmap e203: 8 total, 8 up, 7 in 2023-06-10T21:25:47.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.575 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:47 smithi152 ceph-mon[156819]: failed to encode map e203 with expected crc 2023-06-10T21:25:47.578 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:47.645 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:25:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:47.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:25:47.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:47.798 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:47.872 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:47.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:47.980 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:48.028 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:48.090 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:48.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:48.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:48.267 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:48.346 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 15845 15829 123.636 115.938 0.00803885 0.00808299 2023-06-10T21:25:48.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:48 smithi064 ceph-mon[180297]: failed to encode map e203 with expected crc 2023-06-10T21:25:48.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:48 smithi064 ceph-mon[186706]: failed to encode map e203 with expected crc 2023-06-10T21:25:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:48 smithi152 ceph-mon[156819]: failed to encode map e203 with expected crc 2023-06-10T21:25:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:25:49.346 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 18143 18127 125.852 143.625 0.00690194 0.00794012 2023-06-10T21:25:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:49 smithi064 ceph-mon[180297]: pgmap v736: 89 pgs: 89 active+clean; 651 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:25:49.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:49 smithi064 ceph-mon[186706]: pgmap v736: 89 pgs: 89 active+clean; 651 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:25:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:49 smithi152 ceph-mon[156819]: pgmap v736: 89 pgs: 89 active+clean; 651 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:25:50.347 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 20261 20245 126.502 132.375 0.0091942 0.00789477 2023-06-10T21:25:50.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:50.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:50.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:50.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:50.528 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:50.590 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:50.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:50.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:50 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 3302/51894 objects degraded (6.363%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:50 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 3302/51894 objects degraded (6.363%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:50.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:50 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 3302/51894 objects degraded (6.363%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:50.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:50.732 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:50.780 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:50.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:50.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:25:50.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:51.076 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:25:51.077 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:25:51.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:51.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:25:51.091 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:51.117 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:25:51.174 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:25:51.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:25:51.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:25:51.229 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:51.258 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:25:51.290 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:25:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:51.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:25:51.306 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:25:51.346 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:25:51.348 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 21780 21764 123.623 94.9375 0.0140925 0.00808143 2023-06-10T21:25:51.355 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:51 smithi064 ceph-mon[180297]: pgmap v737: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 169 MiB/s wr, 2.71k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.9 MiB/s, 48 objects/s recovering 2023-06-10T21:25:51.355 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/549457638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:51.382 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:25:51.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.387 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:25:51.396 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:51.455 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:51 smithi064 ceph-mon[186706]: pgmap v737: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 169 MiB/s wr, 2.71k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.9 MiB/s, 48 objects/s recovering 2023-06-10T21:25:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/549457638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:51 smithi152 ceph-mon[156819]: pgmap v737: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 169 MiB/s wr, 2.71k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.9 MiB/s, 48 objects/s recovering 2023-06-10T21:25:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/549457638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:25:51.970 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 6, 7, 0, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:25:51.970 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:25:51.970 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-10T21:25:51.971 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-06-10T21:25:51.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 4 0 2023-06-10T21:25:52.348 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 15 23537 23522 122.47 109.875 0.004787 0.00815938 2023-06-10T21:25:52.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1270975512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:52.376 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:52.424 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:25:52.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1270975512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:52.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1270975512' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:25:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:25:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:25:52] "GET /metrics HTTP/1.1" 200 223827 "" "Prometheus/2.33.4" 2023-06-10T21:25:53.348 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 23725 23709 113.95 11.6875 0.00162633 0.00817474 2023-06-10T21:25:53.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:53.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:53.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:53.564 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:53.642 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:53.647 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: pgmap v738: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.8 MiB/s, 46 objects/s recovering 2023-06-10T21:25:53.647 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:53.647 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.647 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:53.647 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: osdmap e204: 8 total, 8 up, 7 in 2023-06-10T21:25:53.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:53 smithi152 ceph-mon[156819]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:53.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: pgmap v738: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.8 MiB/s, 46 objects/s recovering 2023-06-10T21:25:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:53.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: osdmap e204: 8 total, 8 up, 7 in 2023-06-10T21:25:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[180297]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: pgmap v738: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 3302/51894 objects degraded (6.363%); 547/51894 objects misplaced (1.054%); 2.8 MiB/s, 46 objects/s recovering 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: osdmap e204: 8 total, 8 up, 7 in 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:53 smithi064 ceph-mon[186706]: failed to encode map e204 with expected crc 2023-06-10T21:25:53.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:53.802 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:53.875 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:53.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:53.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:53.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:54.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:54.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 25346 25330 113.04 101.312 0.00994276 0.00883716 2023-06-10T21:25:54.486 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:54.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:54.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[180297]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[180297]: osdmap e205: 8 total, 8 up, 7 in 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[180297]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[180297]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[186706]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[186706]: osdmap e205: 8 total, 8 up, 7 in 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[186706]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:54 smithi064 ceph-mon[186706]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:54.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:54 smithi152 ceph-mon[156819]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:54 smithi152 ceph-mon[156819]: osdmap e205: 8 total, 8 up, 7 in 2023-06-10T21:25:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:54 smithi152 ceph-mon[156819]: failed to encode map e205 with expected crc 2023-06-10T21:25:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:54 smithi152 ceph-mon[156819]: failed to encode map e205 with expected crc 2023-06-10T21:25:55.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 27103 27087 112.824 109.812 0.0165921 0.00885735 2023-06-10T21:25:55.352 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:55.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:55.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:55.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:55.488 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:55.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:55.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:55.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:55 smithi064 ceph-mon[186706]: pgmap v741: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2480/56862 objects degraded (4.361%); 1113/56862 objects misplaced (1.957%); 4.8 MiB/s, 78 objects/s recovering 2023-06-10T21:25:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:55 smithi064 ceph-mon[180297]: pgmap v741: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2480/56862 objects degraded (4.361%); 1113/56862 objects misplaced (1.957%); 4.8 MiB/s, 78 objects/s recovering 2023-06-10T21:25:55.672 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:55 smithi152 ceph-mon[156819]: pgmap v741: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 2480/56862 objects degraded (4.361%); 1113/56862 objects misplaced (1.957%); 4.8 MiB/s, 78 objects/s recovering 2023-06-10T21:25:55.676 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:55.747 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:55.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:55.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:55.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:55.889 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:56.144 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:56.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:56.189 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:56.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:25:56.276 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:56.321 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:25:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:56.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:25:56.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 27310 27294 106.583 12.9375 0.00189554 0.00900729 2023-06-10T21:25:56.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 2480/56862 objects degraded (4.361%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:56.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[186706]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[186706]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[186706]: osdmap e206: 8 total, 8 up, 7 in 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[186706]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 2480/56862 objects degraded (4.361%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[180297]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[180297]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[180297]: osdmap e206: 8 total, 8 up, 7 in 2023-06-10T21:25:56.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:56 smithi064 ceph-mon[180297]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:56.455 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:56.501 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:56.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:56.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:25:56.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:56.667 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 2480/56862 objects degraded (4.361%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:25:56.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:56 smithi152 ceph-mon[156819]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:56 smithi152 ceph-mon[156819]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:56 smithi152 ceph-mon[156819]: osdmap e206: 8 total, 8 up, 7 in 2023-06-10T21:25:56.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:56 smithi152 ceph-mon[156819]: failed to encode map e206 with expected crc 2023-06-10T21:25:56.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:25:57.047 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:25:57.047 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:25:57.172 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:57.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:57.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:25:57.303 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:57.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 27642 27626 101.535 20.75 0.00255133 0.00953444 2023-06-10T21:25:57.350 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:25:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:57.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:25:57.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:25:57.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:57.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: pgmap v742: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 13 MiB/s, 206 objects/s recovering 2023-06-10T21:25:57.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: osdmap e207: 8 total, 8 up, 7 in 2023-06-10T21:25:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[186706]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: pgmap v742: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 13 MiB/s, 206 objects/s recovering 2023-06-10T21:25:57.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: osdmap e207: 8 total, 8 up, 7 in 2023-06-10T21:25:57.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:57 smithi064 ceph-mon[180297]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: pgmap v742: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 113 MiB/s wr, 1.80k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 13 MiB/s, 206 objects/s recovering 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: osdmap e207: 8 total, 8 up, 7 in 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: failed to encode map e207 with expected crc 2023-06-10T21:25:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:57 smithi152 ceph-mon[156819]: failed to encode map e207 with expected crc 2023-06-10T21:25:58.349 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 28801 28785 99.9182 72.4375 0.00435363 0.0100027 2023-06-10T21:25:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 16 MiB/s, 261 objects/s recovering 2023-06-10T21:25:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: osdmap e208: 8 total, 8 up, 7 in 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 16 MiB/s, 261 objects/s recovering 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: osdmap e208: 8 total, 8 up, 7 in 2023-06-10T21:25:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:58.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: pgmap v745: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 1679/74550 objects degraded (2.252%); 806/74550 objects misplaced (1.081%); 16 MiB/s, 261 objects/s recovering 2023-06-10T21:25:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: failed to encode map e208 with expected crc 2023-06-10T21:25:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: osdmap e208: 8 total, 8 up, 7 in 2023-06-10T21:25:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:25:59.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:25:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:59.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:25:59.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 30184 30168 99.2066 86.4375 0.0123837 0.0100751 2023-06-10T21:25:59.414 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:25:59.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:25:59.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:59.646 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:25:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:25:59.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:25:59.699 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:25:59.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:25:59.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:25:59 smithi064 ceph-mon[180297]: failed to encode map e208 with expected crc 2023-06-10T21:25:59.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:25:59 smithi064 ceph-mon[186706]: failed to encode map e208 with expected crc 2023-06-10T21:25:59.895 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:25:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:25:59 smithi152 ceph-mon[156819]: failed to encode map e208 with expected crc 2023-06-10T21:26:00.118 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:00.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:00.150 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:00.223 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:00.253 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:00.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:26:00.348041+0000 min lat: 0.00121051 max lat: 1.76327 avg lat: 0.0100916 2023-06-10T21:26:00.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:26:00.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 30981 30965 96.7372 49.8125 0.0037097 0.0100916 2023-06-10T21:26:00.631 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.4 primary-affinity to 0 (802) 2023-06-10T21:26:00.776 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:00.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: pgmap v747: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 146/89109 objects misplaced (0.164%); 36 MiB/s, 574 objects/s recovering 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1679/74550 objects degraded (2.252%), 2 pgs degraded) 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: osdmap e209: 8 total, 8 up, 7 in 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:26:00.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[186706]: osdmap e210: 8 total, 8 up, 7 in 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: pgmap v747: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 146/89109 objects misplaced (0.164%); 36 MiB/s, 574 objects/s recovering 2023-06-10T21:26:00.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1679/74550 objects degraded (2.252%), 2 pgs degraded) 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: osdmap e209: 8 total, 8 up, 7 in 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:00.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-06-10T21:26:00.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-06-10T21:26:00.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:00 smithi064 ceph-mon[180297]: osdmap e210: 8 total, 8 up, 7 in 2023-06-10T21:26:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: pgmap v747: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 146/89109 objects misplaced (0.164%); 36 MiB/s, 574 objects/s recovering 2023-06-10T21:26:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1679/74550 objects degraded (2.252%), 2 pgs degraded) 2023-06-10T21:26:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: osdmap e209: 8 total, 8 up, 7 in 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: failed to encode map e209 with expected crc 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-06-10T21:26:00.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3877132886' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-06-10T21:26:00.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:00 smithi152 ceph-mon[156819]: osdmap e210: 8 total, 8 up, 7 in 2023-06-10T21:26:01.350 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 31054 31038 92.3488 4.5625 0.00215342 0.0104705 2023-06-10T21:26:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[186706]: osdmap e211: 8 total, 8 up, 7 in 2023-06-10T21:26:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:01 smithi064 ceph-mon[180297]: osdmap e211: 8 total, 8 up, 7 in 2023-06-10T21:26:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: failed to encode map e210 with expected crc 2023-06-10T21:26:01.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:01 smithi152 ceph-mon[156819]: osdmap e211: 8 total, 8 up, 7 in 2023-06-10T21:26:01.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:01.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:02.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:02.099 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:02.175 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:02.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:02.281 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:02.344 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:02.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 32372 32356 91.8931 82.375 0.00978304 0.0108761 2023-06-10T21:26:02.417 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:02.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:02.525 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:26:02.597 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:02.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: pgmap v750: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s; 146/89109 objects misplaced (0.164%); 30 MiB/s, 472 objects/s recovering 2023-06-10T21:26:02.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.687 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.688 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.689 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.689 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.689 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:02 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: pgmap v750: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s; 146/89109 objects misplaced (0.164%); 30 MiB/s, 472 objects/s recovering 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: pgmap v750: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.8 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 61 MiB/s wr, 981 op/s; 146/89109 objects misplaced (0.164%); 30 MiB/s, 472 objects/s recovering 2023-06-10T21:26:02.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:02.829 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:02 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:03.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:02] "GET /metrics HTTP/1.1" 200 223827 "" "Prometheus/2.33.4" 2023-06-10T21:26:03.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 33891 33875 92.025 94.9375 0.0165426 0.0108603 2023-06-10T21:26:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:03 smithi152 ceph-mon[156819]: failed to encode map e211 with expected crc 2023-06-10T21:26:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:04.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[180297]: failed to encode map e211 with expected crc 2023-06-10T21:26:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:04.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[186706]: failed to encode map e211 with expected crc 2023-06-10T21:26:04.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:04.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:04.274 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:04.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:04.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:04.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 35261 35245 91.7581 85.625 0.00982414 0.0108922 2023-06-10T21:26:04.381 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:04.442 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:04.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:04.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:04.758 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:04.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:04 smithi064 ceph-mon[186706]: pgmap v752: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 146/92664 objects misplaced (0.158%); 25 MiB/s, 393 objects/s recovering 2023-06-10T21:26:04.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:04 smithi064 ceph-mon[180297]: pgmap v752: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 146/92664 objects misplaced (0.158%); 25 MiB/s, 393 objects/s recovering 2023-06-10T21:26:04.852 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:04.967 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:04.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:05.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:05.115 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:04 smithi152 ceph-mon[156819]: pgmap v752: 89 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 87 active+clean; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 146/92664 objects misplaced (0.158%); 25 MiB/s, 393 objects/s recovering 2023-06-10T21:26:05.120 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:05.181 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:05.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:05.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:05.350 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:26:05.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 36748 36732 91.8049 92.9375 0.011768 0.0108857 2023-06-10T21:26:05.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:05.392 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:26:05.438 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:26:05.470 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:26:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:26:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:26:05.470 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147711 8.3 8:c8c1f6b8:::benchmark_data_smithi064_208298_object32445:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e108)", 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.923759+0000", 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.54071069899999, 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0108272279999999, 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.471 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147711 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923759+0000", 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923759+0000", 2023-06-10T21:26:05.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923757+0000", 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923767+0000", 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0040999999999999e-05 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923782+0000", 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5332000000000002e-05 2023-06-10T21:26:05.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923794+0000", 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1109e-05 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923852+0000", 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8218999999999999e-05 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923860+0000", 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3950000000000001e-06 2023-06-10T21:26:05.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.928166+0000", 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004306186 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.928178+0000", 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1564e-05 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.934486+0000", 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.006308089 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.934586+0000", 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9976999999999995e-05 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147717 8.3 8:cf20127d:::benchmark_data_smithi064_208298_object32451:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e108)", 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.923811+0000", 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.54065863999995, 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.011170602, 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:26:05.481 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147717 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923811+0000", 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923811+0000", 2023-06-10T21:26:05.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923810+0000", 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923813+0000", 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.056e-06 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923820+0000", 2023-06-10T21:26:05.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6969999999999996e-06 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923826+0000", 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2450000000000003e-06 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923915+0000", 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8572999999999994e-05 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting_for_map not empty", 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.923922+0000", 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9550000000000003e-06 2023-06-10T21:26:05.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.928952+0000", 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050305899999999997 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.928958+0000", 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.733e-06 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.934909+0000", 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005951214 2023-06-10T21:26:05.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.934981+0000", 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1999e-05 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.497 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147733 8.3 8:ca6a8432:::benchmark_data_smithi064_208298_object32467:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.936432+0000", 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.52803777300005, 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.025828368, 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.498 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147733 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936432+0000", 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936432+0000", 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936430+0000", 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936439+0000", 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7460000000000006e-06 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936454+0000", 2023-06-10T21:26:05.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5328e-05 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.936463+0000", 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2110000000000004e-06 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.940627+0000", 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041636310000000001 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.940641+0000", 2023-06-10T21:26:05.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4066000000000001e-05 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.935030+0000", 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99438862299999997 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.935177+0000", 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000147081 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.935740+0000", 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056341799999999999 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.958461+0000", 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022720641 2023-06-10T21:26:05.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962137+0000", 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036758530000000002 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962170+0000", 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3501000000000003e-05 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962219+0000", 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9196999999999997e-05 2023-06-10T21:26:05.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962260+0000", 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0723000000000003e-05 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147751 8.3 8:cde8255e:::benchmark_data_smithi064_208298_object32485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.963077+0000", 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.50139288000003, 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.009975584, 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.511 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147751 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963077+0000", 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963077+0000", 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963075+0000", 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963084+0000", 2023-06-10T21:26:05.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9220000000000003e-06 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963104+0000", 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0004e-05 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.963116+0000", 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2183e-05 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.964860+0000", 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017443859999999999 2023-06-10T21:26:05.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.964872+0000", 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2011000000000001e-05 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.948219+0000", 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98334686999999998 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.948368+0000", 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014880000000000001 2023-06-10T21:26:05.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.948504+0000", 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013624399999999999 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.958471+0000", 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099665369999999993 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.972918+0000", 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014446967999999999 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.972955+0000", 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7830999999999997e-05 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.973008+0000", 2023-06-10T21:26:05.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2643999999999999e-05 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.973052+0000", 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4115e-05 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.522 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147753 8.3 8:c9760a8c:::benchmark_data_smithi064_208298_object32487:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.968336+0000", 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.49613373099999, 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.007200484, 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.523 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147753 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968336+0000", 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968336+0000", 2023-06-10T21:26:05.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968334+0000", 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968343+0000", 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0599999999999997e-06 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968376+0000", 2023-06-10T21:26:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2620999999999998e-05 2023-06-10T21:26:05.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968385+0000", 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6630000000000005e-06 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968473+0000", 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8017999999999997e-05 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.968487+0000", 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.402e-05 2023-06-10T21:26:05.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.951729+0000", 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98324169900000002 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.951880+0000", 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015145999999999999 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.951996+0000", 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011510600000000001 2023-06-10T21:26:05.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.973091+0000", 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021094951000000001 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975425+0000", 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00233431 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975453+0000", 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8082999999999999e-05 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975497+0000", 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3911e-05 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975536+0000", 2023-06-10T21:26:05.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9388000000000003e-05 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147758 8.3 8:cf0756a4:::benchmark_data_smithi064_208298_object32492:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.974034+0000", 2023-06-10T21:26:05.536 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.49043545899997, 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0016242200000001, 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147758 2023-06-10T21:26:05.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974034+0000", 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974034+0000", 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974032+0000", 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974043+0000", 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1127999999999999e-05 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974064+0000", 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0981000000000001e-05 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974074+0000", 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5380000000000008e-06 2023-06-10T21:26:05.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974259+0000", 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018549199999999999 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.974273+0000", 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3903999999999999e-05 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.954741+0000", 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98046745899999999 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.954883+0000", 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142199 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.955030+0000", 2023-06-10T21:26:05.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014674499999999999 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.973121+0000", 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018091006 2023-06-10T21:26:05.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975545+0000", 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024245180000000001 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975590+0000", 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4360000000000002e-05 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975629+0000", 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8909999999999998e-05 2023-06-10T21:26:05.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975658+0000", 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9689e-05 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147767 8.3 8:cc7fed10:::benchmark_data_smithi064_208298_object32501:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.986310+0000", 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.47815925700002, 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98944487800000003, 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147767 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986310+0000", 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986310+0000", 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986309+0000", 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986317+0000", 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.67e-06 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986333+0000", 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6181999999999999e-05 2023-06-10T21:26:05.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986344+0000", 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0342e-05 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986413+0000", 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9436000000000004e-05 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.986427+0000", 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3954000000000001e-05 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.958623+0000", 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97219613100000002 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.958722+0000", 2023-06-10T21:26:05.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8764000000000005e-05 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.958807+0000", 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4921999999999998e-05 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975416+0000", 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016609519999999999 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975550+0000", 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133158 2023-06-10T21:26:05.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975687+0000", 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137174 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975723+0000", 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5782999999999999e-05 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975755+0000", 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2542000000000001e-05 2023-06-10T21:26:05.560 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147775 8.3 8:cb8ece4a:::benchmark_data_smithi064_208298_object32509:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:13.999680+0000", 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.46478900499994, 2023-06-10T21:26:05.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98657181100000002, 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147775 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.562 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999680+0000", 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999680+0000", 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999678+0000", 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999688+0000", 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9340000000000001e-06 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999705+0000", 2023-06-10T21:26:05.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7207e-05 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999715+0000", 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.961e-06 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999778+0000", 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2427000000000004e-05 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:13.999789+0000", 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0852e-05 2023-06-10T21:26:05.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962292+0000", 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96250332599999999 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962435+0000", 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014250800000000001 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.962567+0000", 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013213 2023-06-10T21:26:05.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975554+0000", 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012987143 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975784+0000", 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000230132 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986166+0000", 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010381843 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986210+0000", 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4526e-05 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986252+0000", 2023-06-10T21:26:05.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1955000000000001e-05 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147779 8.3 8:c81e19bb:::benchmark_data_smithi064_208298_object32513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:14.006653+0000", 2023-06-10T21:26:05.573 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.45781666799996, 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99436110300000002, 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147779 2023-06-10T21:26:05.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006653+0000", 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006653+0000", 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006651+0000", 2023-06-10T21:26:05.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006660+0000", 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7940000000000008e-06 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006680+0000", 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9649000000000001e-05 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006690+0000", 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0032e-05 2023-06-10T21:26:05.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006745+0000", 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5466000000000001e-05 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.006763+0000", 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7543000000000001e-05 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.966038+0000", 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95927518300000003 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.966160+0000", 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012248599999999999 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.966283+0000", 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012318499999999999 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975559+0000", 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092751689999999998 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986032+0000", 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010473394 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.000904+0000", 2023-06-10T21:26:05.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014872059999999999 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.000960+0000", 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5930999999999997e-05 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001014+0000", 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3900999999999998e-05 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.585 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147785 8.3 8:cf3e79d6:::benchmark_data_smithi064_208298_object32519:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:14.015489+0000", 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.44898078000006, 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98565999199999998, 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.586 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147785 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015489+0000", 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015489+0000", 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015486+0000", 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015502+0000", 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5709000000000002e-05 2023-06-10T21:26:05.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015525+0000", 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3629999999999999e-05 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015540+0000", 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4149e-05 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015612+0000", 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2115999999999999e-05 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.015625+0000", 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2989999999999999e-05 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.969103+0000", 2023-06-10T21:26:05.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.953478724 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.969185+0000", 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1726999999999999e-05 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.969270+0000", 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4778999999999996e-05 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975563+0000", 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006293005 2023-06-10T21:26:05.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986115+0000", 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010552267000000001 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001065+0000", 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014949370999999999 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001112+0000", 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7397e-05 2023-06-10T21:26:05.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001149+0000", 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6712999999999999e-05 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147804 8.3 8:cccb75b7:::benchmark_data_smithi064_208298_object32538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:14.051120+0000", 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.413349595, 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95230897699999995, 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.598 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147804 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051120+0000", 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051120+0000", 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051118+0000", 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051127+0000", 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2969999999999992e-06 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051146+0000", 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9023000000000001e-05 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051157+0000", 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0475999999999999e-05 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051224+0000", 2023-06-10T21:26:05.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6772000000000004e-05 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.051236+0000", 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2228e-05 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975803+0000", 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92456699099999995 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.975925+0000", 2023-06-10T21:26:05.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012244799999999999 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.976057+0000", 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013219800000000001 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986129+0000", 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01007107 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001198+0000", 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015069279 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003318+0000", 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002120612 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003385+0000", 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6518000000000002e-05 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003429+0000", 2023-06-10T21:26:05.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3912000000000002e-05 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.44528.0:147813 8.3 8:c12ea74c:::benchmark_data_smithi064_208298_object32547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-10T21:26:05.610 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:16:14.070487+0000", 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 591.39398207600004, 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93305414600000003, 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.44528", 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4045686248", 2023-06-10T21:26:05.611 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 147813 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070487+0000", 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070487+0000", 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070486+0000", 2023-06-10T21:26:05.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070496+0000", 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9210000000000004e-06 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070514+0000", 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8388000000000001e-05 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070524+0000", 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5780000000000005e-06 2023-06-10T21:26:05.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070595+0000", 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1212999999999997e-05 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.070608+0000", 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3329000000000001e-05 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.979325+0000", 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90871639800000004 2023-06-10T21:26:05.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.979433+0000", 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010851 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6", 2023-06-10T21:26:05.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.979535+0000", 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101532 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:14.986264+0000", 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067297379999999999 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.001371+0000", 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015106704 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003467+0000", 2023-06-10T21:26:05.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020957340000000001 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003507+0000", 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0639000000000002e-05 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:16:15.003542+0000", 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4183999999999997e-05 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199854 9.3 e171/127 9:c5f79f4f:::benchmark_data_smithi064_247471_object90866:head v 171'12639, mlcod=171'12637)", 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.536832+0000", 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.927637697999998, 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30933095599999999, 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536832+0000", 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536832+0000", 2023-06-10T21:26:05.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536831+0000", 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536838+0000", 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5100000000000004e-06 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536847+0000", 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5680000000000005e-06 2023-06-10T21:26:05.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.536853+0000", 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0680000000000002e-06 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537053+0000", 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000199796 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537072+0000", 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9290999999999999e-05 2023-06-10T21:26:05.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846127+0000", 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.309054367 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846163+0000", 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6006000000000001e-05 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199852 9.0 e171/157 9:0219fd3c:::benchmark_data_smithi064_247471_object90864:head v 171'12112, mlcod=171'12110)", 2023-06-10T21:26:05.630 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.537472+0000", 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.926997942, 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30239325500000003, 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537472+0000", 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537472+0000", 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537471+0000", 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537477+0000", 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2210000000000002e-06 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537488+0000", 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0710000000000001e-05 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537494+0000", 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3770000000000001e-06 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537551+0000", 2023-06-10T21:26:05.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6329999999999998e-05 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537569+0000", 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8927999999999999e-05 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.839826+0000", 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30225702599999998 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.839865+0000", 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8333e-05 2023-06-10T21:26:05.637 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199845 9.f e171/127 9:ff71ad38:::benchmark_data_smithi064_247471_object90857:head v 171'12392, mlcod=171'12386)", 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.537785+0000", 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.926684835000003, 2023-06-10T21:26:05.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29712714200000001, 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537785+0000", 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537785+0000", 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537784+0000", 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537790+0000", 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.122e-06 2023-06-10T21:26:05.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537800+0000", 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0342e-05 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.537807+0000", 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2389999999999999e-06 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.544543+0000", 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067361870000000003 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.544562+0000", 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9474e-05 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834871+0000", 2023-06-10T21:26:05.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29030844500000003 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834912+0000", 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0978e-05 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.645 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199859 9.3 e171/127 9:ca245c36:::benchmark_data_smithi064_247471_object90871:head v 171'12640, mlcod=171'12637)", 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.542055+0000", 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.922414685999996, 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30413052200000001, 2023-06-10T21:26:05.646 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542055+0000", 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542055+0000", 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542054+0000", 2023-06-10T21:26:05.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542060+0000", 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542071+0000", 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0104e-05 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.542077+0000", 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4439999999999996e-06 2023-06-10T21:26:05.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.543418+0000", 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001340819 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.543438+0000", 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9766e-05 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846168+0000", 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30273001700000002 2023-06-10T21:26:05.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846185+0000", 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7666999999999999e-05 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199856 9.0 e171/157 9:0a15bf18:::benchmark_data_smithi064_247471_object90868:head v 171'12113, mlcod=171'12110)", 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.545747+0000", 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.918722556999995, 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29414030699999999, 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.654 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545747+0000", 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545747+0000", 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545746+0000", 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545757+0000", 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0713000000000001e-05 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545768+0000", 2023-06-10T21:26:05.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.096e-05 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545775+0000", 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5240000000000003e-06 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.547406+0000", 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001630411 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.547425+0000", 2023-06-10T21:26:05.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9018e-05 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.839869+0000", 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29244425099999999 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.839887+0000", 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8189000000000002e-05 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.661 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199860 9.d e171/127 9:bf36e690:::benchmark_data_smithi064_247471_object90872:head v 171'12807, mlcod=171'12805)", 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.545974+0000", 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.918495324999995, 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.296512516, 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545974+0000", 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545974+0000", 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545973+0000", 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545987+0000", 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3553000000000001e-05 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.545997+0000", 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546005+0000", 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1389999999999998e-06 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834865+0000", 2023-06-10T21:26:05.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.288860481 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834891+0000", 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5480999999999999e-05 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.842456+0000", 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075650969999999998 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.842487+0000", 2023-06-10T21:26:05.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1006000000000001e-05 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199861 9.3 e171/127 9:c4d624c7:::benchmark_data_smithi064_247471_object90873:head v 171'12641, mlcod=171'12637)", 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.546012+0000", 2023-06-10T21:26:05.669 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.918457611999997, 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30019131599999999, 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546012+0000", 2023-06-10T21:26:05.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546012+0000", 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546012+0000", 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546015+0000", 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7749999999999999e-06 2023-06-10T21:26:05.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546021+0000", 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6579999999999999e-06 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.546026+0000", 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7199999999999997e-06 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.547629+0000", 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001603611 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.547648+0000", 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8397999999999999e-05 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846187+0000", 2023-06-10T21:26:05.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29853970000000002 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846203+0000", 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5747e-05 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.676 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35400.0:199863 9.3 e171/127 9:c858996c:::benchmark_data_smithi064_247471_object90875:head v 171'12642, mlcod=171'12637)", 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:24:41.549934+0000", 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.914534986000007, 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29628522200000001, 2023-06-10T21:26:05.677 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.549934+0000", 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.549934+0000", 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.549934+0000", 2023-06-10T21:26:05.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.549940+0000", 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0179999999999998e-06 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834829+0000", 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28488920000000001 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834843+0000", 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.366e-05 2023-06-10T21:26:05.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834881+0000", 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8730999999999999e-05 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.834901+0000", 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9321999999999999e-05 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846205+0000", 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011303957 2023-06-10T21:26:05.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:24:41.846220+0000", 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5079000000000001e-05 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.684 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:05.685 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:05.685 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:26:05.685 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:26:05.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:05.686 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:05.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:05.687 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:26:05.687 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:26:05.728 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:26:05.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:05.837 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:05.913 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:05.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:06.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:06.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2855562502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:06.089 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3140710515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.089 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.089 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2855562502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:06.090 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3140710515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.090 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:05 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:06.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2855562502' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:06.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3140710515' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:05 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:06.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 38272 38256 91.9369 95.25 0.00190832 0.0108483 2023-06-10T21:26:07.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: pgmap v753: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:26:07.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: osdmap e212: 8 total, 8 up, 7 in 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[180297]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: pgmap v753: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: osdmap e212: 8 total, 8 up, 7 in 2023-06-10T21:26:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:06 smithi064 ceph-mon[186706]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: pgmap v753: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 52 MiB/s wr, 830 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:26:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:26:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: osdmap e212: 8 total, 8 up, 7 in 2023-06-10T21:26:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:06 smithi152 ceph-mon[156819]: failed to encode map e212 with expected crc 2023-06-10T21:26:07.357 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 39469 39453 91.3025 74.8125 0.00652907 0.0109479 2023-06-10T21:26:07.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:26:07.385 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:07.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:07.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:07.594 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:07.702 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:07.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:07.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:07.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:07.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:08.245 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:08.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:08.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 41036 41020 91.5391 97.9375 0.0108448 0.0109195 2023-06-10T21:26:08.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:08.382 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 6, 7, 0, 2] out_osds: [5] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:26:08.383 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:26:08.383 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 3, 4, 6, 7, 0, 2] 2023-06-10T21:26:08.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 6 2023-06-10T21:26:08.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:08.476 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:08.549 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:08.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:08.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:26:08.743 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:08.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:08 smithi064 ceph-mon[180297]: failed to encode map e212 with expected crc 2023-06-10T21:26:08.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:08 smithi064 ceph-mon[180297]: pgmap v755: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-06-10T21:26:08.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:08 smithi064 ceph-mon[186706]: failed to encode map e212 with expected crc 2023-06-10T21:26:08.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:08 smithi064 ceph-mon[186706]: pgmap v755: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-06-10T21:26:08.899 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:08.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:09.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:09.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:08 smithi152 ceph-mon[156819]: failed to encode map e212 with expected crc 2023-06-10T21:26:09.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:08 smithi152 ceph-mon[156819]: pgmap v755: 89 pgs: 89 active+clean; 2.1 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s; 1.3 MiB/s, 20 objects/s recovering 2023-06-10T21:26:09.336 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:09.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:09.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 42502 42486 91.5417 91.625 0.00652845 0.0109183 2023-06-10T21:26:09.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:09.509 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:09.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:26:10.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:10.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:10.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:26:10.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:26:10.351 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 15 43750 43735 91.0923 78.0625 0.0147715 0.0109733 2023-06-10T21:26:10.392 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:10.449 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:10.515 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:10.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:10.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:10.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:10.764 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:10.840 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:10.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:10 smithi064 ceph-mon[180297]: pgmap v756: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:26:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:10 smithi064 ceph-mon[186706]: pgmap v756: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:26:10.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:10.988 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:11.066 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:11.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:11.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:10 smithi152 ceph-mon[156819]: pgmap v756: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 95 MiB/s wr, 1.51k op/s; 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:26:11.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:11.211 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:11.268 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:11.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:11.277 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:44896 10.f e212/196 10:fec0af0e:::benchmark_data_smithi064_312713_object44895:head v 212'2728, mlcod=212'2725)", 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.267529+0000", 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0052927909999999998, 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053252799999999999, 2023-06-10T21:26:11.278 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.267529+0000", 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.267529+0000", 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.267528+0000", 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269166+0000", 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001637788 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269178+0000", 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1823e-05 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269186+0000", 2023-06-10T21:26:11.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8290000000000008e-06 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269227+0000", 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0729999999999998e-05 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269249+0000", 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2643e-05 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:44897 10.7 e212/206 10:e756165c:::benchmark_data_smithi064_312713_object44896:head v 212'2802, mlcod=212'2800)", 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.269199+0000", 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0036224849999999999, 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037569410000000002, 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269199+0000", 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269199+0000", 2023-06-10T21:26:11.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269198+0000", 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269237+0000", 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8676e-05 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269248+0000", 2023-06-10T21:26:11.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0643e-05 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269254+0000", 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2840000000000003e-06 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269314+0000", 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0059000000000002e-05 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.269334+0000", 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0285e-05 2023-06-10T21:26:11.290 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:44907 10.5 10:a3286a7f:::benchmark_data_smithi064_312713_object44906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.270596+0000", 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0022252919999999998, 2023-06-10T21:26:11.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025415469999999999, 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 44907 2023-06-10T21:26:11.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270596+0000", 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270596+0000", 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270589+0000", 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999924 2023-06-10T21:26:11.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270639+0000", 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9913999999999997e-05 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270653+0000", 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3998e-05 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270661+0000", 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9500000000000007e-06 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270710+0000", 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8665999999999998e-05 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270870+0000", 2023-06-10T21:26:11.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159932 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.271027+0000", 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015660000000000001 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.298 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:44909 10.9 10:9a36ef62:::benchmark_data_smithi064_312713_object44908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e212)", 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.270679+0000", 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0021425799999999998, 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002549773, 2023-06-10T21:26:11.299 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 44909 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270679+0000", 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270679+0000", 2023-06-10T21:26:11.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270678+0000", 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270698+0000", 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9559e-05 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270707+0000", 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3910000000000004e-06 2023-06-10T21:26:11.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270714+0000", 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.46e-06 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270777+0000", 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3402e-05 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.270913+0000", 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013575800000000001 2023-06-10T21:26:11.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.271061+0000", 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148417 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:44898 10.7 e212/206)", 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.272660+0000", 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.000161435, 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064971500000000004, 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:26:11.307 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272660+0000", 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272660+0000", 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272659+0000", 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272702+0000", 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3241999999999997e-05 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272714+0000", 2023-06-10T21:26:11.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1439e-05 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272722+0000", 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9559999999999997e-06 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.311 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:44902 10.e e212/210 10:7d0e57ac:::benchmark_data_smithi064_312713_object44901:head v 212'2763, mlcod=212'2761)", 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:11.272741+0000", 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.0128000000000003e-05, 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062642700000000004, 2023-06-10T21:26:11.312 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272741+0000", 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272741+0000", 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:11.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272741+0000", 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272782+0000", 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0655999999999998e-05 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272789+0000", 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7710000000000006e-06 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272795+0000", 2023-06-10T21:26:11.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.733e-06 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272859+0000", 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4350000000000006e-05 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:11.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:11.272881+0000", 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1665e-05 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:11.318 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:26:11.319 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 6 2023-06-10T21:26:11.319 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:26:11.352 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 45013 44997 90.6953 78.875 0.0109175 0.0110205 2023-06-10T21:26:11.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:11.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:12.353 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 46227 46211 90.232 75.875 0.00606333 0.0110766 2023-06-10T21:26:12.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:12 smithi064 ceph-mon[180297]: pgmap v757: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 938 KiB/s, 14 objects/s recovering 2023-06-10T21:26:12.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:12 smithi064 ceph-mon[186706]: pgmap v757: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 938 KiB/s, 14 objects/s recovering 2023-06-10T21:26:12.909 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:12] "GET /metrics HTTP/1.1" 200 223956 "" "Prometheus/2.33.4" 2023-06-10T21:26:13.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:12 smithi152 ceph-mon[156819]: pgmap v757: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 938 KiB/s, 14 objects/s recovering 2023-06-10T21:26:13.352 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 47588 47572 90.0751 85.0625 0.0200836 0.0110943 2023-06-10T21:26:13.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:13.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:13.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:13.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:13.953 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:14.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:26:14.024 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:14.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:14.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:14.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:14.358 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 48873 48857 89.7837 80.3125 0.0109069 0.011132 2023-06-10T21:26:14.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:14.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:14.602 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:14.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:14.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:14.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:14 smithi064 ceph-mon[180297]: pgmap v758: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 934 KiB/s, 14 objects/s recovering 2023-06-10T21:26:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:14 smithi064 ceph-mon[186706]: pgmap v758: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 934 KiB/s, 14 objects/s recovering 2023-06-10T21:26:14.702 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:14.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:26:14.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:26:14.838 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:14.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:14.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:26:14.938 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:26:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:14 smithi152 ceph-mon[156819]: pgmap v758: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s; 934 KiB/s, 14 objects/s recovering 2023-06-10T21:26:14.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:14.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:26:15.020 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:15.033 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:26:15.079 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:26:15.131 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:26:15.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:26:15.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:15.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:15.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:15.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:15 smithi064 conmon[180272]: 2023-06-10T21:26:15.307+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e212 definitely_dead 0 2023-06-10T21:26:15.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:15.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 49981 49965 89.1929 69.25 0.0165859 0.0112062 2023-06-10T21:26:15.594 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.6. 2023-06-10T21:26:15.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3892293626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:15.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2921273712' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[186706]: Client client.admin marked osd.6 out, while it was still marked up 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3892293626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2921273712' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:15 smithi064 ceph-mon[180297]: Client client.admin marked osd.6 out, while it was still marked up 2023-06-10T21:26:15.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3892293626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2921273712' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:15 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-10T21:26:15.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:15 smithi152 ceph-mon[156819]: Client client.admin marked osd.6 out, while it was still marked up 2023-06-10T21:26:16.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 50358 50342 87.37 23.5625 0.126398 0.0112041 2023-06-10T21:26:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.876 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: osdmap e213: 8 total, 8 up, 6 in 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: pgmap v760: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: osdmap e214: 8 total, 8 up, 6 in 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.877 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[186706]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: osdmap e213: 8 total, 8 up, 6 in 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: pgmap v760: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-10T21:26:16.878 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: osdmap e214: 8 total, 8 up, 6 in 2023-06-10T21:26:16.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.879 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:16 smithi064 ceph-mon[180297]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.879 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:16.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-10T21:26:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: osdmap e213: 8 total, 8 up, 6 in 2023-06-10T21:26:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: failed to encode map e213 with expected crc 2023-06-10T21:26:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: pgmap v760: 89 pgs: 89 active+clean; 2.9 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-10T21:26:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: osdmap e214: 8 total, 8 up, 6 in 2023-06-10T21:26:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:16 smithi152 ceph-mon[156819]: failed to encode map e214 with expected crc 2023-06-10T21:26:16.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:17.016 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:17.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:17.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:17.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:17.333 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:17.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 50405 50389 85.0875 2.9375 0.00523191 0.0114483 2023-06-10T21:26:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e214 with expected crc 2023-06-10T21:26:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: osdmap e215: 8 total, 8 up, 6 in 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[180297]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e214 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: osdmap e215: 8 total, 8 up, 6 in 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:17 smithi064 ceph-mon[186706]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e214 with expected crc 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: osdmap e215: 8 total, 8 up, 6 in 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:17.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:17 smithi152 ceph-mon[156819]: failed to encode map e215 with expected crc 2023-06-10T21:26:18.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 50405 50389 82.8489 0 - 0.0114483 2023-06-10T21:26:18.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:18.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:18.664 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:18.749 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:18.775 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:18.821 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:18.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:18.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: pgmap v763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 86 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 6197/145626 objects degraded (4.255%); 0 B/s, 0 objects/s recovering 2023-06-10T21:26:18.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 6197/145626 objects degraded (4.255%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:18.863 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: osdmap e216: 8 total, 8 up, 6 in 2023-06-10T21:26:18.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.864 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:18.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:18.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: pgmap v763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 86 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 6197/145626 objects degraded (4.255%); 0 B/s, 0 objects/s recovering 2023-06-10T21:26:18.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 6197/145626 objects degraded (4.255%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:18.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: osdmap e216: 8 total, 8 up, 6 in 2023-06-10T21:26:18.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:18.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:18.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:26:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: pgmap v763: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 86 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s; 6197/145626 objects degraded (4.255%); 0 B/s, 0 objects/s recovering 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 6197/145626 objects degraded (4.255%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: osdmap e216: 8 total, 8 up, 6 in 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: failed to encode map e216 with expected crc 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:18.964 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:19.024 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:19.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:19.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:19.156 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:19.199 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:19.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:19.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:19.345 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:19.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 50405 50389 80.7196 0 - 0.0114483 2023-06-10T21:26:19.514 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:19.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:19.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:19.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:26:19.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:26:19.673 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:26:19.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[180297]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[180297]: osdmap e217: 8 total, 8 up, 6 in 2023-06-10T21:26:19.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[180297]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1926782660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[186706]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[186706]: osdmap e217: 8 total, 8 up, 6 in 2023-06-10T21:26:19.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[186706]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1926782660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:19 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:19.825 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:26:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:19.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:26:19.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:19.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:19 smithi152 ceph-mon[156819]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:19 smithi152 ceph-mon[156819]: osdmap e217: 8 total, 8 up, 6 in 2023-06-10T21:26:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:19 smithi152 ceph-mon[156819]: failed to encode map e217 with expected crc 2023-06-10T21:26:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1926782660' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:19 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:26:19.957 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:20.000 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:20.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:20.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:20.140 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:20.359 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:26:20.357728+0000 min lat: 0.00121051 max lat: 1.76327 avg lat: 0.0114483 2023-06-10T21:26:20.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:26:20.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 50405 50389 78.7022 0 - 0.0114483 2023-06-10T21:26:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:20 smithi152 ceph-mon[156819]: pgmap v766: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 15 MiB/s wr, 243 op/s; 9290/147735 objects degraded (6.288%); 161 KiB/s, 2 objects/s recovering 2023-06-10T21:26:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:20 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:20 smithi152 ceph-mon[156819]: failed to encode map e218 with expected crc 2023-06-10T21:26:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:20 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:26:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:20 smithi152 ceph-mon[156819]: osdmap e218: 8 total, 8 up, 6 in 2023-06-10T21:26:21.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[180297]: pgmap v766: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 15 MiB/s wr, 243 op/s; 9290/147735 objects degraded (6.288%); 161 KiB/s, 2 objects/s recovering 2023-06-10T21:26:21.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:21.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[180297]: failed to encode map e218 with expected crc 2023-06-10T21:26:21.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:26:21.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[180297]: osdmap e218: 8 total, 8 up, 6 in 2023-06-10T21:26:21.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[186706]: pgmap v766: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 85 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 15 MiB/s wr, 243 op/s; 9290/147735 objects degraded (6.288%); 161 KiB/s, 2 objects/s recovering 2023-06-10T21:26:21.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:21.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[186706]: failed to encode map e218 with expected crc 2023-06-10T21:26:21.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:26:21.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:20 smithi064 ceph-mon[186706]: osdmap e218: 8 total, 8 up, 6 in 2023-06-10T21:26:21.359 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 50405 50389 76.7831 0 - 0.0114483 2023-06-10T21:26:21.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:21.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:21.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:21.852 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:21.903 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:21.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:21.920 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:22.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:22.041 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:22.102 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:22.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:22.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:22.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:22.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 51161 51145 76.0801 9.45 0.0250883 0.0131374 2023-06-10T21:26:22.468 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:22.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:22.512 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:26:22.513 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:26:22.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:22.639 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:22.812 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:22.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:22.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:22 smithi064 ceph-mon[186706]: pgmap v768: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 18831/151260 objects degraded (12.449%); 2650/151260 objects misplaced (1.752%); 6.2 MiB/s, 99 objects/s recovering 2023-06-10T21:26:22.910 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:22] "GET /metrics HTTP/1.1" 200 223986 "" "Prometheus/2.33.4" 2023-06-10T21:26:22.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:22 smithi064 ceph-mon[180297]: pgmap v768: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 18831/151260 objects degraded (12.449%); 2650/151260 objects misplaced (1.752%); 6.2 MiB/s, 99 objects/s recovering 2023-06-10T21:26:22.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:22 smithi152 ceph-mon[156819]: pgmap v768: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 26 MiB/s wr, 414 op/s; 18831/151260 objects degraded (12.449%); 2650/151260 objects misplaced (1.752%); 6.2 MiB/s, 99 objects/s recovering 2023-06-10T21:26:22.953 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:23.008 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:23.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:23.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:23.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:23.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 51985 51969 75.5084 51.5 0.00359787 0.0132262 2023-06-10T21:26:23.651 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:26:23.651 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:26:23.651 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-10T21:26:23.652 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-06-10T21:26:23.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3 2023-06-10T21:26:23.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:26:24.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:23 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18831/151497 objects degraded (12.430%), 6 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:24.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:23 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18831/151497 objects degraded (12.430%), 6 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:24.160 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:26:23 smithi064 systemd[1]: Stopping Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:26:24.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:23 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18831/151497 objects degraded (12.430%), 6 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:24.360 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 15 52765 52750 74.9017 48.8125 0.0100182 0.0133455 2023-06-10T21:26:24.518 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:24.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:24.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:24.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:24 smithi064 ceph-mon[180297]: pgmap v769: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 18831/151497 objects degraded (12.430%); 2650/151497 objects misplaced (1.749%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:26:24.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:24 smithi064 ceph-mon[186706]: pgmap v769: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 18831/151497 objects degraded (12.430%); 2650/151497 objects misplaced (1.749%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:26:25.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:25.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:25.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:25.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:24 smithi152 ceph-mon[156819]: pgmap v769: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 20 MiB/s wr, 325 op/s; 18831/151497 objects degraded (12.430%); 2650/151497 objects misplaced (1.749%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:26:25.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:25.327 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:25.361 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 53601 53585 74.397 52.1875 0.0264307 0.0134366 2023-06-10T21:26:25.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:26:25.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:25.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:25.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:25.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:25.909 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:25.960 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:25.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:26.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:26.160 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:26.160 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:26:25 smithi064 conmon[227562]: 2023-06-10T21:26:25.871+0000 7f8f0f669700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:26:26.161 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:26:25 smithi064 conmon[227562]: 2023-06-10T21:26:25.871+0000 7f8f0f669700 -1 osd.3 218 *** Got signal Terminated *** 2023-06-10T21:26:26.161 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:26:25 smithi064 conmon[227562]: 2023-06-10T21:26:25.871+0000 7f8f0f669700 -1 osd.3 218 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:26:26.173 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:26.298 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:26.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:26.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 54248 54232 73.6556 40.4375 0.0212352 0.0135691 2023-06-10T21:26:26.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:26.442 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:26.496 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:26.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:26.602 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:26.629 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:26.680 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:26.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:26.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:26.698 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:26:26.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:26:26.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:26.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[180297]: pgmap v770: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21 MiB/s wr, 341 op/s; 18831/153762 objects degraded (12.247%); 2650/153762 objects misplaced (1.723%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:26:26.799 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[180297]: osd.3 marked itself down and dead 2023-06-10T21:26:26.800 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/815065111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:26.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[186706]: pgmap v770: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21 MiB/s wr, 341 op/s; 18831/153762 objects degraded (12.247%); 2650/153762 objects misplaced (1.723%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:26:26.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[186706]: osd.3 marked itself down and dead 2023-06-10T21:26:26.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/815065111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:26.805 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:26:26.814 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:26.857 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:26.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:26.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:26.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:26.990 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:27.033 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:27.037 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:26 smithi152 ceph-mon[156819]: pgmap v770: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21 MiB/s wr, 341 op/s; 18831/153762 objects degraded (12.247%); 2650/153762 objects misplaced (1.723%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:26:27.037 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:26 smithi152 ceph-mon[156819]: osd.3 marked itself down and dead 2023-06-10T21:26:27.037 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/815065111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:27.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:27.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:27.165 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:27.208 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:27.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:27.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:27.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:27.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 54581 54565 72.5316 20.8125 0.00742431 0.0136155 2023-06-10T21:26:27.406 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:26:27.425 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:26:27.444 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:26:27.463 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:26:27.463 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.463 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.464 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:26:27.476 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:28.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:26:28.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[186706]: osdmap e219: 8 total, 7 up, 6 in 2023-06-10T21:26:28.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[186706]: failed to encode map e219 with expected crc 2023-06-10T21:26:28.158 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:26:27 smithi064 bash[321993]: b4b26d4dbd297e03c4c2f3e53deb45e420a13c860b9abef5343cd9ced857edcd 2023-06-10T21:26:28.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:26:28.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[180297]: osdmap e219: 8 total, 7 up, 6 in 2023-06-10T21:26:28.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:27 smithi064 ceph-mon[180297]: failed to encode map e219 with expected crc 2023-06-10T21:26:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:27 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:26:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:27 smithi152 ceph-mon[156819]: osdmap e219: 8 total, 7 up, 6 in 2023-06-10T21:26:28.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:27 smithi152 ceph-mon[156819]: failed to encode map e219 with expected crc 2023-06-10T21:26:28.362 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 54590 54574 71.0327 0.5625 0.00342998 0.0136699 2023-06-10T21:26:29.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: pgmap v772: 89 pgs: 2 stale+active+recovery_wait+undersized+degraded+remapped, 56 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 26 stale+active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 31 MiB/s wr, 499 op/s; 18824/159723 objects degraded (11.785%); 2316/159723 objects misplaced (1.450%); 6.4 MiB/s, 102 objects/s recovering 2023-06-10T21:26:29.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: osdmap e220: 8 total, 7 up, 6 in 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:29.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: pgmap v772: 89 pgs: 2 stale+active+recovery_wait+undersized+degraded+remapped, 56 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 26 stale+active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 31 MiB/s wr, 499 op/s; 18824/159723 objects degraded (11.785%); 2316/159723 objects misplaced (1.450%); 6.4 MiB/s, 102 objects/s recovering 2023-06-10T21:26:29.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: osdmap e220: 8 total, 7 up, 6 in 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:28 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:29.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: pgmap v772: 89 pgs: 2 stale+active+recovery_wait+undersized+degraded+remapped, 56 active+clean, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 26 stale+active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 31 MiB/s wr, 499 op/s; 18824/159723 objects degraded (11.785%); 2316/159723 objects misplaced (1.450%); 6.4 MiB/s, 102 objects/s recovering 2023-06-10T21:26:29.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: osdmap e220: 8 total, 7 up, 6 in 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: failed to encode map e220 with expected crc 2023-06-10T21:26:29.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:28 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:29.363 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 55152 55136 70.299 35.125 0.0150261 0.0142194 2023-06-10T21:26:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:30.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[180297]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[180297]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[180297]: osdmap e221: 8 total, 7 up, 6 in 2023-06-10T21:26:30.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[180297]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[180297]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[186706]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[186706]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[186706]: osdmap e221: 8 total, 7 up, 6 in 2023-06-10T21:26:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[186706]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:29 smithi064 ceph-mon[186706]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:29 smithi152 ceph-mon[156819]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:29 smithi152 ceph-mon[156819]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:29 smithi152 ceph-mon[156819]: osdmap e221: 8 total, 7 up, 6 in 2023-06-10T21:26:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:29 smithi152 ceph-mon[156819]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:29 smithi152 ceph-mon[156819]: failed to encode map e221 with expected crc 2023-06-10T21:26:30.363 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 56308 56292 70.3378 72.25 0.00468742 0.0142107 2023-06-10T21:26:30.936 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:30.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:30.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:30 smithi064 ceph-mon[180297]: pgmap v775: 89 pgs: 3 remapped+peering, 44 active+clean, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 23 active+undersized, 1 active+recovering+undersized+remapped, 9 stale+active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 42 MiB/s wr, 677 op/s; 16167/163692 objects degraded (9.876%); 2316/163692 objects misplaced (1.415%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:26:30.947 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:30 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 16167/163692 objects degraded (9.876%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:30.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:30 smithi064 ceph-mon[186706]: pgmap v775: 89 pgs: 3 remapped+peering, 44 active+clean, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 23 active+undersized, 1 active+recovering+undersized+remapped, 9 stale+active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 42 MiB/s wr, 677 op/s; 16167/163692 objects degraded (9.876%); 2316/163692 objects misplaced (1.415%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:26:30.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:30 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 16167/163692 objects degraded (9.876%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:31.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:31.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:31.091 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:30 smithi152 ceph-mon[156819]: pgmap v775: 89 pgs: 3 remapped+peering, 44 active+clean, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 23 active+undersized, 1 active+recovering+undersized+remapped, 9 stale+active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 42 MiB/s wr, 677 op/s; 16167/163692 objects degraded (9.876%); 2316/163692 objects misplaced (1.415%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:26:31.091 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:30 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 16167/163692 objects degraded (9.876%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:26:31.134 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:26:31.161 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:31.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:31.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:31.319 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:31.363 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 57308 57292 70.1841 62.5 0.0267714 0.0142415 2023-06-10T21:26:31.396 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:31.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:31.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:31.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:31.549 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:31.611 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:31.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:31.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:31.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:32.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:32.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:32.363 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 15 58352 58337 70.0903 65.3125 0.02116 0.0142617 2023-06-10T21:26:32.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:32.433 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:32.512 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:32.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:32.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:32.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:32.831 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:26:33.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:26:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:32] "GET /metrics HTTP/1.1" 200 223986 "" "Prometheus/2.33.4" 2023-06-10T21:26:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[186706]: pgmap v776: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 12 MiB/s, 193 objects/s recovering 2023-06-10T21:26:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/5048812' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[180297]: pgmap v776: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 12 MiB/s, 193 objects/s recovering 2023-06-10T21:26:33.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/5048812' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:32 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:32 smithi152 ceph-mon[156819]: pgmap v776: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 51 MiB/s wr, 808 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 12 MiB/s, 193 objects/s recovering 2023-06-10T21:26:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/5048812' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:32 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:33.365 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 59401 59385 70.0028 65.5 0.0104699 0.01428 2023-06-10T21:26:33.476 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:33.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:33.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:33.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:33.641 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:33.721 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:33.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:33.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:33.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:33.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: failed to encode map e222 with expected crc 2023-06-10T21:26:33.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:33.890 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: osdmap e222: 8 total, 7 up, 6 in 2023-06-10T21:26:33.891 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: failed to encode map e222 with expected crc 2023-06-10T21:26:33.891 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:33.891 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:33.898 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:33.969 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:33.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:34.073 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:34.099 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:34.141 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:34.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:34.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:34.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: failed to encode map e222 with expected crc 2023-06-10T21:26:34.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:34.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: osdmap e222: 8 total, 7 up, 6 in 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: failed to encode map e222 with expected crc 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: failed to encode map e222 with expected crc 2023-06-10T21:26:34.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:34.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: osdmap e222: 8 total, 7 up, 6 in 2023-06-10T21:26:34.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: failed to encode map e222 with expected crc 2023-06-10T21:26:34.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:26:34.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:34.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:34.285 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:34.347 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:34.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:34.364 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 60297 60281 69.7435 56 0.0129951 0.0143333 2023-06-10T21:26:34.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:34.533 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:34.683 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:34.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:34.797 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:34.842 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:34.906 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:34.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:35.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:35.114 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:35.129 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:35.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:35.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:35.253 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:35 smithi064 ceph-mon[180297]: pgmap v778: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 30 MiB/s wr, 477 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 8.6 MiB/s, 136 objects/s recovering 2023-06-10T21:26:35.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:35.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:35 smithi064 ceph-mon[186706]: pgmap v778: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 30 MiB/s wr, 477 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 8.6 MiB/s, 136 objects/s recovering 2023-06-10T21:26:35.289 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13232 10.9 10:9012193c:::benchmark_data_smithi064_312713_object13231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e201)", 2023-06-10T21:26:35.290 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.018835805999998, 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7512189839999999, 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.291 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.292 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.292 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13232 2023-06-10T21:26:35.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.292 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264126+0000", 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264155+0000", 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8109000000000001e-05 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264178+0000", 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3536999999999998e-05 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264189+0000", 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0995000000000001e-05 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264267+0000", 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7470000000000002e-05 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264279+0000", 2023-06-10T21:26:35.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2045000000000001e-05 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614347+0000", 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35006879200000002 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614373+0000", 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5544000000000001e-05 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.003258+0000", 2023-06-10T21:26:35.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3888854589999999 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.004667+0000", 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014090960000000001 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015232+0000", 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010564163999999999 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015348+0000", 2023-06-10T21:26:35.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115995 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13262 10.9 10:9b8ca29e:::benchmark_data_smithi064_312713_object13261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:26:35.302 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.001515152000003, 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7392644879999999, 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.303 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13262 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281448+0000", 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281480+0000", 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2228999999999999e-05 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281490+0000", 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8079999999999999e-06 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281496+0000", 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7790000000000001e-06 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281565+0000", 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8689999999999995e-05 2023-06-10T21:26:35.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281579+0000", 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4499e-05 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614432+0000", 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33285274500000001 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614439+0000", 2023-06-10T21:26:35.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.007692+0000", 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.393252581 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.009132+0000", 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014407669999999999 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.011401+0000", 2023-06-10T21:26:35.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002268927 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015111+0000", 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037098830000000002 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015417+0000", 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030584399999999999 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020055+0000", 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046377069999999996 2023-06-10T21:26:35.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020100+0000", 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5522999999999998e-05 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020714+0000", 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00061340799999999997 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.316 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13275 10.9 10:9047ac4e:::benchmark_data_smithi064_312713_object13274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 49.993623984999999, 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7361003699999999, 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.317 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13275 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:26:35.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289339+0000", 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289384+0000", 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5176000000000003e-05 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289404+0000", 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0273999999999999e-05 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289414+0000", 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028e-05 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289471+0000", 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6205999999999997e-05 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289480+0000", 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2690000000000005e-06 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614466+0000", 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32498670400000002 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614474+0000", 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8569999999999995e-06 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015439+0000", 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.400964885 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015551+0000", 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111486 2023-06-10T21:26:35.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015728+0000", 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000177412 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020768+0000", 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050401889999999996 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020809+0000", 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1056000000000001e-05 2023-06-10T21:26:35.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024793+0000", 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003983444 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024844+0000", 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1477000000000002e-05 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025441+0000", 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00059653199999999999 2023-06-10T21:26:35.330 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13295 10.9 10:9b4c8fff:::benchmark_data_smithi064_312713_object13294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.684273+0000", 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.598691524000003, 2023-06-10T21:26:35.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34175610499999998, 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13295 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.332 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684273+0000", 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684273+0000", 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684272+0000", 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:35.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684296+0000", 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3232999999999999e-05 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684301+0000", 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8649999999999998e-06 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.684308+0000", 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2940000000000002e-06 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.993381+0000", 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30907352599999999 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.993522+0000", 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014054700000000001 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.016549+0000", 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023027107000000002 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.016706+0000", 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015697000000000001 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.016884+0000", 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017842699999999999 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020849+0000", 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039643029999999998 2023-06-10T21:26:35.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025567+0000", 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047179509999999997 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025678+0000", 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111569 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025733+0000", 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4471000000000001e-05 2023-06-10T21:26:35.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026029+0000", 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029638199999999998 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13338 10.9 10:9a9eda69:::benchmark_data_smithi064_312713_object13337:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.727564+0000", 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.555400509999998, 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29888168799999998, 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.344 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13338 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727564+0000", 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727564+0000", 2023-06-10T21:26:35.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727562+0000", 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727590+0000", 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7707000000000001e-05 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727601+0000", 2023-06-10T21:26:35.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1021e-05 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.727610+0000", 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4739999999999994e-06 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.993699+0000", 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26608901099999999 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.993872+0000", 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017312499999999999 2023-06-10T21:26:35.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.017561+0000", 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023689116999999999 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.017709+0000", 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014717800000000001 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.017885+0000", 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000176727 2023-06-10T21:26:35.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020888+0000", 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030025999999999998 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025572+0000", 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046838549999999998 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026074+0000", 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050201100000000002 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026127+0000", 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3528e-05 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026446+0000", 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031828899999999997 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13377 10.9 10:9223ccce:::benchmark_data_smithi064_312713_object13376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.356 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.743589+0000", 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.539375702000001, 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28325833700000003, 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.357 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13377 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743589+0000", 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743589+0000", 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743588+0000", 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743613+0000", 2023-06-10T21:26:35.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4352999999999999e-05 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743620+0000", 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6329999999999996e-06 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.743626+0000", 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9159999999999998e-06 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.994047+0000", 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.250420902 2023-06-10T21:26:35.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.994167+0000", 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119486 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.018344+0000", 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024177278 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.018486+0000", 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014212099999999999 2023-06-10T21:26:35.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.018653+0000", 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000167381 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020930+0000", 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00227648 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025578+0000", 2023-06-10T21:26:35.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046485040000000004 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026491+0000", 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000912644 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026538+0000", 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7002000000000002e-05 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026847+0000", 2023-06-10T21:26:35.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030913099999999998 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13383 10.9 10:9525f0ff:::benchmark_data_smithi064_312713_object13382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.369 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.747081+0000", 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.535883722999998, 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28018618200000001, 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.370 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13383 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747081+0000", 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747081+0000", 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747079+0000", 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747102+0000", 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2819999999999998e-05 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747114+0000", 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2261e-05 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.747123+0000", 2023-06-10T21:26:35.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5930000000000006e-06 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.994337+0000", 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24721422600000001 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.994496+0000", 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015884499999999999 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.019135+0000", 2023-06-10T21:26:35.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024639742999999999 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.019267+0000", 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013142899999999999 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.019464+0000", 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000197484 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.022801+0000", 2023-06-10T21:26:35.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033365550000000002 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025623+0000", 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002821877 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026888+0000", 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001265394 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.026937+0000", 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8746999999999999e-05 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.027267+0000", 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000329921 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.381 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13400 10.9 10:9ff89347:::benchmark_data_smithi064_312713_object13399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.756451+0000", 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.526513942999998, 2023-06-10T21:26:35.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27516962700000003, 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13400 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.383 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756451+0000", 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756451+0000", 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756450+0000", 2023-06-10T21:26:35.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756475+0000", 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5375e-05 2023-06-10T21:26:35.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756491+0000", 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5221999999999999e-05 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.756496+0000", 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3510000000000001e-06 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020961+0000", 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26446491500000002 2023-06-10T21:26:35.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.021103+0000", 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000142082 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.021275+0000", 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000171688 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.022840+0000", 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015657640000000001 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025628+0000", 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027877589999999999 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031010+0000", 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053820609999999996 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031062+0000", 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2102999999999999e-05 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031620+0000", 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000557793 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13403 10.9 10:9013f954:::benchmark_data_smithi064_312713_object13402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.757669+0000", 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.525295923000002, 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27441824799999998, 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.394 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13403 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757669+0000", 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757669+0000", 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757668+0000", 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757691+0000", 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3507e-05 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757703+0000", 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.185e-05 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.757710+0000", 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1099999999999997e-06 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.021988+0000", 2023-06-10T21:26:35.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26427800400000001 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.022131+0000", 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014307799999999999 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.022291+0000", 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159798 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024725+0000", 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024336710000000001 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025634+0000", 2023-06-10T21:26:35.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00090936399999999996 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031713+0000", 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060790990000000001 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031758+0000", 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5385e-05 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032087+0000", 2023-06-10T21:26:35.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032830799999999997 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13410 10.9 10:92112788:::benchmark_data_smithi064_312713_object13409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.760581+0000", 2023-06-10T21:26:35.405 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.522383877000003, 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27193989800000001, 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.406 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13410 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760581+0000", 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760581+0000", 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760579+0000", 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760602+0000", 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2799e-05 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760612+0000", 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0199e-05 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.760620+0000", 2023-06-10T21:26:35.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3130000000000001e-06 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.022865+0000", 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26224521299999998 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.023045+0000", 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179789 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.023209+0000", 2023-06-10T21:26:35.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164346 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024768+0000", 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015588799999999999 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025640+0000", 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00087201200000000003 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032164+0000", 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065246330000000002 2023-06-10T21:26:35.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032221+0000", 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6264999999999997e-05 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032520+0000", 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029976599999999998 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13431 10.9 10:9b534e28:::benchmark_data_smithi064_312713_object13430:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.774042+0000", 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.508922169000002, 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25884947800000002, 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.417 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13431 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774042+0000", 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774042+0000", 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774041+0000", 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774063+0000", 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.192e-05 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774072+0000", 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2469999999999997e-06 2023-06-10T21:26:35.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.774078+0000", 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.631e-06 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.023825+0000", 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.249747102 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.023989+0000", 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016439699999999999 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024165+0000", 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000175957 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025615+0000", 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001449393 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025645+0000", 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0434e-05 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032565+0000", 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069200049999999999 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032607+0000", 2023-06-10T21:26:35.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2040999999999999e-05 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032892+0000", 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028462099999999998 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.427 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13444 10.9 10:973763b2:::benchmark_data_smithi064_312713_object13443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:46.780949+0000", 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.502015346999997, 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25235870100000002, 2023-06-10T21:26:35.428 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13444 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.429 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780949+0000", 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780949+0000", 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780948+0000", 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780970+0000", 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1654999999999998e-05 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780984+0000", 2023-06-10T21:26:35.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4293e-05 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:46.780992+0000", 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1200000000000002e-06 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.027407+0000", 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24641533099999999 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.027592+0000", 2023-06-10T21:26:35.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000185179 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.027821+0000", 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022859700000000001 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031635+0000", 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038135320000000001 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.031680+0000", 2023-06-10T21:26:35.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5006000000000002e-05 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032936+0000", 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001256629 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.032990+0000", 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3359999999999997e-05 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.033308+0000", 2023-06-10T21:26:35.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031818599999999998 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54555 10.7 10:e3cbd15b:::benchmark_data_smithi064_312713_object54554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:26:35.439 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4783244040000003, 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1739721809999999, 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.440 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54555 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804638+0000", 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804665+0000", 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6137000000000001e-05 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804675+0000", 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0899e-05 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804691+0000", 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5661000000000001e-05 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804758+0000", 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6879999999999997e-05 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804769+0000", 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1028e-05 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818008+0000", 2023-06-10T21:26:35.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0132391670000001 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818017+0000", 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3829999999999992e-06 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.955590+0000", 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375735200000001 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.956353+0000", 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000762807 2023-06-10T21:26:35.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978548+0000", 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022194875999999999 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978612+0000", 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4449999999999994e-05 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.450 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54557 10.f 10:f1e22a0f:::benchmark_data_smithi064_312713_object54556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4782393329999994, 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1819947270000002, 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.451 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54557 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804724+0000", 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804750+0000", 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5415999999999999e-05 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804759+0000", 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5969999999999998e-06 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804767+0000", 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804793+0000", 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5993e-05 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804800+0000", 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1019999999999999e-06 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818039+0000", 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013239368 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818047+0000", 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5510000000000002e-06 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.977925+0000", 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159878645 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978506+0000", 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058025500000000001 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986600+0000", 2023-06-10T21:26:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080940130000000006 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986720+0000", 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120254 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54569 10.3 10:c761f176:::benchmark_data_smithi064_312713_object54568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4774230900000003, 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0199541459999999, 2023-06-10T21:26:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54569 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805540+0000", 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805563+0000", 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2124e-05 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805572+0000", 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2550000000000003e-06 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805580+0000", 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7929999999999998e-06 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805621+0000", 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1746999999999999e-05 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805631+0000", 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3880000000000006e-06 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822642+0000", 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0170114130000001 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822728+0000", 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6032999999999994e-05 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822844+0000", 2023-06-10T21:26:35.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011603300000000001 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.824327+0000", 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014828739999999999 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825354+0000", 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001026875 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825416+0000", 2023-06-10T21:26:35.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1940999999999995e-05 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825496+0000", 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9591000000000004e-05 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.473 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54574 10.5 10:a853286f:::benchmark_data_smithi064_312713_object54573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4765923169999997, 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0150581569999999, 2023-06-10T21:26:35.474 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54574 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:26:35.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806370+0000", 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806397+0000", 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6996999999999999e-05 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806413+0000", 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6099999999999998e-05 2023-06-10T21:26:35.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806422+0000", 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.833e-06 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809434+0000", 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030114529999999999 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809442+0000", 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1799999999999996e-06 2023-06-10T21:26:35.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818062+0000", 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0086199140000001 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818198+0000", 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013567700000000001 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0", 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818376+0000", 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000178813 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.820582+0000", 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022051589999999999 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821290+0000", 2023-06-10T21:26:35.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070819899999999996 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821344+0000", 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4304e-05 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821430+0000", 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6263999999999999e-05 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54575 10.e 10:708b6ab1:::benchmark_data_smithi064_312713_object54574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4672381960000003, 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0012291179999999, 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.486 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54575 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815725+0000", 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815752+0000", 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7001000000000001e-05 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815769+0000", 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7203000000000001e-05 2023-06-10T21:26:35.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815779+0000", 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6020000000000006e-06 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820630+0000", 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048513219999999999 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820642+0000", 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2081e-05 2023-06-10T21:26:35.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817919+0000", 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99727731900000005 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817927+0000", 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7549999999999994e-06 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810577+0000", 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99265023100000005 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810664+0000", 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6879999999999995e-05 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:26:35.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810771+0000", 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106632 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816578+0000", 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058069870000000004 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816627+0000", 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8791000000000001e-05 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816709+0000", 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2793000000000005e-05 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816771+0000", 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1524000000000003e-05 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816955+0000", 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018453799999999999 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54576 10.f 10:f052f063:::benchmark_data_smithi064_312713_object54575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4621451570000001, 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1709000860000001, 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54576 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820817+0000", 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820849+0000", 2023-06-10T21:26:35.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1776000000000002e-05 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820866+0000", 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6892999999999999e-05 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820875+0000", 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5739999999999996e-06 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820944+0000", 2023-06-10T21:26:35.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9340999999999999e-05 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820957+0000", 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3226e-05 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818068+0000", 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99711057199999997 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818077+0000", 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7480000000000005e-06 2023-06-10T21:26:35.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978558+0000", 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1604804230000001 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979614+0000", 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001056162 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979766+0000", 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015153399999999999 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.984553+0000", 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047875230000000001 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986763+0000", 2023-06-10T21:26:35.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022097850000000001 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990555+0000", 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037920319999999999 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990588+0000", 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3130999999999998e-05 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.991719+0000", 2023-06-10T21:26:35.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001131331 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54580 10.e 10:70306020:::benchmark_data_smithi064_312713_object54579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:26:35.513 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.4541558739999996, 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9886503549999999, 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54580 2023-06-10T21:26:35.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828807+0000", 2023-06-10T21:26:35.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828835+0000", 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7953000000000002e-05 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828857+0000", 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2255999999999998e-05 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828867+0000", 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9520000000000006e-06 2023-06-10T21:26:35.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828928+0000", 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1260999999999995e-05 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828938+0000", 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0195e-05 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817952+0000", 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98901349999999999 2023-06-10T21:26:35.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817959+0000", 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3690000000000001e-06 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812558+0000", 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.994598918 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812679+0000", 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012119499999999999 2023-06-10T21:26:35.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812787+0000", 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010732 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816636+0000", 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038490880000000001 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817109+0000", 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047353 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817227+0000", 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011818 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817283+0000", 2023-06-10T21:26:35.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5247000000000001e-05 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817459+0000", 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017632800000000001 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.526 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54582 10.e 10:78d4d384:::benchmark_data_smithi064_312713_object54581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 7.4935657969999996, 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028370658, 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:26:35.527 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54582 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:26:35.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:26:35.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789397+0000", 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789430+0000", 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3229999999999999e-05 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813125+0000", 2023-06-10T21:26:35.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023694667999999999 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813150+0000", 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5534999999999999e-05 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817984+0000", 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048335940000000001 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817990+0000", 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8869999999999997e-06 2023-06-10T21:26:35.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814293+0000", 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99630294900000005 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:26:35.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814412+0000", 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011934099999999999 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814525+0000", 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011297400000000001 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816642+0000", 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021171509999999998 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817165+0000", 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052269299999999996 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817520+0000", 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035547299999999998 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817583+0000", 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090999999999999e-05 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817769+0000", 2023-06-10T21:26:35.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018677799999999999 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:26:35.539 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:26:35.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:35.541 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:35 smithi152 ceph-mon[156819]: pgmap v778: 89 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 30 MiB/s wr, 477 op/s; 15815/168321 objects degraded (9.396%); 6744/168321 objects misplaced (4.007%); 8.6 MiB/s, 136 objects/s recovering 2023-06-10T21:26:35.541 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 61277 61261 69.589 61.25 0.0117207 0.0143647 2023-06-10T21:26:35.617 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:36.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 62369 62353 69.5633 68.25 0.0108002 0.0143701 2023-06-10T21:26:36.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15815/168321 objects degraded (9.396%), 11 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:36.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15815/168321 objects degraded (9.396%), 11 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:36.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15815/168321 objects degraded (9.396%), 11 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:36.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:26:37.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 63510 63494 69.5939 71.3125 0.0165565 0.0143638 2023-06-10T21:26:37.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:37 smithi064 ceph-mon[180297]: pgmap v779: 89 pgs: 1 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 23396/175926 objects degraded (13.299%); 17016/175926 objects misplaced (9.672%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:26:37.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:37 smithi064 ceph-mon[186706]: pgmap v779: 89 pgs: 1 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 23396/175926 objects degraded (13.299%); 17016/175926 objects misplaced (9.672%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:26:37.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:37 smithi152 ceph-mon[156819]: pgmap v779: 89 pgs: 1 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 43 MiB/s wr, 690 op/s; 23396/175926 objects degraded (13.299%); 17016/175926 objects misplaced (9.672%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:26:37.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:37.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:37.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:37.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:38.037 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:38.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3.service' 2023-06-10T21:26:38.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:26:38.266 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:26:38.267 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-06-10T21:26:38.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 64631 64615 69.6018 70.0625 0.017189 0.0143614 2023-06-10T21:26:38.403 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:38.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:38.510 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:38.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:38.671 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:38.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:38.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:38.818 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:38.863 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:38.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:38.914 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:38.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:38.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:39.092 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:39.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:39.197 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:39.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:39 smithi064 ceph-mon[186706]: pgmap v780: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 20118/183540 objects degraded (10.961%); 20999/183540 objects misplaced (11.441%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:26:39.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:39 smithi064 ceph-mon[180297]: pgmap v780: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 20118/183540 objects degraded (10.961%); 20999/183540 objects misplaced (11.441%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:26:39.258 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:39.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 65504 65488 69.3469 54.5625 0.0152918 0.0144156 2023-06-10T21:26:39.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:39.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:39 smithi152 ceph-mon[156819]: pgmap v780: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 20118/183540 objects degraded (10.961%); 20999/183540 objects misplaced (11.441%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:26:39.454 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:39.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:39.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:39.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:39.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:40.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:40.111 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:40.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:26:40.239 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:26:40.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:26:40.324 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:26:40.360 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:26:40.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:26:40.364352+0000 min lat: 0.00121051 max lat: 5.9617 avg lat: 0.0144417 2023-06-10T21:26:40.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:26:40.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 66491 66475 69.2192 61.6875 0.0179105 0.0144417 2023-06-10T21:26:40.373 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:40.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:40.399 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:26:40.436 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:26:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:26:40.437 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.438 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.438 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.438 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:26:40.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/69786558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:40.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/69786558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:40.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/69786558' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:40.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:40.556 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:40.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:40.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:40.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:40.823 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:40.867 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:40.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:40.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:40.996 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:41.038 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:41.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:41.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:41.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:41 smithi152 ceph-mon[156819]: pgmap v781: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 20797/191376 objects degraded (10.867%); 21947/191376 objects misplaced (11.468%); 7.6 MiB/s, 121 objects/s recovering 2023-06-10T21:26:41.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20797/191376 objects degraded (10.867%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:41.179 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:41.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 67071 67055 68.6788 36.25 0.0192522 0.0145546 2023-06-10T21:26:41.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:41 smithi064 ceph-mon[186706]: pgmap v781: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 20797/191376 objects degraded (10.867%); 21947/191376 objects misplaced (11.468%); 7.6 MiB/s, 121 objects/s recovering 2023-06-10T21:26:41.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20797/191376 objects degraded (10.867%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:41.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:41 smithi064 ceph-mon[180297]: pgmap v781: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 3.9 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 922 op/s; 20797/191376 objects degraded (10.867%); 21947/191376 objects misplaced (11.468%); 7.6 MiB/s, 121 objects/s recovering 2023-06-10T21:26:41.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20797/191376 objects degraded (10.867%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:42.210 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:26:42.210 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:26:42.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 68012 67996 68.5196 58.8125 0.0208417 0.0145899 2023-06-10T21:26:42.466 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:42.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:26:42.555 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:26:42.571 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:42.601 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:42.645 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:42.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:42.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:42.824 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:42.944 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:42.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:42.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:43.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:43.063 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:26:43.108 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:42] "GET /metrics HTTP/1.1" 200 224046 "" "Prometheus/2.33.4" 2023-06-10T21:26:43.208 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:43.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:43.268 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-06-10T21:26:43.268 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:26:43.269 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-06-10T21:26:43.269 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:26:43.269 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:26:43.269 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-06-10T21:26:43.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:43.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 68840 68824 68.2534 51.75 0.0246449 0.0146445 2023-06-10T21:26:43.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:43.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:43 smithi152 ceph-mon[156819]: pgmap v782: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:43.515 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:43.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:43.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:43.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:43 smithi064 ceph-mon[180297]: pgmap v782: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:43.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:43 smithi064 ceph-mon[186706]: pgmap v782: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.00k op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:43.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:43.674 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:43.692 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:43.803 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:43.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:43.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:43.928 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:43.960 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:44.037 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:44.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:44.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:44.199 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:26:44.200 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:26:44.216 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:26:44.217 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:26:44.229 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:44.366 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 69799 69783 68.1234 59.9375 0.0163818 0.0146735 2023-06-10T21:26:44.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:44.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:44.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:44.541 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:44.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:26:44.568 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:44.612 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:44.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:44.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:44.768 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:45.028 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:45.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:45.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:45.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:45.197 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: pgmap v783: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 931 op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:26:45.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:45 smithi152 ceph-mon[156819]: osdmap e223: 8 total, 7 up, 6 in 2023-06-10T21:26:45.367 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 70682 70666 67.9235 55.1875 0.0178651 0.0147169 2023-06-10T21:26:45.478 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:45.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:45.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: pgmap v783: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 931 op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:26:45.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:45.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:26:45.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[180297]: osdmap e223: 8 total, 7 up, 6 in 2023-06-10T21:26:45.485 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: pgmap v783: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.0 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 58 MiB/s wr, 931 op/s; 21101/198486 objects degraded (10.631%); 22324/198486 objects misplaced (11.247%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:26:45.485 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:26:45.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: failed to encode map e223 with expected crc 2023-06-10T21:26:45.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/673083571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:26:45.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:45 smithi064 ceph-mon[186706]: osdmap e223: 8 total, 7 up, 6 in 2023-06-10T21:26:45.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:45.647 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:26:45.648 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:26:45.651 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:45.897 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:45.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:46.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:46.066 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:46.368 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 71769 71753 67.9236 67.9375 0.0347192 0.014716 2023-06-10T21:26:46.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21101/198486 objects degraded (10.631%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:46.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21101/198486 objects degraded (10.631%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:46.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21101/198486 objects degraded (10.631%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:46.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:26:46.640 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:46.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:46.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:46.787 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:46.836 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:46.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:46.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:47.013 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:47.031 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:47.260 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:47 smithi064 ceph-mon[180297]: pgmap v785: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 954 op/s; 21608/204570 objects degraded (10.563%); 23097/204570 objects misplaced (11.291%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:47.261 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:47 smithi064 ceph-mon[186706]: pgmap v785: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 954 op/s; 21608/204570 objects degraded (10.563%); 23097/204570 objects misplaced (11.291%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:47.312 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:47.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:47.368 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 72633 72617 67.7158 54 0.0146801 0.0147625 2023-06-10T21:26:47.400 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:47.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:47.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:47 smithi152 ceph-mon[156819]: pgmap v785: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 954 op/s; 21608/204570 objects degraded (10.563%); 23097/204570 objects misplaced (11.291%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:26:47.485 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:47.764 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:47.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:47.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:47.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:47.898 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:47.930 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:26:47.930 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:26:47.931 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:26:47.941 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:47.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:48.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:48.077 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:48.368 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 73383 73367 67.4093 46.875 0.0090209 0.0148295 2023-06-10T21:26:48.413 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:48.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:48.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:48.502 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:48.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:48.568 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:48.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:48.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:26:48.663 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:48.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:48.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:48.856 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:48.956 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:48.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:49.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:49.123 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:49.166 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:26:49.166 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:26:49.247 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:26:49.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 74276 74260 67.2388 55.8125 0.0274704 0.0148669 2023-06-10T21:26:49.389 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:49.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:49.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:49 smithi152 ceph-mon[156819]: pgmap v786: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s; 21891/211893 objects degraded (10.331%); 23497/211893 objects misplaced (11.089%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:49.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:49.541 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:49.584 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:49.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:49.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:49 smithi064 ceph-mon[180297]: pgmap v786: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s; 21891/211893 objects degraded (10.331%); 23497/211893 objects misplaced (11.089%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:49.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:49 smithi064 ceph-mon[186706]: pgmap v786: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 944 op/s; 21891/211893 objects degraded (10.331%); 23497/211893 objects misplaced (11.089%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:49.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:49.713 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:50.004 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:50.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:50.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:26:50.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:50.170 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:50.275 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:50.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:50.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:50.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 75073 75057 66.9899 49.8125 0.00765135 0.0149226 2023-06-10T21:26:50.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:50.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:50.590 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:50.636 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:50.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:50.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:50.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:50.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:26:50.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:50.794 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:26:50.819 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:50.835 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:26:50.879 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:26:50.903 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:50.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:50.932 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:26:50.983 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:26:51.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:51.021 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:26:51.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.021 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:26:51.022 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.022 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.022 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.022 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:26:51.075 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:51.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[186706]: pgmap v787: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 22447/218673 objects degraded (10.265%); 24344/218673 objects misplaced (11.133%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:51.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22447/218673 objects degraded (10.265%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:51.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3547185355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:51.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:51.324 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:51 smithi152 ceph-mon[156819]: pgmap v787: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 22447/218673 objects degraded (10.265%); 24344/218673 objects misplaced (11.133%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:51.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22447/218673 objects degraded (10.265%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:51.325 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3547185355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:51.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:51.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 76015 75999 66.8755 58.875 0.0253851 0.0149488 2023-06-10T21:26:51.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:51.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[180297]: pgmap v787: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 909 op/s; 22447/218673 objects degraded (10.265%); 24344/218673 objects misplaced (11.133%); 2.8 MiB/s, 45 objects/s recovering 2023-06-10T21:26:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22447/218673 objects degraded (10.265%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3547185355' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:26:52.200 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:52.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:52.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:52.371 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 76837 76821 66.6602 51.375 0.00685122 0.0149959 2023-06-10T21:26:52.376 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:52.467 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:26:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:52.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:26:52.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:52.608 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:26:52.608 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:26:52.608 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:26:52.608 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:26:52.659 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:52.824 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:52.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:52.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:52.984 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:26:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:26:52] "GET /metrics HTTP/1.1" 200 224059 "" "Prometheus/2.33.4" 2023-06-10T21:26:53.370 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 77780 77764 66.5544 58.9375 0.0150792 0.0150203 2023-06-10T21:26:53.516 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:53 smithi152 ceph-mon[156819]: pgmap v788: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:53.542 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:26:53.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:53 smithi064 ceph-mon[180297]: pgmap v788: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:53.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:53 smithi064 ceph-mon[186706]: pgmap v788: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:53.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:53.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:26:53.717 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:26:53.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:53.852 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:53.990 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:53.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:53.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:54.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:54.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:54.174 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:54.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:54.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:54.371 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 78732 78716 66.459 59.5 0.0147905 0.0150416 2023-06-10T21:26:54.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:54.500 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:54.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:54.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:54.698 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:54.763 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:54.879 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:54.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:54.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:55.015 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:55.061 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:55.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:55.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:55.174 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-06-10T21:26:55.174 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:26:55.174 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-06-10T21:26:55.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:26:55.208 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:55.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[180297]: pgmap v789: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:55.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2477891635' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.263 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.265 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[186706]: pgmap v789: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:55.265 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2477891635' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.265 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.317 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:26:55.372 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 79541 79525 66.2457 50.5625 0.0410492 0.01509 2023-06-10T21:26:55.467 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:55.478 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:55 smithi152 ceph-mon[156819]: pgmap v789: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 876 op/s; 22677/224787 objects degraded (10.088%); 24577/224787 objects misplaced (10.933%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:55.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2477891635' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.479 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:26:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:55.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:55.581 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:26:55.633 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:55.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:26:55.768 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:26:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:55.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:26:55.876 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:55.939 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:56.037 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:26:56.037 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:26:56.037 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:26:56.038 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:26:56.038 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:26:56.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 80436 80420 66.1094 55.9375 0.0031872 0.0151123 2023-06-10T21:26:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: osdmap e224: 8 total, 7 up, 6 in 2023-06-10T21:26:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22677/224787 objects degraded (10.088%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:56.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:56 smithi152 ceph-mon[156819]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: osdmap e224: 8 total, 7 up, 6 in 2023-06-10T21:26:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22677/224787 objects degraded (10.088%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[186706]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:26:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:26:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: osdmap e224: 8 total, 7 up, 6 in 2023-06-10T21:26:56.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22677/224787 objects degraded (10.088%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:26:56.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:56 smithi064 ceph-mon[180297]: failed to encode map e224 with expected crc 2023-06-10T21:26:56.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:26:57.006 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:57.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:57.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:57.187 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:57.279 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:57 smithi064 ceph-mon[180297]: pgmap v791: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 23264/231723 objects degraded (10.040%); 25441/231723 objects misplaced (10.979%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:57.290 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:57.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:57.326 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:57.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 81309 81293 65.9595 54.5625 0.0220786 0.0151552 2023-06-10T21:26:57.407 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:26:57.407 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:57.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:57 smithi152 ceph-mon[156819]: pgmap v791: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 23264/231723 objects degraded (10.040%); 25441/231723 objects misplaced (10.979%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:57.453 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:57.497 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:26:57.534 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:57.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:57.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:57.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:57 smithi064 ceph-mon[186706]: pgmap v791: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s; 23264/231723 objects degraded (10.040%); 25441/231723 objects misplaced (10.979%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:26:57.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:26:57.700 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:57.779 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:26:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:57.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:26:57.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:57.953 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:58.183 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:58.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:58.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:26:58.339 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:58.373 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 82204 82188 65.8309 55.9375 0.00769961 0.01518 2023-06-10T21:26:58.383 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:26:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:58.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:26:58.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:26:58.535 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:58.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:26:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:58.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:26:58.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:26:58.813 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:58.870 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:26:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:58.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:26:58.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:26:59.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:59.054 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:26:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:26:59.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:26:59.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:26:59.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:26:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:26:59.362 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:26:59.362 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:26:59.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:26:59.362 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109295 9.9 e141/127 9:9b3a09da:::benchmark_data_smithi064_247471_object307:head v 141'6783, mlcod=141'6781)", 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45901375599999, 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998, 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.363 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898798+0000", 2023-06-10T21:26:59.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2339999999999997e-06 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898805+0000", 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5519999999999993e-06 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898815+0000", 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2270000000000001e-06 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898820+0000", 2023-06-10T21:26:59.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4044e-05 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898864+0000", 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9573e-05 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898884+0000", 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35487087899999997 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253755+0000", 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3262000000000001e-05 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253788+0000", 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998 2023-06-10T21:26:59.370 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109297 9.5 e141/127 9:a2279825:::benchmark_data_smithi064_247471_object309:head v 141'6825, mlcod=141'6823)", 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:26:59.371 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45859693599999, 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998, 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:26:59.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899215+0000", 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8650000000000003e-06 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899222+0000", 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2023-06-10T21:26:59.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899232+0000", 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6350000000000003e-06 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899237+0000", 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0815000000000001e-05 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899278+0000", 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1216000000000002e-05 2023-06-10T21:26:59.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899299+0000", 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.349632361 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248932+0000", 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100882 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249032+0000", 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998 2023-06-10T21:26:59.378 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109299 9.7 9:e2e5e039:::benchmark_data_smithi064_247471_object311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.455682981, 2023-06-10T21:26:59.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998, 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109299 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.380 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902128+0000", 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2023-06-10T21:26:59.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902135+0000", 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0420000000000008e-06 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902144+0000", 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6470000000000003e-06 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902150+0000", 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6499000000000003e-05 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902197+0000", 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011322100000000001 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012462399999999999 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902434+0000", 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046768010000000004 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907111+0000", 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6680000000000004e-06 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907118+0000", 2023-06-10T21:26:59.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34198863299999999 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249107+0000", 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102996 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249210+0000", 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4842000000000002e-05 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249254+0000", 2023-06-10T21:26:59.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109302 9.a 9:593e78fe:::benchmark_data_smithi064_247471_object314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:26:59.390 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45550153099998, 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999, 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109302 2023-06-10T21:26:59.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5360000000000003e-06 2023-06-10T21:26:59.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902315+0000", 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.669e-06 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902322+0000", 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9060000000000004e-06 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902325+0000", 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074886199999999999 2023-06-10T21:26:59.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903074+0000", 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010393800000000001 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903178+0000", 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8347000000000001e-05 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903247+0000", 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.345750055 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248997+0000", 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000337282 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249334+0000", 2023-06-10T21:26:59.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6821e-05 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249361+0000", 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7002000000000002e-05 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249408+0000", 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8299e-05 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249446+0000", 2023-06-10T21:26:59.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109305 9.6 9:68c28d89:::benchmark_data_smithi064_247471_object317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:26:59.401 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45475951600002, 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997, 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109305 2023-06-10T21:26:59.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903052+0000", 2023-06-10T21:26:59.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903058+0000", 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0450000000000006e-06 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903067+0000", 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8790000000000002e-06 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903071+0000", 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8942e-05 2023-06-10T21:26:59.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903120+0000", 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100547 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903221+0000", 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109324 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903330+0000", 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34556609100000002 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248896+0000", 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042370840000000003 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253133+0000", 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3943000000000006e-05 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253217+0000", 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3278999999999997e-05 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253261+0000", 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3470000000000001e-05 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253314+0000", 2023-06-10T21:26:59.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109296 9.9 e141/127 9:9d3096da:::benchmark_data_smithi064_247471_object308:head v 141'6784, mlcod=141'6781)", 2023-06-10T21:26:59.412 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45304923700002, 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001, 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3180000000000003e-06 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904769+0000", 2023-06-10T21:26:59.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0399999999999993e-06 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904777+0000", 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6299999999999997e-06 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904782+0000", 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115028 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904897+0000", 2023-06-10T21:26:59.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8039000000000001e-05 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904915+0000", 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34887546699999999 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253790+0000", 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5639999999999999e-05 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253806+0000", 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001 2023-06-10T21:26:59.419 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109303 9.b e141/127 9:d16859ff:::benchmark_data_smithi064_247471_object315:head v 141'6884, mlcod=141'6882)", 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45301935700002, 2023-06-10T21:26:59.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885, 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.456e-06 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904797+0000", 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9980000000000003e-06 2023-06-10T21:26:59.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904802+0000", 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0400000000000001e-06 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904805+0000", 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34422613600000002 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249031+0000", 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7886000000000001e-05 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249089+0000", 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078164589999999996 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256905+0000", 2023-06-10T21:26:59.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2179999999999999e-05 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256947+0000", 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.427 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109304 9.5 e141/127 9:a38538a9:::benchmark_data_smithi064_247471_object316:head v 141'6826, mlcod=141'6823)", 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.452998057, 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202, 2023-06-10T21:26:59.428 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904814+0000", 2023-06-10T21:26:59.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8880000000000002e-06 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904820+0000", 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904825+0000", 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.965e-06 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904828+0000", 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002189528 2023-06-10T21:26:59.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907018+0000", 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3083e-05 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907031+0000", 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34204052800000001 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249071+0000", 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7423000000000003e-05 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249119+0000", 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109300 9.9 e141/127 9:94ae4f7e:::benchmark_data_smithi064_247471_object312:head v 141'6785, mlcod=141'6781)", 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.45000902300001, 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001, 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.436 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907803+0000", 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.772e-06 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907812+0000", 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34109469599999998 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248906+0000", 2023-06-10T21:26:59.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1025999999999997e-05 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248937+0000", 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016277199999999999 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249100+0000", 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4561999999999997e-05 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249155+0000", 2023-06-10T21:26:59.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083045289999999997 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257459+0000", 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8495000000000002e-05 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.442 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257508+0000", 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.443 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109307 9.f 9:f866cb96:::benchmark_data_smithi064_247471_object319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.44942281200002, 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092, 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.444 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109307 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:26:59.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908388+0000", 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4800000000000004e-06 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908396+0000", 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34054057399999998 2023-06-10T21:26:59.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248936+0000", 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1486000000000001e-05 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248958+0000", 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032944700000000002 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249287+0000", 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101111 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249388+0000", 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123366 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249512+0000", 2023-06-10T21:26:59.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077532640000000002 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257265+0000", 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045560400000000002 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257721+0000", 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003373905 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261094+0000", 2023-06-10T21:26:59.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6616000000000002e-05 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261141+0000", 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6924000000000001e-05 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261178+0000", 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.454 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:109299 9.7 e141/127 ondisk, result = 0)", 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 276.44645895500003, 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999, 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:26:59.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911353+0000", 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7229999999999998e-06 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911359+0000", 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33752210999999999 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248882+0000", 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6460000000000001e-05 2023-06-10T21:26:59.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248908+0000", 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015431399999999999 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249062+0000", 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2586999999999999e-05 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249085+0000", 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017446900000000001 2023-06-10T21:26:59.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249259+0000", 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137750 9.f 9:fe4ed14c:::benchmark_data_smithi064_247471_object28762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 236.92929911499999, 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768, 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.462 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137750 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428513+0000", 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.377e-06 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428516+0000", 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1229999999999997e-06 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428523+0000", 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8249999999999998e-06 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428526+0000", 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00081994200000000004 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429346+0000", 2023-06-10T21:26:59.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2603999999999996e-05 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429439+0000", 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104776 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429544+0000", 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049570079999999997 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.434501+0000", 2023-06-10T21:26:59.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018088150000000001 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436310+0000", 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30228024799999997 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738590+0000", 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2488999999999998e-05 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738642+0000", 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8994e-05 2023-06-10T21:26:59.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738681+0000", 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137758 9.7 9:e64fca9d:::benchmark_data_smithi064_247471_object28770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 236.92143487300001, 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999, 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.473 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137758 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436377+0000", 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2279999999999998e-06 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436382+0000", 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4789999999999996e-06 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436390+0000", 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6550000000000003e-06 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436395+0000", 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5500000000000001e-05 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436441+0000", 2023-06-10T21:26:59.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105098 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436546+0000", 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011236899999999999 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436658+0000", 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30189442999999999 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738552+0000", 2023-06-10T21:26:59.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000176961 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738729+0000", 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5865999999999997e-05 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738785+0000", 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5071999999999999e-05 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738820+0000", 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1973e-05 2023-06-10T21:26:59.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738852+0000", 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.483 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137759 9.7 9:e8bc7c80:::benchmark_data_smithi064_247471_object28771:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 236.92140661600001, 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001, 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.484 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137759 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:26:59.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8289999999999999e-06 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436409+0000", 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5740000000000003e-06 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436414+0000", 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7810000000000005e-06 2023-06-10T21:26:59.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436422+0000", 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004238187 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440660+0000", 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117327 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440777+0000", 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012919200000000001 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440907+0000", 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.297783668 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738690+0000", 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8073999999999999e-05 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738758+0000", 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000156365 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738915+0000", 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6863000000000003e-05 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738952+0000", 2023-06-10T21:26:59.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8658999999999997e-05 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738990+0000", 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.494 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137770 9.7 9:e3c55b93:::benchmark_data_smithi064_247471_object28782:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 236.91280794299999, 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009, 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.495 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137770 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:26:59.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445004+0000", 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.569e-06 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445008+0000", 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1970000000000001e-06 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445014+0000", 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9210000000000002e-06 2023-06-10T21:26:59.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445018+0000", 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28887405700000002 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.733892+0000", 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146957 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734039+0000", 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012832300000000001 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734167+0000", 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049619210000000002 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739129+0000", 2023-06-10T21:26:59.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047693650000000002 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.743899+0000", 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011874 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744017+0000", 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8343e-05 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744076+0000", 2023-06-10T21:26:59.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9201e-05 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744115+0000", 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199845 9.f 9:ff71ad38:::benchmark_data_smithi064_247471_object90857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 137.83342155700001, 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001, 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.506 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199845 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:26:59.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524389+0000", 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524398+0000", 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2369e-05 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524410+0000", 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8360000000000003e-06 2023-06-10T21:26:59.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524417+0000", 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084813649999999994 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532898+0000", 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114187 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533013+0000", 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118496 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533131+0000", 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050821640000000001 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.538213+0000", 2023-06-10T21:26:59.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087860630000000002 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.546999+0000", 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28852924899999999 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835529+0000", 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4122999999999998e-05 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835573+0000", 2023-06-10T21:26:59.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9298999999999998e-05 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835612+0000", 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.516 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201393 9.5 9:a576e4a0:::benchmark_data_smithi064_247471_object92405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 135.23476261499999, 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001, 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.517 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201393 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123049+0000", 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7429999999999999e-06 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123055+0000", 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6000000000000007e-06 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123064+0000", 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9880000000000001e-06 2023-06-10T21:26:59.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123070+0000", 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019396100000000001 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123263+0000", 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018801169999999999 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125144+0000", 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000161557 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125305+0000", 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023599202 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148904+0000", 2023-06-10T21:26:59.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6900000000000004e-06 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148914+0000", 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593960899999999 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124854+0000", 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2537999999999999e-05 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124866+0000", 2023-06-10T21:26:59.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201395 9.5 9:a9ebfe85:::benchmark_data_smithi064_247471_object92407:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:26:59.527 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 135.23446783099999, 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999, 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201395 2023-06-10T21:26:59.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123344+0000", 2023-06-10T21:26:59.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0849999999999996e-06 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123349+0000", 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.354e-06 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123357+0000", 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4389999999999999e-06 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123363+0000", 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011364407 2023-06-10T21:26:59.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134727+0000", 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118065 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:26:59.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134845+0000", 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000154458 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.135000+0000", 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013944756000000001 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148945+0000", 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9670000000000007e-06 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:26:59.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148954+0000", 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593688599999995 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124890+0000", 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2764e-05 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124903+0000", 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201403 9.5 9.ddfae635 (undecoded) ondisk+write+known_if_redirected e171)", 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 135.22436264000001, 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997, 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201403 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9119999999999998e-06 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133453+0000", 2023-06-10T21:26:59.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5379999999999999e-06 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133459+0000", 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0280000000000002e-06 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133463+0000", 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020130169999999999 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153593+0000", 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4200000000000004e-06 2023-06-10T21:26:59.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153600+0000", 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97132188100000005 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124922+0000", 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2189999999999998e-06 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124929+0000", 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997 2023-06-10T21:26:59.545 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201415 9.5 9.ce5b1865 (undecoded) ondisk+write+known_if_redirected e172)", 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 135.19131016200001, 2023-06-10T21:26:59.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998, 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201415 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.547 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:26:59.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166501+0000", 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0360000000000003e-06 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166507+0000", 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8352999999999999e-05 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166526+0000", 2023-06-10T21:26:59.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7720000000000002e-06 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166531+0000", 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5688e-05 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166576+0000", 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8590000000000001e-06 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166582+0000", 2023-06-10T21:26:59.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95836468200000002 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124947+0000", 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3220000000000004e-06 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:26:59.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124954+0000", 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:26:59.554 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:26:59.555 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 82683 82667 65.3766 29.9375 0.00402888 0.0152717 2023-06-10T21:26:59.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:26:59 smithi152 ceph-mon[156819]: pgmap v792: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s; 23549/238143 objects degraded (9.889%); 25691/238143 objects misplaced (10.788%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:26:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:26:59 smithi064 ceph-mon[186706]: pgmap v792: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s; 23549/238143 objects degraded (9.889%); 25691/238143 objects misplaced (10.788%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:26:59.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:26:59 smithi064 ceph-mon[180297]: pgmap v792: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s; 23549/238143 objects degraded (9.889%); 25691/238143 objects misplaced (10.788%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:27:00.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:27:00.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:27:00.371840+0000 min lat: 0.00121051 max lat: 5.9617 avg lat: 0.0153137 2023-06-10T21:27:00.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:27:00.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 83599 83583 65.275 57.25 0.0200161 0.0153137 2023-06-10T21:27:00.397 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:00.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:27:00.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:00.555 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:00.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:27:00.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:27:00.633 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:00.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:00.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:00.786 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:00.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:00.903 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:01.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:01.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:01.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:01.295 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:01.337 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:27:01.364 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:27:01.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 84318 84302 65.0236 44.9375 0.0292126 0.0153728 2023-06-10T21:27:01.380 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:01.396 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:27:01.423 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:01.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:01.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:01.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[180297]: pgmap v793: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s; 24093/244725 objects degraded (9.845%); 26527/244725 objects misplaced (10.840%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:27:01.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 24093/244725 objects degraded (9.845%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:01.444 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/922515992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:01.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[186706]: pgmap v793: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s; 24093/244725 objects degraded (9.845%); 26527/244725 objects misplaced (10.840%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:27:01.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 24093/244725 objects degraded (9.845%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:01.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/922515992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:01.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:01.454 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:01.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:01 smithi152 ceph-mon[156819]: pgmap v793: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 868 op/s; 24093/244725 objects degraded (9.845%); 26527/244725 objects misplaced (10.840%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:27:01.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 24093/244725 objects degraded (9.845%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:01.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/922515992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:01.494 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:01.538 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:01.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:01.576 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.577 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.578 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:27:01.615 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:01.816 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:01.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:01.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:01.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:02.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:02 smithi064 ceph-mon[180297]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:02.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 85052 85036 64.79 45.875 0.00346522 0.0154271 2023-06-10T21:27:02.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:02 smithi152 ceph-mon[156819]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:02 smithi064 ceph-mon[186706]: from='client.45896 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:02] "GET /metrics HTTP/1.1" 200 224059 "" "Prometheus/2.33.4" 2023-06-10T21:27:03.280 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:27:03.280 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:27:03.374 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 85704 85688 64.5004 40.75 0.0174993 0.0154986 2023-06-10T21:27:03.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:03.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:03.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:03.572 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:03.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:03 smithi152 ceph-mon[156819]: pgmap v794: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:03.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:03.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:03.601 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:03.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[186706]: pgmap v794: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:03.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:03.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[180297]: pgmap v794: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:03.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:03.944 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:03.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:03.976 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:04.048 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:04.120 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:27:04.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:04.266 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":793,"stamp":"2023-06-10T21:26:59.601424+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5345122190,"num_objects":81575,"num_object_clones":0,"num_object_copies":244725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24093,"num_objects_misplaced":26527,"num_objects_unfound":0,"num_objects_dirty":81575,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":81812,"num_write_kb":5221710,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7183,"num_bytes_recovered":469902490,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81746,"ondisk_log_size":81746,"up":215,"acting":218,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15917764,"kb_used_data":13465492,"kb_used_omap":20,"kb_used_meta":2452203,"kb_avail":546553148,"statfs":{"total":575970213888,"available":559670423552,"internally_reserved":0,"allocated":13788663808,"data_stored":13784596206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2511056526},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,3,4,1,0,8],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":569114624,"num_objects":8684,"num_object_clones":0,"num_object_copies":26052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1646,"num_objects_misplaced":2183,"num_objects_unfound":0,"num_objects_dirty":8684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8684,"num_write_kb":555776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":34013184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002931"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":214,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.215873+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:26:44.215873+0000","last_peered":"2023-06-10T21:26:44.215873+0000","last_clean":"2023-06-10T21:26:44.215873+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:26:44.215873+0000","last_undegraded":"2023-06-10T21:26:44.215873+0000","last_fullsized":"2023-06-10T21:26:44.215873+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":89,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234168+0000","last_change":"2023-06-10T21:26:27.787856+0000","last_active":"2023-06-10T21:26:44.234168+0000","last_peered":"2023-06-10T21:26:44.234168+0000","last_clean":"2023-06-10T21:13:57.474600+0000","last_became_active":"2023-06-10T21:26:27.787856+0000","last_became_peered":"2023-06-10T21:26:27.787856+0000","last_unstale":"2023-06-10T21:26:44.234168+0000","last_undegraded":"2023-06-10T21:26:44.234168+0000","last_fullsized":"2023-06-10T21:26:27.776169+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:29:11.586416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":144,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234639+0000","last_change":"2023-06-10T21:26:27.787641+0000","last_active":"2023-06-10T21:26:44.234639+0000","last_peered":"2023-06-10T21:26:44.234639+0000","last_clean":"2023-06-10T21:19:00.485389+0000","last_became_active":"2023-06-10T21:26:27.787641+0000","last_became_peered":"2023-06-10T21:26:27.787641+0000","last_unstale":"2023-06-10T21:26:44.234639+0000","last_undegraded":"2023-06-10T21:26:44.234639+0000","last_fullsized":"2023-06-10T21:26:27.775244+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:39:35.438222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":69,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:51.513506+0000","last_change":"2023-06-10T21:26:27.785421+0000","last_active":"2023-06-10T21:26:51.513506+0000","last_peered":"2023-06-10T21:26:51.513506+0000","last_clean":"2023-06-10T21:26:20.391953+0000","last_became_active":"2023-06-10T21:26:27.785421+0000","last_became_peered":"2023-06-10T21:26:27.785421+0000","last_unstale":"2023-06-10T21:26:51.513506+0000","last_undegraded":"2023-06-10T21:26:51.513506+0000","last_fullsized":"2023-06-10T21:26:27.778803+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:54:12.929497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":172,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.217947+0000","last_change":"2023-06-10T21:26:27.788758+0000","last_active":"2023-06-10T21:26:44.217947+0000","last_peered":"2023-06-10T21:26:44.217947+0000","last_clean":"2023-06-10T21:24:36.651687+0000","last_became_active":"2023-06-10T21:26:27.788758+0000","last_became_peered":"2023-06-10T21:26:27.788758+0000","last_unstale":"2023-06-10T21:26:44.217947+0000","last_undegraded":"2023-06-10T21:26:44.217947+0000","last_fullsized":"2023-06-10T21:26:27.776841+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:29:54.271286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.221403+0000","last_change":"2023-06-10T21:26:27.781570+0000","last_active":"2023-06-10T21:26:44.221403+0000","last_peered":"2023-06-10T21:26:44.221403+0000","last_clean":"2023-06-10T21:26:21.435727+0000","last_became_active":"2023-06-10T21:26:27.781570+0000","last_became_peered":"2023-06-10T21:26:27.781570+0000","last_unstale":"2023-06-10T21:26:44.221403+0000","last_undegraded":"2023-06-10T21:26:44.221403+0000","last_fullsized":"2023-06-10T21:26:27.776705+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:53:41.167915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":86,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.221866+0000","last_change":"2023-06-10T21:26:27.781631+0000","last_active":"2023-06-10T21:26:44.221866+0000","last_peered":"2023-06-10T21:26:44.221866+0000","last_clean":"2023-06-10T21:26:20.392386+0000","last_became_active":"2023-06-10T21:26:27.781631+0000","last_became_peered":"2023-06-10T21:26:27.781631+0000","last_unstale":"2023-06-10T21:26:44.221866+0000","last_undegraded":"2023-06-10T21:26:44.221866+0000","last_fullsized":"2023-06-10T21:26:27.777427+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.845813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":188,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.235717+0000","last_change":"2023-06-10T21:26:27.792890+0000","last_active":"2023-06-10T21:26:44.235717+0000","last_peered":"2023-06-10T21:26:44.235717+0000","last_clean":"2023-06-10T21:26:05.730925+0000","last_became_active":"2023-06-10T21:26:27.792890+0000","last_became_peered":"2023-06-10T21:26:27.792890+0000","last_unstale":"2023-06-10T21:26:44.235717+0000","last_undegraded":"2023-06-10T21:26:44.235717+0000","last_fullsized":"2023-06-10T21:26:27.777122+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:51:54.472567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234626+0000","last_change":"2023-06-10T21:26:27.786701+0000","last_active":"2023-06-10T21:26:44.234626+0000","last_peered":"2023-06-10T21:26:44.234626+0000","last_clean":"2023-06-10T21:26:20.562068+0000","last_became_active":"2023-06-10T21:26:27.786701+0000","last_became_peered":"2023-06-10T21:26:27.786701+0000","last_unstale":"2023-06-10T21:26:44.234626+0000","last_undegraded":"2023-06-10T21:26:44.234626+0000","last_fullsized":"2023-06-10T21:26:27.783028+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.235715+0000","last_change":"2023-06-10T21:26:27.786324+0000","last_active":"2023-06-10T21:26:44.235715+0000","last_peered":"2023-06-10T21:26:44.235715+0000","last_clean":"2023-06-10T21:26:05.731138+0000","last_became_active":"2023-06-10T21:26:27.786324+0000","last_became_peered":"2023-06-10T21:26:27.786324+0000","last_unstale":"2023-06-10T21:26:44.235715+0000","last_undegraded":"2023-06-10T21:26:44.235715+0000","last_fullsized":"2023-06-10T21:26:27.781327+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":200,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.237120+0000","last_change":"2023-06-10T21:26:27.785616+0000","last_active":"2023-06-10T21:26:44.237120+0000","last_peered":"2023-06-10T21:26:44.237120+0000","last_clean":"2023-06-10T21:26:20.562388+0000","last_became_active":"2023-06-10T21:26:27.785616+0000","last_became_peered":"2023-06-10T21:26:27.785616+0000","last_unstale":"2023-06-10T21:26:44.237120+0000","last_undegraded":"2023-06-10T21:26:27.781808+0000","last_fullsized":"2023-06-10T21:26:27.781691+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":168,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.215140+0000","last_change":"2023-06-10T21:26:15.746830+0000","last_active":"2023-06-10T21:26:44.215140+0000","last_peered":"2023-06-10T21:26:44.215140+0000","last_clean":"2023-06-10T21:26:44.215140+0000","last_became_active":"2023-06-10T21:26:15.746510+0000","last_became_peered":"2023-06-10T21:26:15.746510+0000","last_unstale":"2023-06-10T21:26:44.215140+0000","last_undegraded":"2023-06-10T21:26:44.215140+0000","last_fullsized":"2023-06-10T21:26:44.215140+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:50:22.116927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":94,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.234761+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:26:44.234761+0000","last_peered":"2023-06-10T21:26:44.234761+0000","last_clean":"2023-06-10T21:26:44.234761+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:26:44.234761+0000","last_undegraded":"2023-06-10T21:26:44.234761+0000","last_fullsized":"2023-06-10T21:26:44.234761+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.216810+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:26:44.216810+0000","last_peered":"2023-06-10T21:26:44.216810+0000","last_clean":"2023-06-10T21:26:44.216810+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:26:44.216810+0000","last_undegraded":"2023-06-10T21:26:44.216810+0000","last_fullsized":"2023-06-10T21:26:44.216810+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":223,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.236253+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:26:44.236253+0000","last_peered":"2023-06-10T21:26:44.236253+0000","last_clean":"2023-06-10T21:26:44.236253+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:26:44.236253+0000","last_undegraded":"2023-06-10T21:26:44.236253+0000","last_fullsized":"2023-06-10T21:26:44.236253+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234679+0000","last_change":"2023-06-10T21:26:27.787457+0000","last_active":"2023-06-10T21:26:44.234679+0000","last_peered":"2023-06-10T21:26:44.234679+0000","last_clean":"2023-06-10T21:13:57.475185+0000","last_became_active":"2023-06-10T21:26:27.787457+0000","last_became_peered":"2023-06-10T21:26:27.787457+0000","last_unstale":"2023-06-10T21:26:44.234679+0000","last_undegraded":"2023-06-10T21:26:44.234679+0000","last_fullsized":"2023-06-10T21:26:27.781392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.218553+0000","last_change":"2023-06-10T21:26:27.783019+0000","last_active":"2023-06-10T21:26:44.218553+0000","last_peered":"2023-06-10T21:26:44.218553+0000","last_clean":"2023-06-10T21:26:20.391274+0000","last_became_active":"2023-06-10T21:26:27.783019+0000","last_became_peered":"2023-06-10T21:26:27.783019+0000","last_unstale":"2023-06-10T21:26:44.218553+0000","last_undegraded":"2023-06-10T21:26:44.218553+0000","last_fullsized":"2023-06-10T21:26:27.777376+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:41:33.972128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":132,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.237343+0000","last_change":"2023-06-10T21:26:27.786424+0000","last_active":"2023-06-10T21:26:44.237343+0000","last_peered":"2023-06-10T21:26:44.237343+0000","last_clean":"2023-06-10T21:26:20.562490+0000","last_became_active":"2023-06-10T21:26:27.786424+0000","last_became_peered":"2023-06-10T21:26:27.786424+0000","last_unstale":"2023-06-10T21:26:44.237343+0000","last_undegraded":"2023-06-10T21:26:44.237343+0000","last_fullsized":"2023-06-10T21:26:27.781372+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":185,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.235635+0000","last_change":"2023-06-10T21:26:15.747593+0000","last_active":"2023-06-10T21:26:44.235635+0000","last_peered":"2023-06-10T21:26:44.235635+0000","last_clean":"2023-06-10T21:26:44.235635+0000","last_became_active":"2023-06-10T21:26:15.747244+0000","last_became_peered":"2023-06-10T21:26:15.747244+0000","last_unstale":"2023-06-10T21:26:44.235635+0000","last_undegraded":"2023-06-10T21:26:44.235635+0000","last_fullsized":"2023-06-10T21:26:44.235635+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.222263+0000","last_change":"2023-06-10T21:26:27.785039+0000","last_active":"2023-06-10T21:26:44.222263+0000","last_peered":"2023-06-10T21:26:44.222263+0000","last_clean":"2023-06-10T21:26:20.391891+0000","last_became_active":"2023-06-10T21:26:27.785039+0000","last_became_peered":"2023-06-10T21:26:27.785039+0000","last_unstale":"2023-06-10T21:26:44.222263+0000","last_undegraded":"2023-06-10T21:26:44.222263+0000","last_fullsized":"2023-06-10T21:26:27.777488+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:49:35.504339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":205,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.218121+0000","last_change":"2023-06-10T21:26:27.785625+0000","last_active":"2023-06-10T21:26:44.218121+0000","last_peered":"2023-06-10T21:26:44.218121+0000","last_clean":"2023-06-10T21:26:21.406183+0000","last_became_active":"2023-06-10T21:26:27.785625+0000","last_became_peered":"2023-06-10T21:26:27.785625+0000","last_unstale":"2023-06-10T21:26:44.218121+0000","last_undegraded":"2023-06-10T21:26:27.779630+0000","last_fullsized":"2023-06-10T21:26:27.779530+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:23:23.310377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.235160+0000","last_change":"2023-06-10T21:26:27.790352+0000","last_active":"2023-06-10T21:26:44.235160+0000","last_peered":"2023-06-10T21:26:44.235160+0000","last_clean":"2023-06-10T21:13:57.474737+0000","last_became_active":"2023-06-10T21:26:27.790352+0000","last_became_peered":"2023-06-10T21:26:27.790352+0000","last_unstale":"2023-06-10T21:26:44.235160+0000","last_undegraded":"2023-06-10T21:26:44.235160+0000","last_fullsized":"2023-06-10T21:26:27.782802+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"224'5059","reported_seq":6758,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:56.389108+0000","last_change":"2023-06-10T21:25:57.635943+0000","last_active":"2023-06-10T21:26:56.389108+0000","last_peered":"2023-06-10T21:26:56.389108+0000","last_clean":"2023-06-10T21:26:56.389108+0000","last_became_active":"2023-06-10T21:25:57.632429+0000","last_became_peered":"2023-06-10T21:25:57.632429+0000","last_unstale":"2023-06-10T21:26:56.389108+0000","last_undegraded":"2023-06-10T21:26:56.389108+0000","last_fullsized":"2023-06-10T21:26:56.389108+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5059,"log_dups_size":0,"ondisk_log_size":5059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:41:00.568268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5059,"num_write_kb":323776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":822,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.235401+0000","last_change":"2023-06-10T21:24:45.162493+0000","last_active":"2023-06-10T21:26:44.235401+0000","last_peered":"2023-06-10T21:26:44.235401+0000","last_clean":"2023-06-10T21:26:44.235401+0000","last_became_active":"2023-06-10T21:24:45.139001+0000","last_became_peered":"2023-06-10T21:24:45.139001+0000","last_unstale":"2023-06-10T21:26:44.235401+0000","last_undegraded":"2023-06-10T21:26:44.235401+0000","last_fullsized":"2023-06-10T21:26:44.235401+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":49,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.238087+0000","last_change":"2023-06-10T21:26:27.789518+0000","last_active":"2023-06-10T21:26:44.238087+0000","last_peered":"2023-06-10T21:26:44.238087+0000","last_clean":"2023-06-10T21:26:20.562380+0000","last_became_active":"2023-06-10T21:26:27.789518+0000","last_became_peered":"2023-06-10T21:26:27.789518+0000","last_unstale":"2023-06-10T21:26:44.238087+0000","last_undegraded":"2023-06-10T21:26:44.238087+0000","last_fullsized":"2023-06-10T21:26:27.781860+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"224'5182","reported_seq":6852,"reported_epoch":224,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:26:58.688445+0000","last_change":"2023-06-10T21:26:28.972723+0000","last_active":"2023-06-10T21:26:58.688445+0000","last_peered":"2023-06-10T21:26:58.688445+0000","last_clean":"2023-06-10T21:26:15.549377+0000","last_became_active":"2023-06-10T21:26:28.952008+0000","last_became_peered":"2023-06-10T21:26:28.952008+0000","last_unstale":"2023-06-10T21:26:58.688445+0000","last_undegraded":"2023-06-10T21:26:28.877253+0000","last_fullsized":"2023-06-10T21:26:58.688445+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5182,"log_dups_size":0,"ondisk_log_size":5182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:07:08.784247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339607552,"num_objects":5182,"num_object_clones":0,"num_object_copies":15546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3395,"num_objects_misplaced":4929,"num_objects_unfound":0,"num_objects_dirty":5182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5182,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":802,"num_bytes_recovered":52559872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4","objects":253},{"shards":"2,4,6","objects":1787},{"shards":"2,6","objects":3142}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":193,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.234750+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:26:44.234750+0000","last_peered":"2023-06-10T21:26:44.234750+0000","last_clean":"2023-06-10T21:26:44.234750+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:26:44.234750+0000","last_undegraded":"2023-06-10T21:26:44.234750+0000","last_fullsized":"2023-06-10T21:26:44.234750+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.229494+0000","last_change":"2023-06-10T21:26:27.790530+0000","last_active":"2023-06-10T21:26:44.229494+0000","last_peered":"2023-06-10T21:26:44.229494+0000","last_clean":"2023-06-10T21:19:00.489965+0000","last_became_active":"2023-06-10T21:26:27.790530+0000","last_became_peered":"2023-06-10T21:26:27.790530+0000","last_unstale":"2023-06-10T21:26:44.229494+0000","last_undegraded":"2023-06-10T21:26:44.229494+0000","last_fullsized":"2023-06-10T21:26:27.780661+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"224'5083","reported_seq":5127,"reported_epoch":224,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:56.376489+0000","last_change":"2023-06-10T21:26:27.785031+0000","last_active":"2023-06-10T21:26:56.376489+0000","last_peered":"2023-06-10T21:26:56.376489+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:26:27.785031+0000","last_became_peered":"2023-06-10T21:26:27.785031+0000","last_unstale":"2023-06-10T21:26:56.376489+0000","last_undegraded":"2023-06-10T21:26:27.778233+0000","last_fullsized":"2023-06-10T21:26:27.778123+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:11:03.215088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333119488,"num_objects":5083,"num_object_clones":0,"num_object_copies":15249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5083,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5083,"num_write_kb":325312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":5083}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":209,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.236891+0000","last_change":"2023-06-10T21:26:27.785962+0000","last_active":"2023-06-10T21:26:44.236891+0000","last_peered":"2023-06-10T21:26:44.236891+0000","last_clean":"2023-06-10T21:26:20.562273+0000","last_became_active":"2023-06-10T21:26:27.785962+0000","last_became_peered":"2023-06-10T21:26:27.785962+0000","last_unstale":"2023-06-10T21:26:44.236891+0000","last_undegraded":"2023-06-10T21:26:27.782163+0000","last_fullsized":"2023-06-10T21:26:27.782058+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":85,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.234519+0000","last_change":"2023-06-10T21:26:15.747743+0000","last_active":"2023-06-10T21:26:44.234519+0000","last_peered":"2023-06-10T21:26:44.234519+0000","last_clean":"2023-06-10T21:26:44.234519+0000","last_became_active":"2023-06-10T21:26:15.747368+0000","last_became_peered":"2023-06-10T21:26:15.747368+0000","last_unstale":"2023-06-10T21:26:44.234519+0000","last_undegraded":"2023-06-10T21:26:44.234519+0000","last_fullsized":"2023-06-10T21:26:44.234519+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"224'5098","reported_seq":5142,"reported_epoch":224,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:58.697713+0000","last_change":"2023-06-10T21:26:27.813144+0000","last_active":"2023-06-10T21:26:58.697713+0000","last_peered":"2023-06-10T21:26:58.697713+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:26:27.813144+0000","last_became_peered":"2023-06-10T21:26:27.813144+0000","last_unstale":"2023-06-10T21:26:58.697713+0000","last_undegraded":"2023-06-10T21:26:27.783475+0000","last_fullsized":"2023-06-10T21:26:27.783342+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:07:57.630232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334102528,"num_objects":5098,"num_object_clones":0,"num_object_copies":15294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5098,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5098,"num_write_kb":326272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":5098}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":121,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.221197+0000","last_change":"2023-06-10T21:26:27.780557+0000","last_active":"2023-06-10T21:26:44.221197+0000","last_peered":"2023-06-10T21:26:44.221197+0000","last_clean":"2023-06-10T21:26:05.731822+0000","last_became_active":"2023-06-10T21:26:27.780557+0000","last_became_peered":"2023-06-10T21:26:27.780557+0000","last_unstale":"2023-06-10T21:26:44.221197+0000","last_undegraded":"2023-06-10T21:26:44.221197+0000","last_fullsized":"2023-06-10T21:26:27.777733+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:54:27.522102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":61,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.221079+0000","last_change":"2023-06-10T21:24:45.144751+0000","last_active":"2023-06-10T21:26:44.221079+0000","last_peered":"2023-06-10T21:26:44.221079+0000","last_clean":"2023-06-10T21:26:44.221079+0000","last_became_active":"2023-06-10T21:24:45.143921+0000","last_became_peered":"2023-06-10T21:24:45.143921+0000","last_unstale":"2023-06-10T21:26:44.221079+0000","last_undegraded":"2023-06-10T21:26:44.221079+0000","last_fullsized":"2023-06-10T21:26:44.221079+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:21:57.739161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"224'5094","reported_seq":5140,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:56.316916+0000","last_change":"2023-06-10T21:26:01.638496+0000","last_active":"2023-06-10T21:26:56.316916+0000","last_peered":"2023-06-10T21:26:56.316916+0000","last_clean":"2023-06-10T21:26:56.316916+0000","last_became_active":"2023-06-10T21:26:01.633949+0000","last_became_peered":"2023-06-10T21:26:01.633949+0000","last_unstale":"2023-06-10T21:26:56.316916+0000","last_undegraded":"2023-06-10T21:26:56.316916+0000","last_fullsized":"2023-06-10T21:26:56.316916+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:09:49.054007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333840384,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":326016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":144,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.220919+0000","last_change":"2023-06-10T21:26:27.785958+0000","last_active":"2023-06-10T21:26:44.220919+0000","last_peered":"2023-06-10T21:26:44.220919+0000","last_clean":"2023-06-10T21:24:36.651972+0000","last_became_active":"2023-06-10T21:26:27.785958+0000","last_became_peered":"2023-06-10T21:26:27.785958+0000","last_unstale":"2023-06-10T21:26:44.220919+0000","last_undegraded":"2023-06-10T21:26:27.780001+0000","last_fullsized":"2023-06-10T21:26:27.779877+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:20:07.512432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.228880+0000","last_change":"2023-06-10T21:26:27.790578+0000","last_active":"2023-06-10T21:26:44.228880+0000","last_peered":"2023-06-10T21:26:44.228880+0000","last_clean":"2023-06-10T21:26:05.731741+0000","last_became_active":"2023-06-10T21:26:27.790578+0000","last_became_peered":"2023-06-10T21:26:27.790578+0000","last_unstale":"2023-06-10T21:26:44.228880+0000","last_undegraded":"2023-06-10T21:26:44.228880+0000","last_fullsized":"2023-06-10T21:26:27.779863+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"224'4990","reported_seq":6706,"reported_epoch":224,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:58.675794+0000","last_change":"2023-06-10T21:26:27.820034+0000","last_active":"2023-06-10T21:26:58.675794+0000","last_peered":"2023-06-10T21:26:58.675794+0000","last_clean":"2023-06-10T21:26:26.761025+0000","last_became_active":"2023-06-10T21:26:27.820034+0000","last_became_peered":"2023-06-10T21:26:27.820034+0000","last_unstale":"2023-06-10T21:26:58.675794+0000","last_undegraded":"2023-06-10T21:26:27.817850+0000","last_fullsized":"2023-06-10T21:26:27.812347+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":4990,"log_dups_size":0,"ondisk_log_size":4990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:28:47.168875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327024640,"num_objects":4990,"num_object_clones":0,"num_object_copies":14970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4990,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4990,"num_write_kb":319360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54198272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":4990}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.229384+0000","last_change":"2023-06-10T21:26:27.785142+0000","last_active":"2023-06-10T21:26:44.229384+0000","last_peered":"2023-06-10T21:26:44.229384+0000","last_clean":"2023-06-10T21:13:57.477830+0000","last_became_active":"2023-06-10T21:26:27.785142+0000","last_became_peered":"2023-06-10T21:26:27.785142+0000","last_unstale":"2023-06-10T21:26:44.229384+0000","last_undegraded":"2023-06-10T21:26:44.229384+0000","last_fullsized":"2023-06-10T21:26:27.782073+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.216570+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:26:44.216570+0000","last_peered":"2023-06-10T21:26:44.216570+0000","last_clean":"2023-06-10T21:26:44.216570+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:26:44.216570+0000","last_undegraded":"2023-06-10T21:26:44.216570+0000","last_fullsized":"2023-06-10T21:26:44.216570+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"224'5067","reported_seq":7541,"reported_epoch":224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:26:56.189606+0000","last_change":"2023-06-10T21:26:28.835778+0000","last_active":"2023-06-10T21:26:56.189606+0000","last_peered":"2023-06-10T21:26:56.189606+0000","last_clean":"2023-06-10T21:26:15.544387+0000","last_became_active":"2023-06-10T21:26:16.777640+0000","last_became_peered":"2023-06-10T21:26:16.777640+0000","last_unstale":"2023-06-10T21:26:56.189606+0000","last_undegraded":"2023-06-10T21:26:28.832228+0000","last_fullsized":"2023-06-10T21:26:16.639321+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:32:08.099604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332070912,"num_objects":5067,"num_object_clones":0,"num_object_copies":15201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5067,"num_write_kb":324288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79626240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":3149},{"shards":"1,4,6","objects":1918}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.214050+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:26:44.214050+0000","last_peered":"2023-06-10T21:26:44.214050+0000","last_clean":"2023-06-10T21:26:44.214050+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:26:44.214050+0000","last_undegraded":"2023-06-10T21:26:44.214050+0000","last_fullsized":"2023-06-10T21:26:44.214050+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.228579+0000","last_change":"2023-06-10T21:26:27.784827+0000","last_active":"2023-06-10T21:26:44.228579+0000","last_peered":"2023-06-10T21:26:44.228579+0000","last_clean":"2023-06-10T21:13:57.670486+0000","last_became_active":"2023-06-10T21:26:27.784827+0000","last_became_peered":"2023-06-10T21:26:27.784827+0000","last_unstale":"2023-06-10T21:26:44.228579+0000","last_undegraded":"2023-06-10T21:26:44.228579+0000","last_fullsized":"2023-06-10T21:26:27.780178+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"224'5201","reported_seq":5247,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:59.521220+0000","last_change":"2023-06-10T21:26:01.630409+0000","last_active":"2023-06-10T21:26:59.521220+0000","last_peered":"2023-06-10T21:26:59.521220+0000","last_clean":"2023-06-10T21:26:59.521220+0000","last_became_active":"2023-06-10T21:26:01.627789+0000","last_became_peered":"2023-06-10T21:26:01.627789+0000","last_unstale":"2023-06-10T21:26:59.521220+0000","last_undegraded":"2023-06-10T21:26:59.521220+0000","last_fullsized":"2023-06-10T21:26:59.521220+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5201,"log_dups_size":0,"ondisk_log_size":5201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340852736,"num_objects":5201,"num_object_clones":0,"num_object_copies":15603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5201,"num_write_kb":332864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.238206+0000","last_change":"2023-06-10T21:26:27.788939+0000","last_active":"2023-06-10T21:26:44.238206+0000","last_peered":"2023-06-10T21:26:44.238206+0000","last_clean":"2023-06-10T21:13:57.474197+0000","last_became_active":"2023-06-10T21:26:27.788939+0000","last_became_peered":"2023-06-10T21:26:27.788939+0000","last_unstale":"2023-06-10T21:26:44.238206+0000","last_undegraded":"2023-06-10T21:26:44.238206+0000","last_fullsized":"2023-06-10T21:26:27.781734+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":145,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234824+0000","last_change":"2023-06-10T21:26:27.784584+0000","last_active":"2023-06-10T21:26:44.234824+0000","last_peered":"2023-06-10T21:26:44.234824+0000","last_clean":"2023-06-10T21:19:10.700014+0000","last_became_active":"2023-06-10T21:26:27.784584+0000","last_became_peered":"2023-06-10T21:26:27.784584+0000","last_unstale":"2023-06-10T21:26:44.234824+0000","last_undegraded":"2023-06-10T21:26:44.234824+0000","last_fullsized":"2023-06-10T21:26:27.776138+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:41:43.266239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"224'5237","reported_seq":5275,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:59.550858+0000","last_change":"2023-06-10T21:25:28.186909+0000","last_active":"2023-06-10T21:26:59.550858+0000","last_peered":"2023-06-10T21:26:59.550858+0000","last_clean":"2023-06-10T21:26:59.550858+0000","last_became_active":"2023-06-10T21:25:28.186604+0000","last_became_peered":"2023-06-10T21:25:28.186604+0000","last_unstale":"2023-06-10T21:26:59.550858+0000","last_undegraded":"2023-06-10T21:26:59.550858+0000","last_fullsized":"2023-06-10T21:26:59.550858+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5237,"log_dups_size":0,"ondisk_log_size":5237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343212032,"num_objects":5237,"num_object_clones":0,"num_object_copies":15711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5237,"num_write_kb":335168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":63,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:55.287442+0000","last_change":"2023-06-10T21:24:45.143914+0000","last_active":"2023-06-10T21:26:55.287442+0000","last_peered":"2023-06-10T21:26:55.287442+0000","last_clean":"2023-06-10T21:26:55.287442+0000","last_became_active":"2023-06-10T21:24:45.143600+0000","last_became_peered":"2023-06-10T21:24:45.143600+0000","last_unstale":"2023-06-10T21:26:55.287442+0000","last_undegraded":"2023-06-10T21:26:55.287442+0000","last_fullsized":"2023-06-10T21:26:55.287442+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:50.288121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":225,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.216845+0000","last_change":"2023-06-10T21:26:27.820466+0000","last_active":"2023-06-10T21:26:44.216845+0000","last_peered":"2023-06-10T21:26:44.216845+0000","last_clean":"2023-06-10T21:26:05.731923+0000","last_became_active":"2023-06-10T21:26:27.820466+0000","last_became_peered":"2023-06-10T21:26:27.820466+0000","last_unstale":"2023-06-10T21:26:44.216845+0000","last_undegraded":"2023-06-10T21:26:44.216845+0000","last_fullsized":"2023-06-10T21:26:27.812254+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:20:50.370098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.218889+0000","last_change":"2023-06-10T21:26:01.634082+0000","last_active":"2023-06-10T21:26:44.218889+0000","last_peered":"2023-06-10T21:26:44.218889+0000","last_clean":"2023-06-10T21:26:44.218889+0000","last_became_active":"2023-06-10T21:26:01.633769+0000","last_became_peered":"2023-06-10T21:26:01.633769+0000","last_unstale":"2023-06-10T21:26:44.218889+0000","last_undegraded":"2023-06-10T21:26:44.218889+0000","last_fullsized":"2023-06-10T21:26:44.218889+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:42:01.535005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"224'5019","reported_seq":7285,"reported_epoch":224,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:26:56.422974+0000","last_change":"2023-06-10T21:26:32.555200+0000","last_active":"2023-06-10T21:26:56.422974+0000","last_peered":"2023-06-10T21:26:56.422974+0000","last_clean":"2023-06-10T21:26:56.422974+0000","last_became_active":"2023-06-10T21:26:28.811989+0000","last_became_peered":"2023-06-10T21:26:28.811989+0000","last_unstale":"2023-06-10T21:26:56.422974+0000","last_undegraded":"2023-06-10T21:26:56.422974+0000","last_fullsized":"2023-06-10T21:26:56.422974+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:41:51.588200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328925184,"num_objects":5019,"num_object_clones":0,"num_object_copies":15057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5019,"num_objects_unfound":0,"num_objects_dirty":5019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5019,"num_write_kb":321216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1096,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":5019}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":99,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.218770+0000","last_change":"2023-06-10T21:26:27.780884+0000","last_active":"2023-06-10T21:26:44.218770+0000","last_peered":"2023-06-10T21:26:44.218770+0000","last_clean":"2023-06-10T21:26:21.407543+0000","last_became_active":"2023-06-10T21:26:27.780884+0000","last_became_peered":"2023-06-10T21:26:27.780884+0000","last_unstale":"2023-06-10T21:26:44.218770+0000","last_undegraded":"2023-06-10T21:26:44.218770+0000","last_fullsized":"2023-06-10T21:26:27.776394+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:28:19.989690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.219044+0000","last_change":"2023-06-10T21:26:15.751423+0000","last_active":"2023-06-10T21:26:44.219044+0000","last_peered":"2023-06-10T21:26:44.219044+0000","last_clean":"2023-06-10T21:26:44.219044+0000","last_became_active":"2023-06-10T21:26:15.751069+0000","last_became_peered":"2023-06-10T21:26:15.751069+0000","last_unstale":"2023-06-10T21:26:44.219044+0000","last_undegraded":"2023-06-10T21:26:44.219044+0000","last_fullsized":"2023-06-10T21:26:44.219044+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:47.921274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":44,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.213433+0000","last_change":"2023-06-10T21:26:27.812850+0000","last_active":"2023-06-10T21:26:44.213433+0000","last_peered":"2023-06-10T21:26:44.213433+0000","last_clean":"2023-06-10T21:26:20.308366+0000","last_became_active":"2023-06-10T21:26:27.812850+0000","last_became_peered":"2023-06-10T21:26:27.812850+0000","last_unstale":"2023-06-10T21:26:44.213433+0000","last_undegraded":"2023-06-10T21:26:44.213433+0000","last_fullsized":"2023-06-10T21:26:27.783047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:47:49.320329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"224'5104","reported_seq":6850,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:58.724978+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:26:58.724978+0000","last_peered":"2023-06-10T21:26:58.724978+0000","last_clean":"2023-06-10T21:26:58.724978+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:26:58.724978+0000","last_undegraded":"2023-06-10T21:26:58.724978+0000","last_fullsized":"2023-06-10T21:26:58.724978+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5104,"log_dups_size":0,"ondisk_log_size":5104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334495744,"num_objects":5104,"num_object_clones":0,"num_object_copies":15312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5104,"num_write_kb":326656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":44,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.214126+0000","last_change":"2023-06-10T21:26:27.818621+0000","last_active":"2023-06-10T21:26:44.214126+0000","last_peered":"2023-06-10T21:26:44.214126+0000","last_clean":"2023-06-10T21:26:20.307980+0000","last_became_active":"2023-06-10T21:26:27.818621+0000","last_became_peered":"2023-06-10T21:26:27.818621+0000","last_unstale":"2023-06-10T21:26:44.214126+0000","last_undegraded":"2023-06-10T21:26:44.214126+0000","last_fullsized":"2023-06-10T21:26:27.812688+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:54:17.367733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.215633+0000","last_change":"2023-06-10T21:26:15.754532+0000","last_active":"2023-06-10T21:26:44.215633+0000","last_peered":"2023-06-10T21:26:44.215633+0000","last_clean":"2023-06-10T21:26:44.215633+0000","last_became_active":"2023-06-10T21:26:15.754198+0000","last_became_peered":"2023-06-10T21:26:15.754198+0000","last_unstale":"2023-06-10T21:26:44.215633+0000","last_undegraded":"2023-06-10T21:26:44.215633+0000","last_fullsized":"2023-06-10T21:26:44.215633+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:20:05.494985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.221769+0000","last_change":"2023-06-10T21:25:45.632992+0000","last_active":"2023-06-10T21:26:44.221769+0000","last_peered":"2023-06-10T21:26:44.221769+0000","last_clean":"2023-06-10T21:26:44.221769+0000","last_became_active":"2023-06-10T21:25:45.632580+0000","last_became_peered":"2023-06-10T21:25:45.632580+0000","last_unstale":"2023-06-10T21:26:44.221769+0000","last_undegraded":"2023-06-10T21:26:44.221769+0000","last_fullsized":"2023-06-10T21:26:44.221769+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:08:44.568827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"224'5043","reported_seq":7661,"reported_epoch":224,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-10T21:26:56.398774+0000","last_change":"2023-06-10T21:26:30.373499+0000","last_active":"2023-06-10T21:26:56.398774+0000","last_peered":"2023-06-10T21:26:56.398774+0000","last_clean":"2023-06-10T21:26:15.547314+0000","last_became_active":"2023-06-10T21:26:21.397611+0000","last_became_peered":"2023-06-10T21:26:21.397611+0000","last_unstale":"2023-06-10T21:26:56.398774+0000","last_undegraded":"2023-06-10T21:26:56.398774+0000","last_fullsized":"2023-06-10T21:26:16.638153+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5043,"log_dups_size":0,"ondisk_log_size":5043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:49:10.119041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330498048,"num_objects":5043,"num_object_clones":0,"num_object_copies":15129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1863,"num_objects_unfound":0,"num_objects_dirty":5043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5043,"num_write_kb":322752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1287,"num_bytes_recovered":84344832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1","objects":3180},{"shards":"0,1,6","objects":1863}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":217,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.216684+0000","last_change":"2023-06-10T21:26:27.819291+0000","last_active":"2023-06-10T21:26:44.216684+0000","last_peered":"2023-06-10T21:26:44.216684+0000","last_clean":"2023-06-10T21:26:20.308048+0000","last_became_active":"2023-06-10T21:26:27.819291+0000","last_became_peered":"2023-06-10T21:26:27.819291+0000","last_unstale":"2023-06-10T21:26:44.216684+0000","last_undegraded":"2023-06-10T21:26:44.216684+0000","last_fullsized":"2023-06-10T21:26:27.818075+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:00:43.956182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":496,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.238746+0000","last_change":"2023-06-10T21:26:18.680332+0000","last_active":"2023-06-10T21:26:44.238746+0000","last_peered":"2023-06-10T21:26:44.238746+0000","last_clean":"2023-06-10T21:26:44.238746+0000","last_became_active":"2023-06-10T21:26:18.680066+0000","last_became_peered":"2023-06-10T21:26:18.680066+0000","last_unstale":"2023-06-10T21:26:44.238746+0000","last_undegraded":"2023-06-10T21:26:44.238746+0000","last_fullsized":"2023-06-10T21:26:44.238746+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":177,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.228088+0000","last_change":"2023-06-10T21:26:27.785005+0000","last_active":"2023-06-10T21:26:44.228088+0000","last_peered":"2023-06-10T21:26:44.228088+0000","last_clean":"2023-06-10T21:26:05.731398+0000","last_became_active":"2023-06-10T21:26:27.785005+0000","last_became_peered":"2023-06-10T21:26:27.785005+0000","last_unstale":"2023-06-10T21:26:44.228088+0000","last_undegraded":"2023-06-10T21:26:44.228088+0000","last_fullsized":"2023-06-10T21:26:27.779673+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":23,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.237240+0000","last_change":"2023-06-10T21:26:15.747662+0000","last_active":"2023-06-10T21:26:44.237240+0000","last_peered":"2023-06-10T21:26:44.237240+0000","last_clean":"2023-06-10T21:26:44.237240+0000","last_became_active":"2023-06-10T21:26:15.747331+0000","last_became_peered":"2023-06-10T21:26:15.747331+0000","last_unstale":"2023-06-10T21:26:44.237240+0000","last_undegraded":"2023-06-10T21:26:44.237240+0000","last_fullsized":"2023-06-10T21:26:44.237240+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"224'5114","reported_seq":5162,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:56.388593+0000","last_change":"2023-06-10T21:25:30.191474+0000","last_active":"2023-06-10T21:26:56.388593+0000","last_peered":"2023-06-10T21:26:56.388593+0000","last_clean":"2023-06-10T21:26:56.388593+0000","last_became_active":"2023-06-10T21:25:30.191133+0000","last_became_peered":"2023-06-10T21:25:30.191133+0000","last_unstale":"2023-06-10T21:26:56.388593+0000","last_undegraded":"2023-06-10T21:26:56.388593+0000","last_fullsized":"2023-06-10T21:26:56.388593+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:09:11.858115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335151104,"num_objects":5114,"num_object_clones":0,"num_object_copies":15342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5114,"num_write_kb":327296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":228,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.234381+0000","last_change":"2023-06-10T21:24:45.142686+0000","last_active":"2023-06-10T21:26:44.234381+0000","last_peered":"2023-06-10T21:26:44.234381+0000","last_clean":"2023-06-10T21:26:44.234381+0000","last_became_active":"2023-06-10T21:24:45.142396+0000","last_became_peered":"2023-06-10T21:24:45.142396+0000","last_unstale":"2023-06-10T21:26:44.234381+0000","last_undegraded":"2023-06-10T21:26:44.234381+0000","last_fullsized":"2023-06-10T21:26:44.234381+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":69,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:51.513277+0000","last_change":"2023-06-10T21:26:27.785270+0000","last_active":"2023-06-10T21:26:51.513277+0000","last_peered":"2023-06-10T21:26:51.513277+0000","last_clean":"2023-06-10T21:26:20.391145+0000","last_became_active":"2023-06-10T21:26:27.785270+0000","last_became_peered":"2023-06-10T21:26:27.785270+0000","last_unstale":"2023-06-10T21:26:51.513277+0000","last_undegraded":"2023-06-10T21:26:51.513277+0000","last_fullsized":"2023-06-10T21:26:27.778144+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:13.225551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":81,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:55.287200+0000","last_change":"2023-06-10T21:25:30.194023+0000","last_active":"2023-06-10T21:26:55.287200+0000","last_peered":"2023-06-10T21:26:55.287200+0000","last_clean":"2023-06-10T21:26:55.287200+0000","last_became_active":"2023-06-10T21:25:30.193693+0000","last_became_peered":"2023-06-10T21:25:30.193693+0000","last_unstale":"2023-06-10T21:26:55.287200+0000","last_undegraded":"2023-06-10T21:26:55.287200+0000","last_fullsized":"2023-06-10T21:26:55.287200+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:46:39.703458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"224'5105","reported_seq":5662,"reported_epoch":224,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:26:56.372013+0000","last_change":"2023-06-10T21:26:30.366646+0000","last_active":"2023-06-10T21:26:56.372013+0000","last_peered":"2023-06-10T21:26:56.372013+0000","last_clean":"2023-06-10T21:26:56.372013+0000","last_became_active":"2023-06-10T21:26:28.821113+0000","last_became_peered":"2023-06-10T21:26:28.821113+0000","last_unstale":"2023-06-10T21:26:56.372013+0000","last_undegraded":"2023-06-10T21:26:56.372013+0000","last_fullsized":"2023-06-10T21:26:56.372013+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5105,"log_dups_size":0,"ondisk_log_size":5105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:17:52.662271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5105,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5105,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":251,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":5105}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":213,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.237026+0000","last_change":"2023-06-10T21:24:45.139255+0000","last_active":"2023-06-10T21:26:44.237026+0000","last_peered":"2023-06-10T21:26:44.237026+0000","last_clean":"2023-06-10T21:26:44.237026+0000","last_became_active":"2023-06-10T21:24:45.138895+0000","last_became_peered":"2023-06-10T21:24:45.138895+0000","last_unstale":"2023-06-10T21:26:44.237026+0000","last_undegraded":"2023-06-10T21:26:44.237026+0000","last_fullsized":"2023-06-10T21:26:44.237026+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.213291+0000","last_change":"2023-06-10T21:25:45.624738+0000","last_active":"2023-06-10T21:26:44.213291+0000","last_peered":"2023-06-10T21:26:44.213291+0000","last_clean":"2023-06-10T21:26:44.213291+0000","last_became_active":"2023-06-10T21:25:45.624241+0000","last_became_peered":"2023-06-10T21:25:45.624241+0000","last_unstale":"2023-06-10T21:26:44.213291+0000","last_undegraded":"2023-06-10T21:26:44.213291+0000","last_fullsized":"2023-06-10T21:26:44.213291+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:16:55.688963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":71,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:51.513402+0000","last_change":"2023-06-10T21:25:45.632825+0000","last_active":"2023-06-10T21:26:51.513402+0000","last_peered":"2023-06-10T21:26:51.513402+0000","last_clean":"2023-06-10T21:26:51.513402+0000","last_became_active":"2023-06-10T21:25:45.632493+0000","last_became_peered":"2023-06-10T21:25:45.632493+0000","last_unstale":"2023-06-10T21:26:51.513402+0000","last_undegraded":"2023-06-10T21:26:51.513402+0000","last_fullsized":"2023-06-10T21:26:51.513402+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:07.022926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"224'5111","reported_seq":5186,"reported_epoch":224,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:26:58.654060+0000","last_change":"2023-06-10T21:26:28.810386+0000","last_active":"2023-06-10T21:26:58.654060+0000","last_peered":"2023-06-10T21:26:58.654060+0000","last_clean":"2023-06-10T21:26:15.570081+0000","last_became_active":"2023-06-10T21:26:28.807447+0000","last_became_peered":"2023-06-10T21:26:28.807447+0000","last_unstale":"2023-06-10T21:26:58.654060+0000","last_undegraded":"2023-06-10T21:26:28.793793+0000","last_fullsized":"2023-06-10T21:26:58.654060+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:20.877709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334954496,"num_objects":5111,"num_object_clones":0,"num_object_copies":15333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":264,"num_objects_misplaced":4847,"num_objects_unfound":0,"num_objects_dirty":5111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":327104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":264},{"shards":"2,4,6","objects":4847}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":33,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.235031+0000","last_change":"2023-06-10T21:26:27.787375+0000","last_active":"2023-06-10T21:26:44.235031+0000","last_peered":"2023-06-10T21:26:44.235031+0000","last_clean":"2023-06-10T20:59:27.427422+0000","last_became_active":"2023-06-10T21:26:27.787375+0000","last_became_peered":"2023-06-10T21:26:27.787375+0000","last_unstale":"2023-06-10T21:26:44.235031+0000","last_undegraded":"2023-06-10T21:26:27.783256+0000","last_fullsized":"2023-06-10T21:26:27.783170+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":130,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.234350+0000","last_change":"2023-06-10T21:26:27.788119+0000","last_active":"2023-06-10T21:26:44.234350+0000","last_peered":"2023-06-10T21:26:44.234350+0000","last_clean":"2023-06-10T20:58:33.354602+0000","last_became_active":"2023-06-10T21:26:27.788119+0000","last_became_peered":"2023-06-10T21:26:27.788119+0000","last_unstale":"2023-06-10T21:26:44.234350+0000","last_undegraded":"2023-06-10T21:26:44.234350+0000","last_fullsized":"2023-06-10T21:26:27.776800+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:41.776787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":137,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.238313+0000","last_change":"2023-06-10T21:26:27.787854+0000","last_active":"2023-06-10T21:26:44.238313+0000","last_peered":"2023-06-10T21:26:44.238313+0000","last_clean":"2023-06-10T21:24:36.651653+0000","last_became_active":"2023-06-10T21:26:27.787854+0000","last_became_peered":"2023-06-10T21:26:27.787854+0000","last_unstale":"2023-06-10T21:26:44.238313+0000","last_undegraded":"2023-06-10T21:26:44.238313+0000","last_fullsized":"2023-06-10T21:26:27.781321+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"224'5044","reported_seq":5106,"reported_epoch":224,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:26:58.694156+0000","last_change":"2023-06-10T21:26:28.977278+0000","last_active":"2023-06-10T21:26:58.694156+0000","last_peered":"2023-06-10T21:26:58.694156+0000","last_clean":"2023-06-10T21:26:15.549220+0000","last_became_active":"2023-06-10T21:26:28.973820+0000","last_became_peered":"2023-06-10T21:26:28.973820+0000","last_unstale":"2023-06-10T21:26:58.694156+0000","last_undegraded":"2023-06-10T21:26:28.947847+0000","last_fullsized":"2023-06-10T21:26:58.694156+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:46.994093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330563584,"num_objects":5044,"num_object_clones":0,"num_object_copies":15132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3335,"num_objects_misplaced":4764,"num_objects_unfound":0,"num_objects_dirty":5044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":322816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2"],"object_location_counts":[{"shards":"2,4","objects":280},{"shards":"2,4,6","objects":1709},{"shards":"2,6","objects":3055}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":205,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.235275+0000","last_change":"2023-06-10T21:26:27.818580+0000","last_active":"2023-06-10T21:26:44.235275+0000","last_peered":"2023-06-10T21:26:44.235275+0000","last_clean":"2023-06-10T21:26:20.562258+0000","last_became_active":"2023-06-10T21:26:27.818580+0000","last_became_peered":"2023-06-10T21:26:27.818580+0000","last_unstale":"2023-06-10T21:26:44.235275+0000","last_undegraded":"2023-06-10T21:26:44.235275+0000","last_fullsized":"2023-06-10T21:26:27.783435+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":67,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-06-10T21:26:55.287285+0000","last_change":"2023-06-10T21:24:45.166159+0000","last_active":"2023-06-10T21:26:55.287285+0000","last_peered":"2023-06-10T21:26:55.287285+0000","last_clean":"2023-06-10T21:26:55.287285+0000","last_became_active":"2023-06-10T21:24:45.147781+0000","last_became_peered":"2023-06-10T21:24:45.147781+0000","last_unstale":"2023-06-10T21:26:55.287285+0000","last_undegraded":"2023-06-10T21:26:55.287285+0000","last_fullsized":"2023-06-10T21:26:55.287285+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:25:35.150006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":178,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.218195+0000","last_change":"2023-06-10T21:26:15.749404+0000","last_active":"2023-06-10T21:26:44.218195+0000","last_peered":"2023-06-10T21:26:44.218195+0000","last_clean":"2023-06-10T21:26:44.218195+0000","last_became_active":"2023-06-10T21:26:15.748925+0000","last_became_peered":"2023-06-10T21:26:15.748925+0000","last_unstale":"2023-06-10T21:26:44.218195+0000","last_undegraded":"2023-06-10T21:26:44.218195+0000","last_fullsized":"2023-06-10T21:26:44.218195+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:13.913738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":61,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.218999+0000","last_change":"2023-06-10T21:24:45.144952+0000","last_active":"2023-06-10T21:26:44.218999+0000","last_peered":"2023-06-10T21:26:44.218999+0000","last_clean":"2023-06-10T21:26:44.218999+0000","last_became_active":"2023-06-10T21:24:45.144474+0000","last_became_peered":"2023-06-10T21:24:45.144474+0000","last_unstale":"2023-06-10T21:26:44.218999+0000","last_undegraded":"2023-06-10T21:26:44.218999+0000","last_fullsized":"2023-06-10T21:26:44.218999+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:25:09.157193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":53,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.213141+0000","last_change":"2023-06-10T21:26:27.817806+0000","last_active":"2023-06-10T21:26:44.213141+0000","last_peered":"2023-06-10T21:26:44.213141+0000","last_clean":"2023-06-10T21:26:20.308187+0000","last_became_active":"2023-06-10T21:26:27.817806+0000","last_became_peered":"2023-06-10T21:26:27.817806+0000","last_unstale":"2023-06-10T21:26:44.213141+0000","last_undegraded":"2023-06-10T21:26:44.213141+0000","last_fullsized":"2023-06-10T21:26:27.783382+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:28:39.270974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":24,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:44.234486+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:26:44.234486+0000","last_peered":"2023-06-10T21:26:44.234486+0000","last_clean":"2023-06-10T21:26:44.234486+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:26:44.234486+0000","last_undegraded":"2023-06-10T21:26:44.234486+0000","last_fullsized":"2023-06-10T21:26:44.234486+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.228495+0000","last_change":"2023-06-10T21:26:27.782629+0000","last_active":"2023-06-10T21:26:44.228495+0000","last_peered":"2023-06-10T21:26:44.228495+0000","last_clean":"2023-06-10T21:19:00.489490+0000","last_became_active":"2023-06-10T21:26:27.782629+0000","last_became_peered":"2023-06-10T21:26:27.782629+0000","last_unstale":"2023-06-10T21:26:44.228495+0000","last_undegraded":"2023-06-10T21:26:44.228495+0000","last_fullsized":"2023-06-10T21:26:27.779384+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":46,"reported_epoch":223,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:26:44.229874+0000","last_change":"2023-06-10T21:26:27.784405+0000","last_active":"2023-06-10T21:26:44.229874+0000","last_peered":"2023-06-10T21:26:44.229874+0000","last_clean":"2023-06-10T21:25:59.679038+0000","last_became_active":"2023-06-10T21:26:27.784405+0000","last_became_peered":"2023-06-10T21:26:27.784405+0000","last_unstale":"2023-06-10T21:26:44.229874+0000","last_undegraded":"2023-06-10T21:26:27.780168+0000","last_fullsized":"2023-06-10T21:26:27.780047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":178,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.218324+0000","last_change":"2023-06-10T21:26:27.785614+0000","last_active":"2023-06-10T21:26:44.218324+0000","last_peered":"2023-06-10T21:26:44.218324+0000","last_clean":"2023-06-10T21:24:36.631470+0000","last_became_active":"2023-06-10T21:26:27.785614+0000","last_became_peered":"2023-06-10T21:26:27.785614+0000","last_unstale":"2023-06-10T21:26:44.218324+0000","last_undegraded":"2023-06-10T21:26:44.218324+0000","last_fullsized":"2023-06-10T21:26:27.776892+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:46.260297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":197,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.213630+0000","last_change":"2023-06-10T21:26:27.818384+0000","last_active":"2023-06-10T21:26:44.213630+0000","last_peered":"2023-06-10T21:26:44.213630+0000","last_clean":"2023-06-10T21:25:28.189114+0000","last_became_active":"2023-06-10T21:26:27.818384+0000","last_became_peered":"2023-06-10T21:26:27.818384+0000","last_unstale":"2023-06-10T21:26:44.213630+0000","last_undegraded":"2023-06-10T21:26:44.213630+0000","last_fullsized":"2023-06-10T21:26:27.812469+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:26:46.689756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":227,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.213940+0000","last_change":"2023-06-10T21:26:27.820844+0000","last_active":"2023-06-10T21:26:44.213940+0000","last_peered":"2023-06-10T21:26:44.213940+0000","last_clean":"2023-06-10T21:25:28.188991+0000","last_became_active":"2023-06-10T21:26:27.820844+0000","last_became_peered":"2023-06-10T21:26:27.820844+0000","last_unstale":"2023-06-10T21:26:44.213940+0000","last_undegraded":"2023-06-10T21:26:44.213940+0000","last_fullsized":"2023-06-10T21:26:27.817733+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:10.760495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":129,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-06-10T21:26:51.513548+0000","last_change":"2023-06-10T21:26:15.944630+0000","last_active":"2023-06-10T21:26:51.513548+0000","last_peered":"2023-06-10T21:26:51.513548+0000","last_clean":"2023-06-10T21:26:51.513548+0000","last_became_active":"2023-06-10T21:26:15.750701+0000","last_became_peered":"2023-06-10T21:26:15.750701+0000","last_unstale":"2023-06-10T21:26:51.513548+0000","last_undegraded":"2023-06-10T21:26:51.513548+0000","last_fullsized":"2023-06-10T21:26:51.513548+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:47:02.331086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":205,"reported_epoch":223,"state":"active+undersized","last_fresh":"2023-06-10T21:26:44.228036+0000","last_change":"2023-06-10T21:26:27.791062+0000","last_active":"2023-06-10T21:26:44.228036+0000","last_peered":"2023-06-10T21:26:44.228036+0000","last_clean":"2023-06-10T21:25:15.648591+0000","last_became_active":"2023-06-10T21:26:27.791062+0000","last_became_peered":"2023-06-10T21:26:27.791062+0000","last_unstale":"2023-06-10T21:26:44.228036+0000","last_undegraded":"2023-06-10T21:26:44.228036+0000","last_fullsized":"2023-06-10T21:26:27.779951+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5344526336,"num_objects":81551,"num_object_clones":0,"num_object_copies":244653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24083,"num_objects_misplaced":26527,"num_objects_unfound":0,"num_objects_dirty":81551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81551,"num_write_kb":5219264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7143,"num_bytes_recovered":468123648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15896215552,"data_stored":15896215552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81551,"ondisk_log_size":81551,"up":40,"acting":43,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":7738,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":78,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430559,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2037672,"kb_used_data":1542176,"kb_used_omap":0,"kb_used_meta":495488,"kb_avail":91707480,"statfs":{"total":95995035648,"available":93908459520,"internally_reserved":0,"allocated":1579188224,"data_stored":1578656940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":507379712},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:26:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":3.375,"15min":2.0419999999999998},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":42.994,"5min":53.706000000000003,"15min":53.706000000000003},"last":3.3359999999999999},{"interface":"front","average":{"1min":2.9710000000000001,"5min":3.1659999999999999,"15min":2.2189999999999999},"min":{"1min":0.32500000000000001,"5min":0.247,"15min":0.245},"max":{"1min":43.348999999999997,"5min":53.777000000000001,"15min":53.777000000000001},"last":3.7170000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:25:58 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":4.4870000000000001,"15min":2.5179999999999998},"min":{"1min":0.36899999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":3.1520000000000001,"5min":130.19399999999999,"15min":130.19399999999999},"last":3.9049999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":6.1310000000000002,"15min":2.9969999999999999},"min":{"1min":0.36899999999999999,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":1.698,"5min":212.20599999999999,"15min":212.20599999999999},"last":0.73399999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:26:11 2023","interfaces":[{"interface":"back","average":{"1min":10.27,"5min":3.996,"15min":1.681},"min":{"1min":0.34799999999999998,"5min":0.317,"15min":0.315},"max":{"1min":160.99299999999999,"5min":160.99299999999999,"15min":160.99299999999999},"last":3.7280000000000002},{"interface":"front","average":{"1min":10.010999999999999,"5min":6.0289999999999999,"15min":2.3700000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.309},"max":{"1min":160.53700000000001,"5min":212.13399999999999,"15min":212.13399999999999},"last":0.71199999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":5.0629999999999997,"15min":2.0619999999999998},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":14.526,"5min":63.406999999999996,"15min":63.406999999999996},"last":0.44},{"interface":"front","average":{"1min":0.83199999999999996,"5min":5.0449999999999999,"15min":2.0720000000000001},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4319999999999999,"5min":63.549999999999997,"15min":63.549999999999997},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.097,"15min":0.97199999999999998},"min":{"1min":0.311,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":2.702,"5min":8.1419999999999995,"15min":8.1419999999999995},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.887,"15min":1.347},"min":{"1min":0.29099999999999998,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":3.3610000000000002,"5min":14.657999999999999,"15min":14.657999999999999},"last":3.202}]},{"osd":5,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":3.8610000000000002,"15min":2.0920000000000001},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":0.69599999999999995,"5min":211.99799999999999,"15min":211.99799999999999},"last":3.0430000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":3.7109999999999999,"15min":1.9490000000000001},"min":{"1min":0.35499999999999998,"5min":0.315,"15min":0.30199999999999999},"max":{"1min":2.1859999999999999,"5min":212.23699999999999,"15min":212.23699999999999},"last":2.2109999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.663,"15min":1.2330000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":2.1469999999999998,"5min":10.159000000000001,"15min":10.159000000000001},"last":3.0129999999999999},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.6890000000000001,"15min":1.151},"min":{"1min":0.39600000000000002,"5min":0.374,"15min":0.32800000000000001},"max":{"1min":2.2130000000000001,"5min":10.874000000000001,"15min":10.874000000000001},"last":0.68500000000000005}]}]},{"osd":6,"up_from":0,"seq":180388626787,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566229,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1697680,"kb_used_data":1285192,"kb_used_omap":4,"kb_used_meta":412475,"kb_avail":92047472,"statfs":{"total":95995035648,"available":94256611328,"internally_reserved":0,"allocated":1316036608,"data_stored":1315293949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":422375310},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:26:00 2023","interfaces":[{"interface":"back","average":{"1min":8.0519999999999996,"5min":6.1719999999999997,"15min":3.2000000000000002},"min":{"1min":0.23699999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":121.917,"5min":121.917,"15min":121.917},"last":0.253},{"interface":"front","average":{"1min":8.5020000000000007,"5min":6.9699999999999998,"15min":3.7919999999999998},"min":{"1min":0.312,"5min":0.25,"15min":0.22600000000000001},"max":{"1min":121.455,"5min":121.455,"15min":121.455},"last":2.2549999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:26:18 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5680000000000001,"15min":1.2},"min":{"1min":0.255,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7990000000000004,"5min":18.152000000000001,"15min":18.152000000000001},"last":2.169},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.6140000000000001,"15min":0.85699999999999998},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.218},"max":{"1min":1.278,"5min":12.081,"15min":12.081},"last":0.28000000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:25:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":4.2030000000000003,"15min":1.641},"min":{"1min":0.29299999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65200000000000002,"5min":118.348,"15min":118.348},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":4.9160000000000004,"15min":1.9159999999999999},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.1870000000000003,"5min":117.75,"15min":117.75},"last":5.1109999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:26:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.9570000000000001,"15min":1.8320000000000001},"min":{"1min":0.39500000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.6319999999999997,"5min":12.507999999999999,"15min":12.94},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.736,"15min":2.1040000000000001},"min":{"1min":0.32300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.3049999999999999,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:26:00 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":9.2219999999999995,"15min":4.4249999999999998},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.31900000000000001},"max":{"1min":4.6139999999999999,"5min":383.83100000000002,"15min":383.83100000000002},"last":2.6139999999999999},{"interface":"front","average":{"1min":0.85999999999999999,"5min":8.6980000000000004,"15min":4.3979999999999997},"min":{"1min":0.41599999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.7050000000000001,"5min":383.84800000000001,"15min":383.84800000000001},"last":2.6499999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":2.6949999999999998,"15min":2.266},"min":{"1min":0.40899999999999997,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":13.465,"5min":14.858000000000001,"15min":14.858000000000001},"last":2.2229999999999999},{"interface":"front","average":{"1min":2.056,"5min":2.2200000000000002,"15min":2.2330000000000001},"min":{"1min":0.47299999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":13.48,"5min":13.48,"15min":13.48},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:26:00 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":4.4180000000000001,"15min":3.2989999999999999},"min":{"1min":0.47099999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":10.285,"5min":162.399,"15min":162.399},"last":1.0549999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":3.6699999999999999,"15min":2.5539999999999998},"min":{"1min":0.45900000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.9379999999999999,"5min":162.27199999999999,"15min":162.27199999999999},"last":1.526}]}]},{"osd":0,"up_from":86,"seq":369367187651,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3738312,"kb_used_data":3241024,"kb_used_omap":5,"kb_used_meta":497274,"kb_avail":90006840,"statfs":{"total":95995035648,"available":92167004160,"internally_reserved":0,"allocated":3318808576,"data_stored":3318012791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":509209210},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:25:58 2023","interfaces":[{"interface":"back","average":{"1min":5.9900000000000002,"5min":8.1509999999999998,"15min":3.9660000000000002},"min":{"1min":0.23300000000000001,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":97.390000000000001,"5min":175.352,"15min":175.352},"last":2.637},{"interface":"front","average":{"1min":6.1390000000000002,"5min":7.8209999999999997,"15min":4.4980000000000002},"min":{"1min":0.27000000000000002,"5min":0.189,"15min":0.189},"max":{"1min":97.058999999999997,"5min":175.30500000000001,"15min":175.30500000000001},"last":3.4790000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":2.0030000000000001,"15min":0.90600000000000003},"min":{"1min":0.224,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.1299999999999999,"5min":14.542999999999999,"15min":14.542999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.093,"5min":2,"15min":0.92100000000000004},"min":{"1min":0.26200000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":6.3010000000000002,"5min":16.782,"15min":16.782},"last":1.6679999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:25:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":8.7550000000000008,"15min":3.1859999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":1.524,"5min":181.886,"15min":181.886},"last":0.371},{"interface":"front","average":{"1min":0.70199999999999996,"5min":9.5440000000000005,"15min":3.4390000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9049999999999998,"5min":180.04900000000001,"15min":180.04900000000001},"last":3.8759999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":2.012,"15min":1.95},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.4669999999999996,"5min":10.733000000000001,"15min":10.948},"last":6.2199999999999998},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.821,"15min":1.6830000000000001},"min":{"1min":0.38300000000000001,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":2.5190000000000001,"5min":25.491,"15min":25.491},"last":9.0679999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":2.177,"15min":1.903},"min":{"1min":0.42499999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.4420000000000002,"5min":25.545000000000002,"15min":25.545000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.96099999999999997,"5min":1.9870000000000001,"15min":2.206},"min":{"1min":0.46000000000000002,"5min":0.375,"15min":0.32400000000000001},"max":{"1min":3.972,"5min":8.5879999999999992,"15min":17.460000000000001},"last":7.2729999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":2.1059999999999999,"15min":1.956},"min":{"1min":0.45200000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.7400000000000002,"5min":25.608000000000001,"15min":25.608000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.111,"5min":2.1909999999999998,"15min":2.3130000000000002},"min":{"1min":0.41699999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":3.726,"5min":10.231999999999999,"15min":17.411000000000001},"last":8.1099999999999994}]},{"osd":7,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":2.1930000000000001,"15min":2.2909999999999999},"min":{"1min":0.45200000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.0090000000000003,"5min":10.269,"15min":17.486999999999998},"last":4.2190000000000003},{"interface":"front","average":{"1min":0.746,"5min":2.1219999999999999,"15min":2.0720000000000001},"min":{"1min":0.39500000000000002,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":2.0960000000000001,"5min":10.691000000000001,"15min":10.867000000000001},"last":0.73899999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075618,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2903108,"kb_used_data":2597948,"kb_used_omap":7,"kb_used_meta":305144,"kb_avail":90842044,"statfs":{"total":95995035648,"available":93022253056,"internally_reserved":0,"allocated":2660298752,"data_stored":2659568457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":312467537},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,8],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":2.2189999999999999,"15min":1.0149999999999999},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.3120000000000003,"5min":31.873000000000001,"15min":31.873000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":3.0230000000000001,"15min":1.256},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.8529999999999998,"5min":62.451999999999998,"15min":62.451999999999998},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.5139999999999998,"15min":1.095},"min":{"1min":0.23100000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.3079999999999998,"5min":29.486000000000001,"15min":29.486000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":2.23,"15min":1.0429999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":2.2290000000000001,"5min":28.013999999999999,"15min":28.013999999999999},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":4.2549999999999999,"15min":1.6839999999999999},"min":{"1min":0.32200000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6989999999999998,"5min":96.295000000000002,"15min":96.295000000000002},"last":4.157},{"interface":"front","average":{"1min":1.0449999999999999,"5min":4.8460000000000001,"15min":1.899},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.9649999999999999,"5min":99.504999999999995,"15min":99.504999999999995},"last":2.9489999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.851,"15min":1.0569999999999999},"min":{"1min":0.34699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.2880000000000003,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":60.981000000000002},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.97,"15min":1.0960000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.2290000000000001,"5min":17.295000000000002,"15min":17.295000000000002},"last":60.279000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.7210000000000001,"15min":1.0469999999999999},"min":{"1min":0.39300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.399,"5min":10.254,"15min":10.254},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.125,"5min":2.254,"15min":1.2},"min":{"1min":0.375,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.6829999999999998,"5min":10.345000000000001,"15min":10.345000000000001},"last":0.441}]},{"osd":6,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":2.1549999999999998,"15min":1.1919999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.3740000000000006,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2410000000000001,"5min":2.4209999999999998,"15min":1.288},"min":{"1min":0.44600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.5439999999999996,"5min":17.244,"15min":17.244},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:26:17 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.413,"15min":0.98199999999999998},"min":{"1min":0.42199999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.4320000000000004,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.423,"5min":2.1909999999999998,"15min":1.22},"min":{"1min":0.436,"5min":0.434,"15min":0.434},"max":{"1min":8.3369999999999997,"5min":8.8140000000000001,"15min":8.8140000000000001},"last":0.48199999999999998}]}]},{"osd":3,"up_from":125,"seq":536870912085,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267612,"kb_used_data":1002452,"kb_used_omap":2,"kb_used_meta":265149,"kb_avail":92477540,"statfs":{"total":95995035648,"available":94697000960,"internally_reserved":0,"allocated":1026510848,"data_stored":1025770762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":271512629},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":2.234,"15min":0.98599999999999999},"min":{"1min":0.29399999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.0419999999999998,"5min":46.677999999999997,"15min":46.677999999999997},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.5049999999999999,"5min":4.2110000000000003,"15min":1.639},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":10.273999999999999,"5min":130.893,"15min":130.893},"last":5.5300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.6869999999999998,"15min":1.1639999999999999},"min":{"1min":0.33900000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.0030000000000001,"5min":54.112000000000002,"15min":54.112000000000002},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":3.282,"15min":1.3700000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29299999999999998},"max":{"1min":8.2810000000000006,"5min":57.814,"15min":57.814},"last":5.5609999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":2.2639999999999998,"15min":1.014},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":10.047000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":5.4859999999999998},{"interface":"front","average":{"1min":1.514,"5min":1.7989999999999999,"15min":0.86099999999999999},"min":{"1min":0.313,"5min":0.249,"15min":0.249},"max":{"1min":6.0030000000000001,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.0409999999999999,"15min":1.0629999999999999},"min":{"1min":0.39100000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.3190000000000008,"5min":13.656000000000001,"15min":13.656000000000001},"last":2.4910000000000001},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.0089999999999999,"15min":1.052},"min":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.3200000000000003,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":9.0730000000000004,"15min":3.4220000000000002},"min":{"1min":0.498,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.3580000000000005,"5min":349.47399999999999,"15min":349.47399999999999},"last":2.448},{"interface":"front","average":{"1min":2.8180000000000001,"5min":8.0099999999999998,"15min":3.0760000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.3740000000000006,"5min":349.65100000000001,"15min":349.65100000000001},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.2610000000000001,"15min":1.1479999999999999},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.9800000000000004,"5min":9.9619999999999997,"15min":9.9619999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.675,"5min":2.0339999999999998,"15min":1.1220000000000001},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":8.3339999999999996,"5min":12.835000000000001,"15min":12.835000000000001},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":5.3150000000000004,"15min":2.1789999999999998},"min":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.2230000000000008,"5min":174.44800000000001,"15min":174.44800000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.6320000000000001,"5min":3.8570000000000002,"15min":1.734},"min":{"1min":0.40400000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9450000000000003,"5min":173.18100000000001,"15min":173.18100000000001},"last":0.72999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019243,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4273380,"kb_used_data":3796700,"kb_used_omap":0,"kb_used_meta":476672,"kb_avail":89471772,"statfs":{"total":95995035648,"available":91619094528,"internally_reserved":0,"allocated":3887820800,"data_stored":3887293307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":488112128},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":2.3599999999999999,"15min":1.7989999999999999},"min":{"1min":0.26400000000000001,"5min":0.255,"15min":0.22},"max":{"1min":2.1139999999999999,"5min":40.186999999999998,"15min":40.186999999999998},"last":1.113},{"interface":"front","average":{"1min":0.95199999999999996,"5min":2.758,"15min":2.1419999999999999},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.23100000000000001},"max":{"1min":3.6779999999999999,"5min":39.395000000000003,"15min":39.395000000000003},"last":1.018}]},{"osd":1,"last update":"Sat Jun 10 21:26:08 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":5.8399999999999999,"15min":3.0470000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.6400000000000001,"5min":182.035,"15min":182.035},"last":2.6240000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":5.1079999999999997,"15min":3.0510000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27000000000000002},"max":{"1min":1.972,"5min":181.73099999999999,"15min":181.73099999999999},"last":4.9930000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.6870000000000001,"15min":0.91300000000000003},"min":{"1min":0.372,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":2.4140000000000001,"5min":9.9610000000000003,"15min":9.9610000000000003},"last":1.1890000000000001},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.8089999999999999,"15min":1.0209999999999999},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.311},"max":{"1min":3.2370000000000001,"5min":13.888,"15min":13.888},"last":1.0600000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":6.9969999999999999,"5min":7.4119999999999999,"15min":2.8420000000000001},"min":{"1min":0.498,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":59.084000000000003,"5min":103.575,"15min":103.575},"last":6.601},{"interface":"front","average":{"1min":5.8949999999999996,"5min":6.7610000000000001,"15min":2.6669999999999998},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":59.128,"5min":103.565,"15min":103.565},"last":6.6120000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:26:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.8420000000000001,"15min":1.107},"min":{"1min":0.32400000000000001,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":2.5190000000000001,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.378},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.663,"15min":1.2150000000000001},"min":{"1min":0.35499999999999998,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":2.0649999999999999,"5min":11.904999999999999,"15min":11.904999999999999},"last":0.314}]},{"osd":6,"last update":"Sat Jun 10 21:26:42 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.782,"15min":1.284},"min":{"1min":0.35799999999999998,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":2.2530000000000001,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":0.436},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.843,"15min":1.282},"min":{"1min":0.379,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":2.448,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:26:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.1150000000000002,"15min":1.484},"min":{"1min":0.35299999999999998,"5min":0.34100000000000003,"15min":0.308},"max":{"1min":3.2999999999999998,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.875,"15min":1.298},"min":{"1min":0.34499999999999997,"5min":0.33300000000000002,"15min":0.314},"max":{"1min":3.7469999999999999,"5min":9.5060000000000002,"15min":9.5060000000000002},"last":0.46300000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823014,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3318415360,"data_stored":3318415360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1325989888,"data_stored":1325989888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2680422400,"data_stored":2680422400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1040711680,"data_stored":1040711680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3886088192,"data_stored":3886088192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":15204352,"data_stored":15204352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2048786432,"data_stored":2048786432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1580597248,"data_stored":1580597248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:27:04.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_5 pgp_num 16 2023-06-10T21:27:04.289 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:04.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:04.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:04.377 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 86414 86398 64.2586 44.375 0.0251069 0.0155567 2023-06-10T21:27:04.405 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:04.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:04.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:04.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:04.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:04.907 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:04.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:05.023 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:05.111 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:05.265 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:05.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:05.377 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 87175 87159 64.0621 47.5625 0.0421271 0.0156037 2023-06-10T21:27:05.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:05.393 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:27:05.482 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:05.643 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:05 smithi152 ceph-mon[156819]: pgmap v795: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:05.643 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:05.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:05 smithi064 ceph-mon[186706]: pgmap v795: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:05.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:05.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:05 smithi064 ceph-mon[180297]: pgmap v795: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s; 24320/250449 objects degraded (9.711%); 26679/250449 objects misplaced (10.652%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:05.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:05.959 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:05.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:05.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:27:06.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:06.106 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:06.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: osdmap e225: 8 total, 7 up, 6 in 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 24320/250449 objects degraded (9.711%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:06.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:06.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:06.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: osdmap e225: 8 total, 7 up, 6 in 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.366 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 24320/250449 objects degraded (9.711%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1037936412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: osdmap e225: 8 total, 7 up, 6 in 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: failed to encode map e225 with expected crc 2023-06-10T21:27:06.368 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 24320/250449 objects degraded (9.711%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:06.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 87954 87938 63.8834 48.6875 0.0165341 0.0156488 2023-06-10T21:27:06.636 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:06.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:06.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:06.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:07.200 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:27:07.200 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:27:07.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 88712 88696 63.6936 47.375 0.00445502 0.0156954 2023-06-10T21:27:07.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:07.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:07.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:07.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:07 smithi064 ceph-mon[186706]: pgmap v797: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 799 op/s; 24772/255723 objects degraded (9.687%); 27318/255723 objects misplaced (10.683%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:07.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:07 smithi064 ceph-mon[180297]: pgmap v797: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 799 op/s; 24772/255723 objects degraded (9.687%); 27318/255723 objects misplaced (10.683%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:07 smithi152 ceph-mon[156819]: pgmap v797: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 50 MiB/s wr, 799 op/s; 24772/255723 objects degraded (9.687%); 27318/255723 objects misplaced (10.683%); 3.5 MiB/s, 55 objects/s recovering 2023-06-10T21:27:07.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:07.725 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:08.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:27:08.161 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:08.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:08.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:08.332 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:08.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 89419 89403 63.472 44.1875 0.0263386 0.015749 2023-06-10T21:27:08.443 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:08.493 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:08.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:08.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:08.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:08.730 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:08.843 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:08.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:08.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:08.995 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:09.090 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:09.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:09.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[180297]: pgmap v798: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s; 24973/261108 objects degraded (9.564%); 27582/261108 objects misplaced (10.563%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:09.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:09.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:09.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:09.280 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:09.354 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:09 smithi152 ceph-mon[156819]: pgmap v798: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s; 24973/261108 objects degraded (9.564%); 27582/261108 objects misplaced (10.563%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:09.354 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:09.354 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:09.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 90200 90184 63.3073 48.8125 0.019403 0.0157909 2023-06-10T21:27:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[186706]: pgmap v798: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 765 op/s; 24973/261108 objects degraded (9.564%); 27582/261108 objects misplaced (10.563%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:09.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:10.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:27:10.145 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:10.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:10.203 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pgp_num to 16 2023-06-10T21:27:10.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:10.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:10.357 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:10.358 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:10.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:10.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0179 2023-06-10T21:27:10.378 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 90753 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 63.0104 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 39.4219 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 194.188 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1008 2023-06-10T21:27:10.379 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 630.754 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3107 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0158658 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0828012 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 5.9617 2023-06-10T21:27:10.380 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00121051 2023-06-10T21:27:10.383 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:27:10.509 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:10.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:10.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:10.590 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:27:10.590 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:27:10.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:10.687 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:10.821 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:10.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:10.925 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:10.977 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:11.074 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:11.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:11.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:11.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:11.247 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:11.374 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: pgmap v799: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 734 op/s; 25456/266781 objects degraded (9.542%); 28314/266781 objects misplaced (10.613%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:11.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.375 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:11.376 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: osdmap e226: 8 total, 7 up, 6 in 2023-06-10T21:27:11.376 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25456/266781 objects degraded (9.542%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:11.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:27:11.516 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:11.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:11.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: pgmap v799: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 734 op/s; 25456/266781 objects degraded (9.542%); 28314/266781 objects misplaced (10.613%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:11.528 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:11.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: osdmap e226: 8 total, 7 up, 6 in 2023-06-10T21:27:11.530 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25456/266781 objects degraded (9.542%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:11.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: pgmap v799: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 46 MiB/s wr, 734 op/s; 25456/266781 objects degraded (9.542%); 28314/266781 objects misplaced (10.613%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:27:11.530 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: failed to encode map e226 with expected crc 2023-06-10T21:27:11.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2971015504' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:11.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: osdmap e226: 8 total, 7 up, 6 in 2023-06-10T21:27:11.531 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25456/266781 objects degraded (9.542%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:11.629 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:11.703 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:11.781 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:11.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:11.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:11.903 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:11.962 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:11.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:12.061 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:27:12.093 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:12.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:12.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:12.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:12.177 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:12.210 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/321889683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:12.213 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:12.213 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/321889683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:12.247 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:12.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:12.290 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:12.292 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:12.292 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.293 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:27:12.471 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:12.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:12.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/321889683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:12.577 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:12.635 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:12.718 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:12.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:12.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:12.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:12.848 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:12.892 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:12.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:12.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:13.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:13.083 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:12] "GET /metrics HTTP/1.1" 200 224043 "" "Prometheus/2.33.4" 2023-06-10T21:27:13.088 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:13.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:13 smithi064 ceph-mon[180297]: pgmap v801: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 25456/267303 objects degraded (9.523%); 28314/267303 objects misplaced (10.592%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:27:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:13 smithi064 ceph-mon[186706]: pgmap v801: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 25456/267303 objects degraded (9.523%); 28314/267303 objects misplaced (10.592%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:27:13.588 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:13 smithi152 ceph-mon[156819]: pgmap v801: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 44 MiB/s wr, 710 op/s; 25456/267303 objects degraded (9.523%); 28314/267303 objects misplaced (10.592%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:27:13.843 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:27:13.843 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:27:13.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.843 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:93239 10.c 10:3e3d4987:::benchmark_data_smithi064_312713_object2483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:27:13.843 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:13.825773+0000", 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.010137167000000001, 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0101739, 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:13.844 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 93239 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825773+0000", 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:13.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825773+0000", 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:13.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825771+0000", 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4239999999999999e-06 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825780+0000", 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9284999999999999e-05 2023-06-10T21:27:13.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825800+0000", 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2470000000000008e-06 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:13.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825808+0000", 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9209000000000002e-05 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825867+0000", 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016466699999999999 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826032+0000", 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015692899999999999 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826189+0000", 2023-06-10T21:27:13.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032407930000000001 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.829429+0000", 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003211148 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.832641+0000", 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0068678879999999999 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:93241 10.1 10:881edb58:::benchmark_data_smithi064_312713_object2485:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:13.826000+0000", 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0099096210000000004, 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010053681, 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:13.853 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 93241 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826000+0000", 2023-06-10T21:27:13.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826000+0000", 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.825999+0000", 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.336e-06 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826006+0000", 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1610000000000001e-05 2023-06-10T21:27:13.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826017+0000", 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1715e-05 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:13.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826029+0000", 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000311852 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826341+0000", 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8227999999999995e-05 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826439+0000", 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000137935 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:13.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.826577+0000", 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061093220000000004 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.832686+0000", 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051165300000000004 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.833198+0000", 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.007197719 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:93246 10.0 e226/214 10:04c46566:::benchmark_data_smithi064_312713_object2490:head v 226'5888, mlcod=203'1215)", 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:13.835255+0000", 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00065435099999999996, 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000888385, 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:13.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835255+0000", 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835255+0000", 2023-06-10T21:27:13.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835254+0000", 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1154e-05 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835266+0000", 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6569e-05 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835282+0000", 2023-06-10T21:27:13.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5310000000000004e-06 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835290+0000", 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4524000000000003e-05 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835334+0000", 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8810999999999999e-05 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:13.835363+0000", 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010749199999999999 2023-06-10T21:27:13.867 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 3 2023-06-10T21:27:13.868 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:27:14.065 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:14.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:14.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:14.204 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:14.249 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:14.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:14.353 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:14.385 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:14.427 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:14.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:14.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:14.589 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:14.643 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:14.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:27:14.847 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:14.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:14.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:14.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:15.009 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:15.106 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:15.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:15.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:15.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:15.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:15.362 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:15 smithi152 ceph-mon[156819]: pgmap v802: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 124 B/s rd, 39 MiB/s wr, 678 op/s; 25586/269559 objects degraded (9.492%); 28219/269559 objects misplaced (10.469%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:27:15.570 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:15.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:15.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:15.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:15 smithi064 ceph-mon[180297]: pgmap v802: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 124 B/s rd, 39 MiB/s wr, 678 op/s; 25586/269559 objects degraded (9.492%); 28219/269559 objects misplaced (10.469%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:27:15.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:15 smithi064 ceph-mon[186706]: pgmap v802: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 124 B/s rd, 39 MiB/s wr, 678 op/s; 25586/269559 objects degraded (9.492%); 28219/269559 objects misplaced (10.469%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:27:15.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:15.718 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:15.792 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:15.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:15.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:15.993 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:16.094 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:16.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:16.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:16.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:16.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:16.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:16.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:16.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:16.397 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-06-10T21:27:16.398 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:27:16.398 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-06-10T21:27:16.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:27:16.430 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:16.527 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:16.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:16.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: pgmap v803: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 725 op/s; 25142/267162 objects degraded (9.411%); 28005/267162 objects misplaced (10.482%); 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25586/269559 objects degraded (9.492%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2555360018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:27:16.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: pgmap v803: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 725 op/s; 25142/267162 objects degraded (9.411%); 28005/267162 objects misplaced (10.482%); 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25586/269559 objects degraded (9.492%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2555360018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:27:16.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:27:16.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:16 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:16.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:27:16.688 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:16.711 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:27:16.789 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:16.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: pgmap v803: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 102 B/s rd, 35 MiB/s wr, 725 op/s; 25142/267162 objects degraded (9.411%); 28005/267162 objects misplaced (10.482%); 5.0 MiB/s, 79 objects/s recovering 2023-06-10T21:27:16.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25586/269559 objects degraded (9.492%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:27:16.796 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2555360018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:16.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:27:16.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:27:16.797 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:16 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:16.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:16.899 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:16.954 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:17.137 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:17.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:17.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:17.273 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:27:17.273 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:27:17.273 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:27:17.274 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:27:17.274 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:27:17.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:17.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[180297]: osdmap e227: 8 total, 7 up, 6 in 2023-06-10T21:27:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[180297]: failed to encode map e227 with expected crc 2023-06-10T21:27:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[180297]: failed to encode map e227 with expected crc 2023-06-10T21:27:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[186706]: osdmap e227: 8 total, 7 up, 6 in 2023-06-10T21:27:17.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[186706]: failed to encode map e227 with expected crc 2023-06-10T21:27:17.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:17 smithi064 ceph-mon[186706]: failed to encode map e227 with expected crc 2023-06-10T21:27:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:17 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:17 smithi152 ceph-mon[156819]: osdmap e227: 8 total, 7 up, 6 in 2023-06-10T21:27:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:17 smithi152 ceph-mon[156819]: failed to encode map e227 with expected crc 2023-06-10T21:27:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:17 smithi152 ceph-mon[156819]: failed to encode map e227 with expected crc 2023-06-10T21:27:18.133 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:18.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:18.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:27:18.240 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:18.339 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:18.406 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:18.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:18.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:18.498 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:18.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:18.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:18.764 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:18.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:18.774 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:27:18.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:18.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:19.083 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:19.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:19.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[186706]: pgmap v805: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 651 op/s; 24936/262245 objects degraded (9.509%); 27238/262245 objects misplaced (10.386%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:27:19.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:19.155 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:19.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:19.275 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[180297]: pgmap v805: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 651 op/s; 24936/262245 objects degraded (9.509%); 27238/262245 objects misplaced (10.386%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:27:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:19.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:19 smithi152 ceph-mon[156819]: pgmap v805: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 651 op/s; 24936/262245 objects degraded (9.509%); 27238/262245 objects misplaced (10.386%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:27:19.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:19.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:19.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:19.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:19.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:19.575 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:19.672 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:19.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:19.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:19.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:19.803 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:20.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:27:20.658 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:27:20.658 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:27:20.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:27:20.658 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20066 9.d 9:b754690b:::benchmark_data_smithi064_247471_object20065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66767480499999, 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999, 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.659 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20066 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:27:20.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983875+0000", 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019127799999999999 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984066+0000", 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2920000000000001e-06 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984075+0000", 2023-06-10T21:27:20.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9660000000000001e-06 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984081+0000", 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8265999999999999e-05 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984159+0000", 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121333 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984281+0000", 2023-06-10T21:27:20.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144637 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984425+0000", 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014625199999999999 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985888+0000", 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024362569999999998 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988324+0000", 2023-06-10T21:27:20.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29050814800000002 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278832+0000", 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5896999999999998e-05 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278878+0000", 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7633999999999999e-05 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278916+0000", 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999 2023-06-10T21:27:20.669 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20069 9.3 9:c33031bf:::benchmark_data_smithi064_247471_object20068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.670 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66720715399998, 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999, 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.671 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20069 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984342+0000", 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012432649999999999 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985585+0000", 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6175e-05 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985602+0000", 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8390000000000001e-06 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985609+0000", 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1567999999999998e-05 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985660+0000", 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013685800000000001 2023-06-10T21:27:20.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985797+0000", 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016983999999999999 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985967+0000", 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026175999999999999 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988584+0000", 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012783530000000001 2023-06-10T21:27:20.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989863+0000", 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28834436000000002 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278207+0000", 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278244+0000", 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8973e-05 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278293+0000", 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20072 9.3 9:ce7b60fb:::benchmark_data_smithi064_247471_object20071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66592677800003, 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998, 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.682 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20072 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985623+0000", 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7206e-05 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985650+0000", 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7629e-05 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985668+0000", 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985674+0000", 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001225133 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986899+0000", 2023-06-10T21:27:20.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012472500000000001 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987024+0000", 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000152232 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987176+0000", 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014119930000000001 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988588+0000", 2023-06-10T21:27:20.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017869019999999999 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990375+0000", 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28840961799999998 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278785+0000", 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9700000000000002e-05 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278834+0000", 2023-06-10T21:27:20.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8235999999999998e-05 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278872+0000", 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.692 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20074 9.3 9:cf5d96ce:::benchmark_data_smithi064_247471_object20073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66587054000001, 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001, 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.693 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20074 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:27:20.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985679+0000", 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9383e-05 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985709+0000", 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985716+0000", 2023-06-10T21:27:20.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.048e-06 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985720+0000", 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002102862 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987822+0000", 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4449000000000004e-05 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987907+0000", 2023-06-10T21:27:20.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134979 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988042+0000", 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064599799999999995 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988688+0000", 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001729993 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990418+0000", 2023-06-10T21:27:20.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28863366400000001 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279052+0000", 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3007999999999998e-05 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279105+0000", 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8399000000000003e-05 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279143+0000", 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001 2023-06-10T21:27:20.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20077 9.d 9:bf31cec4:::benchmark_data_smithi064_247471_object20076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:27:20.704 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66468588999999, 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002, 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20077 2023-06-10T21:27:20.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986863+0000", 2023-06-10T21:27:20.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1553000000000001e-05 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986895+0000", 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5759999999999995e-06 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986905+0000", 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7969999999999997e-06 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986910+0000", 2023-06-10T21:27:20.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1326000000000001e-05 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986982+0000", 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012606299999999999 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987108+0000", 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014393300000000001 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987252+0000", 2023-06-10T21:27:20.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001079055 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988331+0000", 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117989 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988449+0000", 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29228185899999998 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280731+0000", 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4913000000000002e-05 2023-06-10T21:27:20.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280776+0000", 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8541000000000003e-05 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280814+0000", 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20079 9.d 9:b60e4f03:::benchmark_data_smithi064_247471_object20078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:27:20.715 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 380.66255817500002, 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267, 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20079 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988991+0000", 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9812000000000001e-05 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989071+0000", 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1182000000000001e-05 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989082+0000", 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1869999999999996e-06 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989087+0000", 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6656999999999998e-05 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989124+0000", 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013583000000000001 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989260+0000", 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9172999999999995e-05 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989349+0000", 2023-06-10T21:27:20.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00098969899999999992 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990339+0000", 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017819999999999999 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990517+0000", 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29222891400000001 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282746+0000", 2023-06-10T21:27:20.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8776000000000001e-05 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282805+0000", 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4026000000000002e-05 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282849+0000", 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.726 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137764 9.d 9:bf6cd1ea:::benchmark_data_smithi064_247471_object28776:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 258.210096905, 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998, 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.727 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137764 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:27:20.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441454+0000", 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441452+0000", 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7470000000000002e-06 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441458+0000", 2023-06-10T21:27:20.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8309999999999996e-06 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441468+0000", 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4750000000000001e-06 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441473+0000", 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3132e-05 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441516+0000", 2023-06-10T21:27:20.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114258 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441631+0000", 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119329 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.441750+0000", 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041105380000000004 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445860+0000", 2023-06-10T21:27:20.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074270170000000002 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.453288+0000", 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28097103499999998 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734259+0000", 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2166000000000003e-05 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734301+0000", 2023-06-10T21:27:20.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6820000000000003e-05 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734338+0000", 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29288399199999998 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.737 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:197262 9.d 9:bc8c96a1:::benchmark_data_smithi064_247471_object88274:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 163.439879402, 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29079689600000003, 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.738 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 197262 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:27:20.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211671+0000", 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1159999999999999e-06 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211674+0000", 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6749999999999999e-06 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211679+0000", 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1860000000000002e-06 2023-06-10T21:27:20.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.211683+0000", 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033231910000000001 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215007+0000", 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119659 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215126+0000", 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012273399999999999 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.215249+0000", 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26155416500000001 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.476803+0000", 2023-06-10T21:27:20.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8266000000000002e-05 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.476852+0000", 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025525157 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502377+0000", 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1844e-05 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502429+0000", 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9428999999999998e-05 2023-06-10T21:27:20.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:37.502468+0000", 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29079689600000003 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.748 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199854 9.3 9:c5f79f4f:::benchmark_data_smithi064_247471_object90866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.11511508199999, 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362, 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.749 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199854 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:27:20.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536433+0000", 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7489999999999998e-06 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536441+0000", 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5624000000000002e-05 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536457+0000", 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5969999999999997e-06 2023-06-10T21:27:20.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536463+0000", 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107921 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536571+0000", 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011766500000000001 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536689+0000", 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5592000000000003e-05 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536774+0000", 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037939639999999999 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540568+0000", 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012242509 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552811+0000", 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.296973082 2023-06-10T21:27:20.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849784+0000", 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6795999999999997e-05 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849831+0000", 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4078e-05 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849875+0000", 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.759 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199859 9.3 9:ca245c36:::benchmark_data_smithi064_247471_object90871:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.11064324099999, 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001, 2023-06-10T21:27:20.760 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199859 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.761 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.915e-06 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540910+0000", 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9649999999999996e-06 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540916+0000", 2023-06-10T21:27:20.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7190000000000001e-06 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540920+0000", 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9440999999999997e-05 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540960+0000", 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111699 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541072+0000", 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053361700000000001 2023-06-10T21:27:20.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541605+0000", 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011243814 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552849+0000", 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039226670000000003 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556772+0000", 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29326851399999998 2023-06-10T21:27:20.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850040+0000", 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7537999999999999e-05 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850078+0000", 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4517999999999997e-05 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850112+0000", 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001 2023-06-10T21:27:20.770 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199860 9.d 9:bf36e690:::benchmark_data_smithi064_247471_object90872:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.11061855200001, 2023-06-10T21:27:20.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999, 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199860 2023-06-10T21:27:20.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540931+0000", 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8059999999999999e-06 2023-06-10T21:27:20.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540934+0000", 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2140000000000001e-06 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540940+0000", 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7560000000000001e-06 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540944+0000", 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003930256 2023-06-10T21:27:20.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544874+0000", 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122638 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544997+0000", 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012774499999999999 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545125+0000", 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046761019999999997 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549801+0000", 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00087457099999999999 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.550675+0000", 2023-06-10T21:27:20.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29424700599999998 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844922+0000", 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6332000000000002e-05 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844979+0000", 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3044000000000003e-05 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.845022+0000", 2023-06-10T21:27:20.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199861 9.3 9:c4d624c7:::benchmark_data_smithi064_247471_object90873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:27:20.782 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.11045299, 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997, 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199861 2023-06-10T21:27:20.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9880000000000004e-06 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541102+0000", 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6309999999999992e-06 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541110+0000", 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319999999999999e-06 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541115+0000", 2023-06-10T21:27:20.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045389230000000003 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545654+0000", 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010809799999999999 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545762+0000", 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000124232 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545887+0000", 2023-06-10T21:27:20.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069982400000000002 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552885+0000", 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003894041 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556779+0000", 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29336341300000002 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850142+0000", 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3971999999999999e-05 2023-06-10T21:27:20.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850176+0000", 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3324999999999997e-05 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850210+0000", 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.793 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199863 9.3 9:c858996c:::benchmark_data_smithi064_247471_object90875:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.11039281999999, 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986, 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199863 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541157+0000", 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0770000000000001e-06 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541161+0000", 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3150000000000003e-06 2023-06-10T21:27:20.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541168+0000", 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0239999999999997e-06 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541173+0000", 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084016230000000004 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549575+0000", 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011957599999999999 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549695+0000", 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013125600000000001 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549826+0000", 2023-06-10T21:27:20.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070024570000000001 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556828+0000", 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036556430000000001 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560484+0000", 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28975610600000001 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850240+0000", 2023-06-10T21:27:20.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2453999999999997e-05 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850283+0000", 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3065999999999998e-05 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850316+0000", 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199865 9.3 9:cef2937d:::benchmark_data_smithi064_247471_object90877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.101912314, 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124, 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.805 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199865 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.806 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549637+0000", 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549642+0000", 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7441000000000001e-05 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549660+0000", 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4330000000000001e-06 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549666+0000", 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003246578 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552913+0000", 2023-06-10T21:27:20.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000139326 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553052+0000", 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147138 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553199+0000", 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036693580000000002 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556869+0000", 2023-06-10T21:27:20.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003640038 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560509+0000", 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28983566300000002 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850344+0000", 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3887999999999998e-05 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850378+0000", 2023-06-10T21:27:20.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1239999999999999e-05 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850409+0000", 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.815 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199870 9.3 9:cea33e0b:::benchmark_data_smithi064_247471_object90882:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 159.09492870400001, 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003, 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.816 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199870 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:27:20.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556620+0000", 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.57e-06 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556626+0000", 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5829999999999996e-06 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556635+0000", 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1599999999999997e-06 2023-06-10T21:27:20.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556641+0000", 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026981200000000002 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556910+0000", 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113881 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557024+0000", 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121586 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557146+0000", 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033591770000000001 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560505+0000", 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018934659999999999 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.562399+0000", 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28803842299999999 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850437+0000", 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2833000000000002e-05 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850470+0000", 2023-06-10T21:27:20.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1829999999999998e-05 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850502+0000", 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.826 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95682 10.c 10:3eea8292:::benchmark_data_smithi064_312713_object4926:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2021207880000002, 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32186570199999998, 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.827 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95682 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449427+0000", 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3190000000000001e-06 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449437+0000", 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4035e-05 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449451+0000", 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3659999999999997e-06 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449457+0000", 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8087999999999997e-05 2023-06-10T21:27:20.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449515+0000", 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013555500000000001 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449651+0000", 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013275299999999999 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449783+0000", 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098969969999999994 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459680+0000", 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7057000000000001e-05 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459717+0000", 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31138205299999999 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771099+0000", 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148877 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771248+0000", 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6993999999999998e-05 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771295+0000", 2023-06-10T21:27:20.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32186570199999998 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95683 10.1 10:82260f40:::benchmark_data_smithi064_312713_object4927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:27:20.838 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2020747959999998, 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604, 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:20.839 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95683 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449475+0000", 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3629999999999998e-06 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449478+0000", 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4200000000000007e-06 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449487+0000", 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8169999999999998e-06 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449492+0000", 2023-06-10T21:27:20.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034418460000000001 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.452934+0000", 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123182 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453057+0000", 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012961900000000001 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453186+0000", 2023-06-10T21:27:20.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058584609999999997 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459045+0000", 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6512e-05 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459061+0000", 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32578302999999997 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784844+0000", 2023-06-10T21:27:20.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3819000000000001e-05 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784898+0000", 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2079999999999997e-05 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784940+0000", 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.849 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95684 10.c 10:3b0e142b:::benchmark_data_smithi064_312713_object4928:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.1985408469999999, 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.326603953, 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:20.850 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95684 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453008+0000", 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.619e-06 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453016+0000", 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0013000000000001e-05 2023-06-10T21:27:20.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453026+0000", 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6809999999999997e-06 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453031+0000", 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042010499999999999 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453451+0000", 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010725 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453558+0000", 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116557 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:20.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453675+0000", 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060171319999999997 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459692+0000", 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003798283 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.463490+0000", 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31602760499999999 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779518+0000", 2023-06-10T21:27:20.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3182e-05 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779571+0000", 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2423999999999999e-05 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779614+0000", 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.326603953 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95687 10.1 10:884db9c7:::benchmark_data_smithi064_312713_object4931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.1985084029999999, 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002, 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.861 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95687 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:27:20.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453041+0000", 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5729999999999999e-06 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453045+0000", 2023-06-10T21:27:20.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.665e-06 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453051+0000", 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.253e-06 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453054+0000", 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027574850000000001 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455811+0000", 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2564999999999994e-05 2023-06-10T21:27:20.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455884+0000", 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6180999999999994e-05 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:20.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455960+0000", 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031903819999999999 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459151+0000", 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015862529999999999 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.460737+0000", 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32425166599999999 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784988+0000", 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6248999999999997e-05 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785035+0000", 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8260000000000003e-05 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785073+0000", 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95691 10.c 10:35e47876:::benchmark_data_smithi064_312713_object4935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.1970544489999999, 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32524502799999999, 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:27:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95691 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:27:20.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454495+0000", 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3430000000000003e-06 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454500+0000", 2023-06-10T21:27:20.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0764999999999999e-05 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454511+0000", 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8950000000000003e-06 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454517+0000", 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021666849999999998 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456683+0000", 2023-06-10T21:27:20.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6739999999999999e-05 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456780+0000", 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010805 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456888+0000", 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029155219999999998 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459804+0000", 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038126929999999998 2023-06-10T21:27:20.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.463616+0000", 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31604337700000001 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:27:20.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779660+0000", 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5500000000000001e-05 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779705+0000", 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5839e-05 2023-06-10T21:27:20.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779741+0000", 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32524502799999999 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.882 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:27:20.883 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:27:20.883 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:27:21.146 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:21.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:21.253 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:21.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:21 smithi064 ceph-mon[186706]: pgmap v806: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 108 B/s rd, 12 MiB/s wr, 726 op/s; 24115/257292 objects degraded (9.373%); 26609/257292 objects misplaced (10.342%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:27:21.313 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 24115/257292 objects degraded (9.373%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:21.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:21 smithi064 ceph-mon[180297]: pgmap v806: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 108 B/s rd, 12 MiB/s wr, 726 op/s; 24115/257292 objects degraded (9.373%); 26609/257292 objects misplaced (10.342%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:27:21.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 24115/257292 objects degraded (9.373%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:21.316 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:21.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:21 smithi152 ceph-mon[156819]: pgmap v806: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 108 B/s rd, 12 MiB/s wr, 726 op/s; 24115/257292 objects degraded (9.373%); 26609/257292 objects misplaced (10.342%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:27:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 24115/257292 objects degraded (9.373%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:21.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:27:21.622 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:21.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:21.668 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:21.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:21.765 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:21.840 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:21.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:21.854 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:27:21.909 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:21.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:21.979 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:22.013 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:22.031 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:22.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:22.124 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:22.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.166 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.167 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:27:22.285 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/367086385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:22.285 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/367086385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:22.309 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:22.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:22.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:22.415 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/367086385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:22.448 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:22.494 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:27:22.505 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:27:22.531 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:22.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:22.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:22.701 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:22] "GET /metrics HTTP/1.1" 200 224083 "" "Prometheus/2.33.4" 2023-06-10T21:27:23.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:23 smithi152 ceph-mon[156819]: pgmap v807: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 683 op/s; 24115/256647 objects degraded (9.396%); 26609/256647 objects misplaced (10.368%); 8.4 MiB/s, 133 objects/s recovering 2023-06-10T21:27:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:23 smithi152 ceph-mon[156819]: from='client.45926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:23.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:23 smithi064 ceph-mon[186706]: pgmap v807: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 683 op/s; 24115/256647 objects degraded (9.396%); 26609/256647 objects misplaced (10.368%); 8.4 MiB/s, 133 objects/s recovering 2023-06-10T21:27:23.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:23 smithi064 ceph-mon[186706]: from='client.45926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:23.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:23 smithi064 ceph-mon[180297]: pgmap v807: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 102 B/s rd, 10 MiB/s wr, 683 op/s; 24115/256647 objects degraded (9.396%); 26609/256647 objects misplaced (10.368%); 8.4 MiB/s, 133 objects/s recovering 2023-06-10T21:27:23.560 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:23 smithi064 ceph-mon[180297]: from='client.45926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:27:24.583 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:24.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:24.627 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:24.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:24.730 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:24.829 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:24.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:24.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:24.985 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:25.010 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:25.036 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":807,"stamp":"2023-06-10T21:27:21.608414+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5605496742,"num_objects":85549,"num_object_clones":0,"num_object_copies":256647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24115,"num_objects_misplaced":26609,"num_objects_unfound":0,"num_objects_dirty":85549,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":96244,"num_write_kb":5810639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9050,"num_bytes_recovered":592258202,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96178,"ondisk_log_size":96178,"up":215,"acting":218,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16676296,"kb_used_data":14173144,"kb_used_omap":20,"kb_used_meta":2503083,"kb_avail":545794616,"statfs":{"total":575970213888,"available":558893686784,"internally_reserved":0,"allocated":14513299456,"data_stored":14509208318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2563157646},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,0,1,0,1,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-232849384,"num_objects":-3552,"num_object_clones":0,"num_object_copies":-10656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1341,"num_objects_misplaced":-1705,"num_objects_unfound":0,"num_objects_dirty":-3552,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6836,"num_write_kb":105025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1340,"num_bytes_recovered":87818240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003122"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":217,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.176584+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:27:10.176584+0000","last_peered":"2023-06-10T21:27:10.176584+0000","last_clean":"2023-06-10T21:27:10.176584+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:27:10.176584+0000","last_undegraded":"2023-06-10T21:27:10.176584+0000","last_fullsized":"2023-06-10T21:27:10.176584+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.184812+0000","last_change":"2023-06-10T21:26:27.787856+0000","last_active":"2023-06-10T21:27:10.184812+0000","last_peered":"2023-06-10T21:27:10.184812+0000","last_clean":"2023-06-10T21:13:57.474600+0000","last_became_active":"2023-06-10T21:26:27.787856+0000","last_became_peered":"2023-06-10T21:26:27.787856+0000","last_unstale":"2023-06-10T21:27:10.184812+0000","last_undegraded":"2023-06-10T21:27:10.184812+0000","last_fullsized":"2023-06-10T21:26:27.776169+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:29:11.586416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.184863+0000","last_change":"2023-06-10T21:26:27.787641+0000","last_active":"2023-06-10T21:27:10.184863+0000","last_peered":"2023-06-10T21:27:10.184863+0000","last_clean":"2023-06-10T21:19:00.485389+0000","last_became_active":"2023-06-10T21:26:27.787641+0000","last_became_peered":"2023-06-10T21:26:27.787641+0000","last_unstale":"2023-06-10T21:27:10.184863+0000","last_undegraded":"2023-06-10T21:27:10.184863+0000","last_fullsized":"2023-06-10T21:26:27.775244+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:39:35.438222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":71,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.388992+0000","last_change":"2023-06-10T21:26:27.785421+0000","last_active":"2023-06-10T21:27:05.388992+0000","last_peered":"2023-06-10T21:27:05.388992+0000","last_clean":"2023-06-10T21:26:20.391953+0000","last_became_active":"2023-06-10T21:26:27.785421+0000","last_became_peered":"2023-06-10T21:26:27.785421+0000","last_unstale":"2023-06-10T21:27:05.388992+0000","last_undegraded":"2023-06-10T21:27:05.388992+0000","last_fullsized":"2023-06-10T21:26:27.778803+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:54:12.929497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":174,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.385004+0000","last_change":"2023-06-10T21:26:27.788758+0000","last_active":"2023-06-10T21:27:05.385004+0000","last_peered":"2023-06-10T21:27:05.385004+0000","last_clean":"2023-06-10T21:24:36.651687+0000","last_became_active":"2023-06-10T21:26:27.788758+0000","last_became_peered":"2023-06-10T21:26:27.788758+0000","last_unstale":"2023-06-10T21:27:05.385004+0000","last_undegraded":"2023-06-10T21:27:05.385004+0000","last_fullsized":"2023-06-10T21:26:27.776841+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:29:54.271286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.396018+0000","last_change":"2023-06-10T21:26:27.781570+0000","last_active":"2023-06-10T21:27:05.396018+0000","last_peered":"2023-06-10T21:27:05.396018+0000","last_clean":"2023-06-10T21:26:21.435727+0000","last_became_active":"2023-06-10T21:26:27.781570+0000","last_became_peered":"2023-06-10T21:26:27.781570+0000","last_unstale":"2023-06-10T21:27:05.396018+0000","last_undegraded":"2023-06-10T21:27:05.396018+0000","last_fullsized":"2023-06-10T21:26:27.776705+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:53:41.167915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":89,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.177347+0000","last_change":"2023-06-10T21:26:27.781631+0000","last_active":"2023-06-10T21:27:10.177347+0000","last_peered":"2023-06-10T21:27:10.177347+0000","last_clean":"2023-06-10T21:26:20.392386+0000","last_became_active":"2023-06-10T21:26:27.781631+0000","last_became_peered":"2023-06-10T21:26:27.781631+0000","last_unstale":"2023-06-10T21:27:10.177347+0000","last_undegraded":"2023-06-10T21:27:10.177347+0000","last_fullsized":"2023-06-10T21:26:27.777427+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.845813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.185282+0000","last_change":"2023-06-10T21:26:27.792890+0000","last_active":"2023-06-10T21:27:10.185282+0000","last_peered":"2023-06-10T21:27:10.185282+0000","last_clean":"2023-06-10T21:26:05.730925+0000","last_became_active":"2023-06-10T21:26:27.792890+0000","last_became_peered":"2023-06-10T21:26:27.792890+0000","last_unstale":"2023-06-10T21:27:10.185282+0000","last_undegraded":"2023-06-10T21:27:10.185282+0000","last_fullsized":"2023-06-10T21:26:27.777122+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:51:54.472567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":107,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169420+0000","last_change":"2023-06-10T21:26:27.786701+0000","last_active":"2023-06-10T21:27:10.169420+0000","last_peered":"2023-06-10T21:27:10.169420+0000","last_clean":"2023-06-10T21:26:20.562068+0000","last_became_active":"2023-06-10T21:26:27.786701+0000","last_became_peered":"2023-06-10T21:26:27.786701+0000","last_unstale":"2023-06-10T21:27:10.169420+0000","last_undegraded":"2023-06-10T21:27:10.169420+0000","last_fullsized":"2023-06-10T21:26:27.783028+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.172340+0000","last_change":"2023-06-10T21:26:27.786324+0000","last_active":"2023-06-10T21:27:10.172340+0000","last_peered":"2023-06-10T21:27:10.172340+0000","last_clean":"2023-06-10T21:26:05.731138+0000","last_became_active":"2023-06-10T21:26:27.786324+0000","last_became_peered":"2023-06-10T21:26:27.786324+0000","last_unstale":"2023-06-10T21:27:10.172340+0000","last_undegraded":"2023-06-10T21:27:10.172340+0000","last_fullsized":"2023-06-10T21:26:27.781327+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":203,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:10.169601+0000","last_change":"2023-06-10T21:26:27.785616+0000","last_active":"2023-06-10T21:27:10.169601+0000","last_peered":"2023-06-10T21:27:10.169601+0000","last_clean":"2023-06-10T21:26:20.562388+0000","last_became_active":"2023-06-10T21:26:27.785616+0000","last_became_peered":"2023-06-10T21:26:27.785616+0000","last_unstale":"2023-06-10T21:27:10.169601+0000","last_undegraded":"2023-06-10T21:26:27.781808+0000","last_fullsized":"2023-06-10T21:26:27.781691+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":171,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.176478+0000","last_change":"2023-06-10T21:26:15.746830+0000","last_active":"2023-06-10T21:27:10.176478+0000","last_peered":"2023-06-10T21:27:10.176478+0000","last_clean":"2023-06-10T21:27:10.176478+0000","last_became_active":"2023-06-10T21:26:15.746510+0000","last_became_peered":"2023-06-10T21:26:15.746510+0000","last_unstale":"2023-06-10T21:27:10.176478+0000","last_undegraded":"2023-06-10T21:27:10.176478+0000","last_fullsized":"2023-06-10T21:27:10.176478+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:50:22.116927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":97,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169472+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:27:10.169472+0000","last_peered":"2023-06-10T21:27:10.169472+0000","last_clean":"2023-06-10T21:27:10.169472+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:27:10.169472+0000","last_undegraded":"2023-06-10T21:27:10.169472+0000","last_fullsized":"2023-06-10T21:27:10.169472+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":24,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.180585+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:27:10.180585+0000","last_peered":"2023-06-10T21:27:10.180585+0000","last_clean":"2023-06-10T21:27:10.180585+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:27:10.180585+0000","last_undegraded":"2023-06-10T21:27:10.180585+0000","last_fullsized":"2023-06-10T21:27:10.180585+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":226,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.172080+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:27:10.172080+0000","last_peered":"2023-06-10T21:27:10.172080+0000","last_clean":"2023-06-10T21:27:10.172080+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:27:10.172080+0000","last_undegraded":"2023-06-10T21:27:10.172080+0000","last_fullsized":"2023-06-10T21:27:10.172080+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.178054+0000","last_change":"2023-06-10T21:26:27.787457+0000","last_active":"2023-06-10T21:27:10.178054+0000","last_peered":"2023-06-10T21:27:10.178054+0000","last_clean":"2023-06-10T21:13:57.475185+0000","last_became_active":"2023-06-10T21:26:27.787457+0000","last_became_peered":"2023-06-10T21:26:27.787457+0000","last_unstale":"2023-06-10T21:27:10.178054+0000","last_undegraded":"2023-06-10T21:27:10.178054+0000","last_fullsized":"2023-06-10T21:26:27.781392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:06.604962+0000","last_change":"2023-06-10T21:26:27.783019+0000","last_active":"2023-06-10T21:27:06.604962+0000","last_peered":"2023-06-10T21:27:06.604962+0000","last_clean":"2023-06-10T21:26:20.391274+0000","last_became_active":"2023-06-10T21:26:27.783019+0000","last_became_peered":"2023-06-10T21:26:27.783019+0000","last_unstale":"2023-06-10T21:27:06.604962+0000","last_undegraded":"2023-06-10T21:27:06.604962+0000","last_fullsized":"2023-06-10T21:26:27.777376+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:41:33.972128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":135,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169741+0000","last_change":"2023-06-10T21:26:27.786424+0000","last_active":"2023-06-10T21:27:10.169741+0000","last_peered":"2023-06-10T21:27:10.169741+0000","last_clean":"2023-06-10T21:26:20.562490+0000","last_became_active":"2023-06-10T21:26:27.786424+0000","last_became_peered":"2023-06-10T21:26:27.786424+0000","last_unstale":"2023-06-10T21:27:10.169741+0000","last_undegraded":"2023-06-10T21:27:10.169741+0000","last_fullsized":"2023-06-10T21:26:27.781372+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":188,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.172251+0000","last_change":"2023-06-10T21:26:15.747593+0000","last_active":"2023-06-10T21:27:10.172251+0000","last_peered":"2023-06-10T21:27:10.172251+0000","last_clean":"2023-06-10T21:27:10.172251+0000","last_became_active":"2023-06-10T21:26:15.747244+0000","last_became_peered":"2023-06-10T21:26:15.747244+0000","last_unstale":"2023-06-10T21:27:10.172251+0000","last_undegraded":"2023-06-10T21:27:10.172251+0000","last_fullsized":"2023-06-10T21:27:10.172251+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.177697+0000","last_change":"2023-06-10T21:26:27.785039+0000","last_active":"2023-06-10T21:27:10.177697+0000","last_peered":"2023-06-10T21:27:10.177697+0000","last_clean":"2023-06-10T21:26:20.391891+0000","last_became_active":"2023-06-10T21:26:27.785039+0000","last_became_peered":"2023-06-10T21:26:27.785039+0000","last_unstale":"2023-06-10T21:27:10.177697+0000","last_undegraded":"2023-06-10T21:27:10.177697+0000","last_fullsized":"2023-06-10T21:26:27.777488+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:49:35.504339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":207,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:05.388664+0000","last_change":"2023-06-10T21:26:27.785625+0000","last_active":"2023-06-10T21:27:05.388664+0000","last_peered":"2023-06-10T21:27:05.388664+0000","last_clean":"2023-06-10T21:26:21.406183+0000","last_became_active":"2023-06-10T21:26:27.785625+0000","last_became_peered":"2023-06-10T21:26:27.785625+0000","last_unstale":"2023-06-10T21:27:05.388664+0000","last_undegraded":"2023-06-10T21:26:27.779630+0000","last_fullsized":"2023-06-10T21:26:27.779530+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:23:23.310377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169949+0000","last_change":"2023-06-10T21:26:27.790352+0000","last_active":"2023-06-10T21:27:10.169949+0000","last_peered":"2023-06-10T21:27:10.169949+0000","last_clean":"2023-06-10T21:13:57.474737+0000","last_became_active":"2023-06-10T21:26:27.790352+0000","last_became_peered":"2023-06-10T21:26:27.790352+0000","last_unstale":"2023-06-10T21:27:10.169949+0000","last_undegraded":"2023-06-10T21:27:10.169949+0000","last_fullsized":"2023-06-10T21:26:27.782802+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"226'5896","reported_seq":7597,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:16.583473+0000","last_change":"2023-06-10T21:25:57.635943+0000","last_active":"2023-06-10T21:27:16.583473+0000","last_peered":"2023-06-10T21:27:16.583473+0000","last_clean":"2023-06-10T21:27:16.583473+0000","last_became_active":"2023-06-10T21:25:57.632429+0000","last_became_peered":"2023-06-10T21:25:57.632429+0000","last_unstale":"2023-06-10T21:27:16.583473+0000","last_undegraded":"2023-06-10T21:27:16.583473+0000","last_fullsized":"2023-06-10T21:27:16.583473+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5896,"log_dups_size":0,"ondisk_log_size":5896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:41:00.568268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357957632,"num_objects":5462,"num_object_clones":0,"num_object_copies":16386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5896,"num_write_kb":363456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":822,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":221,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.170193+0000","last_change":"2023-06-10T21:24:45.162493+0000","last_active":"2023-06-10T21:27:10.170193+0000","last_peered":"2023-06-10T21:27:10.170193+0000","last_clean":"2023-06-10T21:27:10.170193+0000","last_became_active":"2023-06-10T21:24:45.139001+0000","last_became_peered":"2023-06-10T21:24:45.139001+0000","last_unstale":"2023-06-10T21:27:10.170193+0000","last_undegraded":"2023-06-10T21:27:10.170193+0000","last_fullsized":"2023-06-10T21:27:10.170193+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169132+0000","last_change":"2023-06-10T21:26:27.789518+0000","last_active":"2023-06-10T21:27:10.169132+0000","last_peered":"2023-06-10T21:27:10.169132+0000","last_clean":"2023-06-10T21:26:20.562380+0000","last_became_active":"2023-06-10T21:26:27.789518+0000","last_became_peered":"2023-06-10T21:26:27.789518+0000","last_unstale":"2023-06-10T21:27:10.169132+0000","last_undegraded":"2023-06-10T21:27:10.169132+0000","last_fullsized":"2023-06-10T21:26:27.781860+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"227'6065","reported_seq":8451,"reported_epoch":227,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:27:18.771852+0000","last_change":"2023-06-10T21:26:28.972723+0000","last_active":"2023-06-10T21:27:18.771852+0000","last_peered":"2023-06-10T21:27:18.771852+0000","last_clean":"2023-06-10T21:26:15.549377+0000","last_became_active":"2023-06-10T21:26:28.952008+0000","last_became_peered":"2023-06-10T21:26:28.952008+0000","last_unstale":"2023-06-10T21:27:18.771852+0000","last_undegraded":"2023-06-10T21:26:28.877253+0000","last_fullsized":"2023-06-10T21:27:18.771852+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:07:08.784247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350945280,"num_objects":5355,"num_object_clones":0,"num_object_copies":16065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3040,"num_objects_misplaced":5102,"num_objects_unfound":0,"num_objects_dirty":5355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6065,"num_write_kb":365440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1157,"num_bytes_recovered":75825152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,4","objects":253},{"shards":"2,4,6","objects":2315},{"shards":"2,6","objects":2787}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":196,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169547+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:27:10.169547+0000","last_peered":"2023-06-10T21:27:10.169547+0000","last_clean":"2023-06-10T21:27:10.169547+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:27:10.169547+0000","last_undegraded":"2023-06-10T21:27:10.169547+0000","last_fullsized":"2023-06-10T21:27:10.169547+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":205,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165558+0000","last_change":"2023-06-10T21:26:27.790530+0000","last_active":"2023-06-10T21:27:10.165558+0000","last_peered":"2023-06-10T21:27:10.165558+0000","last_clean":"2023-06-10T21:19:00.489965+0000","last_became_active":"2023-06-10T21:26:27.790530+0000","last_became_peered":"2023-06-10T21:26:27.790530+0000","last_unstale":"2023-06-10T21:27:10.165558+0000","last_undegraded":"2023-06-10T21:27:10.165558+0000","last_fullsized":"2023-06-10T21:26:27.780661+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"226'5987","reported_seq":6033,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:16.612363+0000","last_change":"2023-06-10T21:26:27.785031+0000","last_active":"2023-06-10T21:27:16.612363+0000","last_peered":"2023-06-10T21:27:16.612363+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:26:27.785031+0000","last_became_peered":"2023-06-10T21:26:27.785031+0000","last_unstale":"2023-06-10T21:27:16.612363+0000","last_undegraded":"2023-06-10T21:26:27.778233+0000","last_fullsized":"2023-06-10T21:26:27.778123+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5987,"log_dups_size":0,"ondisk_log_size":5987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:11:03.215088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356188160,"num_objects":5435,"num_object_clones":0,"num_object_copies":16305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5435,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5987,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":5435}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":212,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:10.169439+0000","last_change":"2023-06-10T21:26:27.785962+0000","last_active":"2023-06-10T21:27:10.169439+0000","last_peered":"2023-06-10T21:27:10.169439+0000","last_clean":"2023-06-10T21:26:20.562273+0000","last_became_active":"2023-06-10T21:26:27.785962+0000","last_became_peered":"2023-06-10T21:26:27.785962+0000","last_unstale":"2023-06-10T21:27:10.169439+0000","last_undegraded":"2023-06-10T21:26:27.782163+0000","last_fullsized":"2023-06-10T21:26:27.782058+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169030+0000","last_change":"2023-06-10T21:26:15.747743+0000","last_active":"2023-06-10T21:27:10.169030+0000","last_peered":"2023-06-10T21:27:10.169030+0000","last_clean":"2023-06-10T21:27:10.169030+0000","last_became_active":"2023-06-10T21:26:15.747368+0000","last_became_peered":"2023-06-10T21:26:15.747368+0000","last_unstale":"2023-06-10T21:27:10.169030+0000","last_undegraded":"2023-06-10T21:27:10.169030+0000","last_fullsized":"2023-06-10T21:27:10.169030+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"227'5969","reported_seq":6015,"reported_epoch":227,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:18.769589+0000","last_change":"2023-06-10T21:26:27.813144+0000","last_active":"2023-06-10T21:27:18.769589+0000","last_peered":"2023-06-10T21:27:18.769589+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:26:27.813144+0000","last_became_peered":"2023-06-10T21:26:27.813144+0000","last_unstale":"2023-06-10T21:27:18.769589+0000","last_undegraded":"2023-06-10T21:26:27.783475+0000","last_fullsized":"2023-06-10T21:26:27.783342+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:07:57.630232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346882048,"num_objects":5293,"num_object_clones":0,"num_object_copies":15879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5293,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5969,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":5293}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":123,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.389133+0000","last_change":"2023-06-10T21:26:27.780557+0000","last_active":"2023-06-10T21:27:05.389133+0000","last_peered":"2023-06-10T21:27:05.389133+0000","last_clean":"2023-06-10T21:26:05.731822+0000","last_became_active":"2023-06-10T21:26:27.780557+0000","last_became_peered":"2023-06-10T21:26:27.780557+0000","last_unstale":"2023-06-10T21:27:05.389133+0000","last_undegraded":"2023-06-10T21:27:05.389133+0000","last_fullsized":"2023-06-10T21:26:27.777733+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:54:27.522102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":64,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.390697+0000","last_change":"2023-06-10T21:24:45.144751+0000","last_active":"2023-06-10T21:27:05.390697+0000","last_peered":"2023-06-10T21:27:05.390697+0000","last_clean":"2023-06-10T21:27:05.390697+0000","last_became_active":"2023-06-10T21:24:45.143921+0000","last_became_peered":"2023-06-10T21:24:45.143921+0000","last_unstale":"2023-06-10T21:27:05.390697+0000","last_undegraded":"2023-06-10T21:27:05.390697+0000","last_fullsized":"2023-06-10T21:27:05.390697+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:21:57.739161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"226'6033","reported_seq":6081,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:16.504865+0000","last_change":"2023-06-10T21:26:01.638496+0000","last_active":"2023-06-10T21:27:16.504865+0000","last_peered":"2023-06-10T21:27:16.504865+0000","last_clean":"2023-06-10T21:27:16.504865+0000","last_became_active":"2023-06-10T21:26:01.633949+0000","last_became_peered":"2023-06-10T21:26:01.633949+0000","last_unstale":"2023-06-10T21:27:16.504865+0000","last_undegraded":"2023-06-10T21:27:16.504865+0000","last_fullsized":"2023-06-10T21:27:16.504865+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6033,"log_dups_size":0,"ondisk_log_size":6033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:09:49.054007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359596032,"num_objects":5487,"num_object_clones":0,"num_object_copies":16461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6033,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":146,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:05.390588+0000","last_change":"2023-06-10T21:26:27.785958+0000","last_active":"2023-06-10T21:27:05.390588+0000","last_peered":"2023-06-10T21:27:05.390588+0000","last_clean":"2023-06-10T21:24:36.651972+0000","last_became_active":"2023-06-10T21:26:27.785958+0000","last_became_peered":"2023-06-10T21:26:27.785958+0000","last_unstale":"2023-06-10T21:27:05.390588+0000","last_undegraded":"2023-06-10T21:26:27.780001+0000","last_fullsized":"2023-06-10T21:26:27.779877+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:20:07.512432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":190,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165866+0000","last_change":"2023-06-10T21:26:27.790578+0000","last_active":"2023-06-10T21:27:10.165866+0000","last_peered":"2023-06-10T21:27:10.165866+0000","last_clean":"2023-06-10T21:26:05.731741+0000","last_became_active":"2023-06-10T21:26:27.790578+0000","last_became_peered":"2023-06-10T21:26:27.790578+0000","last_unstale":"2023-06-10T21:27:10.165866+0000","last_undegraded":"2023-06-10T21:27:10.165866+0000","last_fullsized":"2023-06-10T21:26:27.779863+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"227'5892","reported_seq":7611,"reported_epoch":227,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:18.755699+0000","last_change":"2023-06-10T21:26:27.820034+0000","last_active":"2023-06-10T21:27:18.755699+0000","last_peered":"2023-06-10T21:27:18.755699+0000","last_clean":"2023-06-10T21:26:26.761025+0000","last_became_active":"2023-06-10T21:26:27.820034+0000","last_became_peered":"2023-06-10T21:26:27.820034+0000","last_unstale":"2023-06-10T21:27:18.755699+0000","last_undegraded":"2023-06-10T21:26:27.817850+0000","last_fullsized":"2023-06-10T21:26:27.812347+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:28:47.168875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338427904,"num_objects":5164,"num_object_clones":0,"num_object_copies":15492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5164,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54198272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":5164}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165845+0000","last_change":"2023-06-10T21:26:27.785142+0000","last_active":"2023-06-10T21:27:10.165845+0000","last_peered":"2023-06-10T21:27:10.165845+0000","last_clean":"2023-06-10T21:13:57.477830+0000","last_became_active":"2023-06-10T21:26:27.785142+0000","last_became_peered":"2023-06-10T21:26:27.785142+0000","last_unstale":"2023-06-10T21:27:10.165845+0000","last_undegraded":"2023-06-10T21:27:10.165845+0000","last_fullsized":"2023-06-10T21:26:27.782073+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":215,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.181400+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:27:10.181400+0000","last_peered":"2023-06-10T21:27:10.181400+0000","last_clean":"2023-06-10T21:27:10.181400+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:27:10.181400+0000","last_undegraded":"2023-06-10T21:27:10.181400+0000","last_fullsized":"2023-06-10T21:27:10.181400+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"227'6193","reported_seq":8670,"reported_epoch":227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:27:21.030053+0000","last_change":"2023-06-10T21:26:28.835778+0000","last_active":"2023-06-10T21:27:21.030053+0000","last_peered":"2023-06-10T21:27:21.030053+0000","last_clean":"2023-06-10T21:26:15.544387+0000","last_became_active":"2023-06-10T21:26:16.777640+0000","last_became_peered":"2023-06-10T21:26:16.777640+0000","last_unstale":"2023-06-10T21:27:21.030053+0000","last_undegraded":"2023-06-10T21:26:28.832228+0000","last_fullsized":"2023-06-10T21:26:16.639321+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6193,"log_dups_size":0,"ondisk_log_size":6193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:32:08.099604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345964544,"num_objects":5279,"num_object_clones":0,"num_object_copies":15837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6193,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79626240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":3361},{"shards":"1,4,6","objects":1918}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.176810+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:27:10.176810+0000","last_peered":"2023-06-10T21:27:10.176810+0000","last_clean":"2023-06-10T21:27:10.176810+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:27:10.176810+0000","last_undegraded":"2023-06-10T21:27:10.176810+0000","last_fullsized":"2023-06-10T21:27:10.176810+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":190,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165887+0000","last_change":"2023-06-10T21:26:27.784827+0000","last_active":"2023-06-10T21:27:10.165887+0000","last_peered":"2023-06-10T21:27:10.165887+0000","last_clean":"2023-06-10T21:13:57.670486+0000","last_became_active":"2023-06-10T21:26:27.784827+0000","last_became_peered":"2023-06-10T21:26:27.784827+0000","last_unstale":"2023-06-10T21:27:10.165887+0000","last_undegraded":"2023-06-10T21:27:10.165887+0000","last_fullsized":"2023-06-10T21:26:27.780178+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"227'6090","reported_seq":6139,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-06-10T21:27:19.482723+0000","last_change":"2023-06-10T21:26:01.630409+0000","last_active":"2023-06-10T21:27:19.482723+0000","last_peered":"2023-06-10T21:27:19.482723+0000","last_clean":"2023-06-10T21:27:19.482723+0000","last_became_active":"2023-06-10T21:26:01.627789+0000","last_became_peered":"2023-06-10T21:26:01.627789+0000","last_unstale":"2023-06-10T21:27:19.482723+0000","last_undegraded":"2023-06-10T21:27:19.482723+0000","last_fullsized":"2023-06-10T21:27:19.482723+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6090,"log_dups_size":0,"ondisk_log_size":6090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343932928,"num_objects":5248,"num_object_clones":0,"num_object_copies":15744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6090,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169283+0000","last_change":"2023-06-10T21:26:27.788939+0000","last_active":"2023-06-10T21:27:10.169283+0000","last_peered":"2023-06-10T21:27:10.169283+0000","last_clean":"2023-06-10T21:13:57.474197+0000","last_became_active":"2023-06-10T21:26:27.788939+0000","last_became_peered":"2023-06-10T21:26:27.788939+0000","last_unstale":"2023-06-10T21:27:10.169283+0000","last_undegraded":"2023-06-10T21:27:10.169283+0000","last_fullsized":"2023-06-10T21:26:27.781734+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.186542+0000","last_change":"2023-06-10T21:26:27.784584+0000","last_active":"2023-06-10T21:27:10.186542+0000","last_peered":"2023-06-10T21:27:10.186542+0000","last_clean":"2023-06-10T21:19:10.700014+0000","last_became_active":"2023-06-10T21:26:27.784584+0000","last_became_peered":"2023-06-10T21:26:27.784584+0000","last_unstale":"2023-06-10T21:27:10.186542+0000","last_undegraded":"2023-06-10T21:27:10.186542+0000","last_fullsized":"2023-06-10T21:26:27.776138+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:41:43.266239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"227'6092","reported_seq":6133,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-06-10T21:27:19.361518+0000","last_change":"2023-06-10T21:25:28.186909+0000","last_active":"2023-06-10T21:27:19.361518+0000","last_peered":"2023-06-10T21:27:19.361518+0000","last_clean":"2023-06-10T21:27:19.361518+0000","last_became_active":"2023-06-10T21:25:28.186604+0000","last_became_peered":"2023-06-10T21:25:28.186604+0000","last_unstale":"2023-06-10T21:27:19.361518+0000","last_undegraded":"2023-06-10T21:27:19.361518+0000","last_fullsized":"2023-06-10T21:27:19.361518+0000","mapping_epoch":194,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349306880,"num_objects":5330,"num_object_clones":0,"num_object_copies":15990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6092,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":65,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.389410+0000","last_change":"2023-06-10T21:24:45.143914+0000","last_active":"2023-06-10T21:27:05.389410+0000","last_peered":"2023-06-10T21:27:05.389410+0000","last_clean":"2023-06-10T21:27:05.389410+0000","last_became_active":"2023-06-10T21:24:45.143600+0000","last_became_peered":"2023-06-10T21:24:45.143600+0000","last_unstale":"2023-06-10T21:27:05.389410+0000","last_undegraded":"2023-06-10T21:27:05.389410+0000","last_fullsized":"2023-06-10T21:27:05.389410+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:50.288121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.181638+0000","last_change":"2023-06-10T21:26:27.820466+0000","last_active":"2023-06-10T21:27:10.181638+0000","last_peered":"2023-06-10T21:27:10.181638+0000","last_clean":"2023-06-10T21:26:05.731923+0000","last_became_active":"2023-06-10T21:26:27.820466+0000","last_became_peered":"2023-06-10T21:26:27.820466+0000","last_unstale":"2023-06-10T21:27:10.181638+0000","last_undegraded":"2023-06-10T21:27:10.181638+0000","last_fullsized":"2023-06-10T21:26:27.812254+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:20:50.370098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.393698+0000","last_change":"2023-06-10T21:26:01.634082+0000","last_active":"2023-06-10T21:27:05.393698+0000","last_peered":"2023-06-10T21:27:05.393698+0000","last_clean":"2023-06-10T21:27:05.393698+0000","last_became_active":"2023-06-10T21:26:01.633769+0000","last_became_peered":"2023-06-10T21:26:01.633769+0000","last_unstale":"2023-06-10T21:27:05.393698+0000","last_undegraded":"2023-06-10T21:27:05.393698+0000","last_fullsized":"2023-06-10T21:27:05.393698+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:42:01.535005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"226'5944","reported_seq":8212,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:27:16.522820+0000","last_change":"2023-06-10T21:26:32.555200+0000","last_active":"2023-06-10T21:27:16.522820+0000","last_peered":"2023-06-10T21:27:16.522820+0000","last_clean":"2023-06-10T21:27:16.522820+0000","last_became_active":"2023-06-10T21:26:28.811989+0000","last_became_peered":"2023-06-10T21:26:28.811989+0000","last_unstale":"2023-06-10T21:27:16.522820+0000","last_undegraded":"2023-06-10T21:27:16.522820+0000","last_fullsized":"2023-06-10T21:27:16.522820+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:41:51.588200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353763328,"num_objects":5398,"num_object_clones":0,"num_object_copies":16194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5398,"num_objects_unfound":0,"num_objects_dirty":5398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5944,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1096,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":5398}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":101,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.393614+0000","last_change":"2023-06-10T21:26:27.780884+0000","last_active":"2023-06-10T21:27:05.393614+0000","last_peered":"2023-06-10T21:27:05.393614+0000","last_clean":"2023-06-10T21:26:21.407543+0000","last_became_active":"2023-06-10T21:26:27.780884+0000","last_became_peered":"2023-06-10T21:26:27.780884+0000","last_unstale":"2023-06-10T21:27:05.393614+0000","last_undegraded":"2023-06-10T21:27:05.393614+0000","last_fullsized":"2023-06-10T21:26:27.776394+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:28:19.989690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388789+0000","last_change":"2023-06-10T21:26:15.751423+0000","last_active":"2023-06-10T21:27:05.388789+0000","last_peered":"2023-06-10T21:27:05.388789+0000","last_clean":"2023-06-10T21:27:05.388789+0000","last_became_active":"2023-06-10T21:26:15.751069+0000","last_became_peered":"2023-06-10T21:26:15.751069+0000","last_unstale":"2023-06-10T21:27:05.388789+0000","last_undegraded":"2023-06-10T21:27:05.388789+0000","last_fullsized":"2023-06-10T21:27:05.388789+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:47.921274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":47,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.180859+0000","last_change":"2023-06-10T21:26:27.812850+0000","last_active":"2023-06-10T21:27:10.180859+0000","last_peered":"2023-06-10T21:27:10.180859+0000","last_clean":"2023-06-10T21:26:20.308366+0000","last_became_active":"2023-06-10T21:26:27.812850+0000","last_became_peered":"2023-06-10T21:26:27.812850+0000","last_unstale":"2023-06-10T21:27:10.180859+0000","last_undegraded":"2023-06-10T21:27:10.180859+0000","last_fullsized":"2023-06-10T21:26:27.783047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:47:49.320329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"227'5964","reported_seq":7713,"reported_epoch":227,"state":"active+clean","last_fresh":"2023-06-10T21:27:18.755362+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:27:18.755362+0000","last_peered":"2023-06-10T21:27:18.755362+0000","last_clean":"2023-06-10T21:27:18.755362+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:27:18.755362+0000","last_undegraded":"2023-06-10T21:27:18.755362+0000","last_fullsized":"2023-06-10T21:27:18.755362+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345112576,"num_objects":5266,"num_object_clones":0,"num_object_copies":15798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5964,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":47,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.176895+0000","last_change":"2023-06-10T21:26:27.818621+0000","last_active":"2023-06-10T21:27:10.176895+0000","last_peered":"2023-06-10T21:27:10.176895+0000","last_clean":"2023-06-10T21:26:20.307980+0000","last_became_active":"2023-06-10T21:26:27.818621+0000","last_became_peered":"2023-06-10T21:26:27.818621+0000","last_unstale":"2023-06-10T21:27:10.176895+0000","last_undegraded":"2023-06-10T21:27:10.176895+0000","last_fullsized":"2023-06-10T21:26:27.812688+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:54:17.367733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":50,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.177226+0000","last_change":"2023-06-10T21:26:15.754532+0000","last_active":"2023-06-10T21:27:10.177226+0000","last_peered":"2023-06-10T21:27:10.177226+0000","last_clean":"2023-06-10T21:27:10.177226+0000","last_became_active":"2023-06-10T21:26:15.754198+0000","last_became_peered":"2023-06-10T21:26:15.754198+0000","last_unstale":"2023-06-10T21:27:10.177226+0000","last_undegraded":"2023-06-10T21:27:10.177226+0000","last_fullsized":"2023-06-10T21:27:10.177226+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:20:05.494985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":129,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388855+0000","last_change":"2023-06-10T21:25:45.632992+0000","last_active":"2023-06-10T21:27:05.388855+0000","last_peered":"2023-06-10T21:27:05.388855+0000","last_clean":"2023-06-10T21:27:05.388855+0000","last_became_active":"2023-06-10T21:25:45.632580+0000","last_became_peered":"2023-06-10T21:25:45.632580+0000","last_unstale":"2023-06-10T21:27:05.388855+0000","last_undegraded":"2023-06-10T21:27:05.388855+0000","last_fullsized":"2023-06-10T21:27:05.388855+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:08:44.568827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"226'5928","reported_seq":10883,"reported_epoch":226,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-10T21:27:16.623297+0000","last_change":"2023-06-10T21:26:30.373499+0000","last_active":"2023-06-10T21:27:16.623297+0000","last_peered":"2023-06-10T21:27:16.623297+0000","last_clean":"2023-06-10T21:26:15.547314+0000","last_became_active":"2023-06-10T21:26:21.397611+0000","last_became_peered":"2023-06-10T21:26:21.397611+0000","last_unstale":"2023-06-10T21:27:16.623297+0000","last_undegraded":"2023-06-10T21:27:16.623297+0000","last_fullsized":"2023-06-10T21:26:16.638153+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:49:10.119041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353632256,"num_objects":5396,"num_object_clones":0,"num_object_copies":16188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":695,"num_objects_unfound":0,"num_objects_dirty":5396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5928,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2455,"num_bytes_recovered":160890880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1","objects":4701},{"shards":"0,1,6","objects":695}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":220,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.181058+0000","last_change":"2023-06-10T21:26:27.819291+0000","last_active":"2023-06-10T21:27:10.181058+0000","last_peered":"2023-06-10T21:27:10.181058+0000","last_clean":"2023-06-10T21:26:20.308048+0000","last_became_active":"2023-06-10T21:26:27.819291+0000","last_became_peered":"2023-06-10T21:26:27.819291+0000","last_unstale":"2023-06-10T21:27:10.181058+0000","last_undegraded":"2023-06-10T21:27:10.181058+0000","last_fullsized":"2023-06-10T21:26:27.818075+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:00:43.956182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":499,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169036+0000","last_change":"2023-06-10T21:26:18.680332+0000","last_active":"2023-06-10T21:27:10.169036+0000","last_peered":"2023-06-10T21:27:10.169036+0000","last_clean":"2023-06-10T21:27:10.169036+0000","last_became_active":"2023-06-10T21:26:18.680066+0000","last_became_peered":"2023-06-10T21:26:18.680066+0000","last_unstale":"2023-06-10T21:27:10.169036+0000","last_undegraded":"2023-06-10T21:27:10.169036+0000","last_fullsized":"2023-06-10T21:27:10.169036+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":180,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165100+0000","last_change":"2023-06-10T21:26:27.785005+0000","last_active":"2023-06-10T21:27:10.165100+0000","last_peered":"2023-06-10T21:27:10.165100+0000","last_clean":"2023-06-10T21:26:05.731398+0000","last_became_active":"2023-06-10T21:26:27.785005+0000","last_became_peered":"2023-06-10T21:26:27.785005+0000","last_unstale":"2023-06-10T21:27:10.165100+0000","last_undegraded":"2023-06-10T21:27:10.165100+0000","last_fullsized":"2023-06-10T21:26:27.779673+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":26,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169674+0000","last_change":"2023-06-10T21:26:15.747662+0000","last_active":"2023-06-10T21:27:10.169674+0000","last_peered":"2023-06-10T21:27:10.169674+0000","last_clean":"2023-06-10T21:27:10.169674+0000","last_became_active":"2023-06-10T21:26:15.747331+0000","last_became_peered":"2023-06-10T21:26:15.747331+0000","last_unstale":"2023-06-10T21:27:10.169674+0000","last_undegraded":"2023-06-10T21:27:10.169674+0000","last_fullsized":"2023-06-10T21:27:10.169674+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"226'6050","reported_seq":6097,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:16.619053+0000","last_change":"2023-06-10T21:25:30.191474+0000","last_active":"2023-06-10T21:27:16.619053+0000","last_peered":"2023-06-10T21:27:16.619053+0000","last_clean":"2023-06-10T21:27:16.619053+0000","last_became_active":"2023-06-10T21:25:30.191133+0000","last_became_peered":"2023-06-10T21:25:30.191133+0000","last_unstale":"2023-06-10T21:27:16.619053+0000","last_undegraded":"2023-06-10T21:27:16.619053+0000","last_fullsized":"2023-06-10T21:27:16.619053+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:09:11.858115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357433344,"num_objects":5454,"num_object_clones":0,"num_object_copies":16362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6050,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":231,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.168937+0000","last_change":"2023-06-10T21:24:45.142686+0000","last_active":"2023-06-10T21:27:10.168937+0000","last_peered":"2023-06-10T21:27:10.168937+0000","last_clean":"2023-06-10T21:27:10.168937+0000","last_became_active":"2023-06-10T21:24:45.142396+0000","last_became_peered":"2023-06-10T21:24:45.142396+0000","last_unstale":"2023-06-10T21:27:10.168937+0000","last_undegraded":"2023-06-10T21:27:10.168937+0000","last_fullsized":"2023-06-10T21:27:10.168937+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.388812+0000","last_change":"2023-06-10T21:26:27.785270+0000","last_active":"2023-06-10T21:27:05.388812+0000","last_peered":"2023-06-10T21:27:05.388812+0000","last_clean":"2023-06-10T21:26:20.391145+0000","last_became_active":"2023-06-10T21:26:27.785270+0000","last_became_peered":"2023-06-10T21:26:27.785270+0000","last_unstale":"2023-06-10T21:27:05.388812+0000","last_undegraded":"2023-06-10T21:27:05.388812+0000","last_fullsized":"2023-06-10T21:26:27.778144+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:13.225551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388687+0000","last_change":"2023-06-10T21:25:30.194023+0000","last_active":"2023-06-10T21:27:05.388687+0000","last_peered":"2023-06-10T21:27:05.388687+0000","last_clean":"2023-06-10T21:27:05.388687+0000","last_became_active":"2023-06-10T21:25:30.193693+0000","last_became_peered":"2023-06-10T21:25:30.193693+0000","last_unstale":"2023-06-10T21:27:05.388687+0000","last_undegraded":"2023-06-10T21:27:05.388687+0000","last_fullsized":"2023-06-10T21:27:05.388687+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:46:39.703458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"226'6000","reported_seq":6560,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:27:16.612181+0000","last_change":"2023-06-10T21:26:30.366646+0000","last_active":"2023-06-10T21:27:16.612181+0000","last_peered":"2023-06-10T21:27:16.612181+0000","last_clean":"2023-06-10T21:27:16.612181+0000","last_became_active":"2023-06-10T21:26:28.821113+0000","last_became_peered":"2023-06-10T21:26:28.821113+0000","last_unstale":"2023-06-10T21:27:16.612181+0000","last_undegraded":"2023-06-10T21:27:16.612181+0000","last_fullsized":"2023-06-10T21:27:16.612181+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6000,"log_dups_size":0,"ondisk_log_size":6000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:17:52.662271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356974616,"num_objects":5448,"num_object_clones":0,"num_object_copies":16344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5448,"num_objects_unfound":0,"num_objects_dirty":5448,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6000,"num_write_kb":366273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":251,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":5448}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":216,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169547+0000","last_change":"2023-06-10T21:24:45.139255+0000","last_active":"2023-06-10T21:27:10.169547+0000","last_peered":"2023-06-10T21:27:10.169547+0000","last_clean":"2023-06-10T21:27:10.169547+0000","last_became_active":"2023-06-10T21:24:45.138895+0000","last_became_peered":"2023-06-10T21:24:45.138895+0000","last_unstale":"2023-06-10T21:27:10.169547+0000","last_undegraded":"2023-06-10T21:27:10.169547+0000","last_fullsized":"2023-06-10T21:27:10.169547+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":50,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.180778+0000","last_change":"2023-06-10T21:25:45.624738+0000","last_active":"2023-06-10T21:27:10.180778+0000","last_peered":"2023-06-10T21:27:10.180778+0000","last_clean":"2023-06-10T21:27:10.180778+0000","last_became_active":"2023-06-10T21:25:45.624241+0000","last_became_peered":"2023-06-10T21:25:45.624241+0000","last_unstale":"2023-06-10T21:27:10.180778+0000","last_undegraded":"2023-06-10T21:27:10.180778+0000","last_fullsized":"2023-06-10T21:27:10.180778+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:16:55.688963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388875+0000","last_change":"2023-06-10T21:25:45.632825+0000","last_active":"2023-06-10T21:27:05.388875+0000","last_peered":"2023-06-10T21:27:05.388875+0000","last_clean":"2023-06-10T21:27:05.388875+0000","last_became_active":"2023-06-10T21:25:45.632493+0000","last_became_peered":"2023-06-10T21:25:45.632493+0000","last_unstale":"2023-06-10T21:27:05.388875+0000","last_undegraded":"2023-06-10T21:27:05.388875+0000","last_fullsized":"2023-06-10T21:27:05.388875+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:07.022926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"227'5989","reported_seq":6068,"reported_epoch":227,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:27:18.739236+0000","last_change":"2023-06-10T21:26:28.810386+0000","last_active":"2023-06-10T21:27:18.739236+0000","last_peered":"2023-06-10T21:27:18.739236+0000","last_clean":"2023-06-10T21:26:15.570081+0000","last_became_active":"2023-06-10T21:26:28.807447+0000","last_became_peered":"2023-06-10T21:26:28.807447+0000","last_unstale":"2023-06-10T21:27:18.739236+0000","last_undegraded":"2023-06-10T21:26:28.793793+0000","last_fullsized":"2023-06-10T21:27:18.739236+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5989,"log_dups_size":0,"ondisk_log_size":5989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:20.877709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347799552,"num_objects":5307,"num_object_clones":0,"num_object_copies":15921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":264,"num_objects_misplaced":5043,"num_objects_unfound":0,"num_objects_dirty":5307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5989,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":264},{"shards":"2,4,6","objects":5043}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":36,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:10.169823+0000","last_change":"2023-06-10T21:26:27.787375+0000","last_active":"2023-06-10T21:27:10.169823+0000","last_peered":"2023-06-10T21:27:10.169823+0000","last_clean":"2023-06-10T20:59:27.427422+0000","last_became_active":"2023-06-10T21:26:27.787375+0000","last_became_peered":"2023-06-10T21:26:27.787375+0000","last_unstale":"2023-06-10T21:27:10.169823+0000","last_undegraded":"2023-06-10T21:26:27.783256+0000","last_fullsized":"2023-06-10T21:26:27.783170+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":133,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.186336+0000","last_change":"2023-06-10T21:26:27.788119+0000","last_active":"2023-06-10T21:27:10.186336+0000","last_peered":"2023-06-10T21:27:10.186336+0000","last_clean":"2023-06-10T20:58:33.354602+0000","last_became_active":"2023-06-10T21:26:27.788119+0000","last_became_peered":"2023-06-10T21:26:27.788119+0000","last_unstale":"2023-06-10T21:27:10.186336+0000","last_undegraded":"2023-06-10T21:27:10.186336+0000","last_fullsized":"2023-06-10T21:26:27.776800+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:41.776787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":140,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.169327+0000","last_change":"2023-06-10T21:26:27.787854+0000","last_active":"2023-06-10T21:27:10.169327+0000","last_peered":"2023-06-10T21:27:10.169327+0000","last_clean":"2023-06-10T21:24:36.651653+0000","last_became_active":"2023-06-10T21:26:27.787854+0000","last_became_peered":"2023-06-10T21:26:27.787854+0000","last_unstale":"2023-06-10T21:27:10.169327+0000","last_undegraded":"2023-06-10T21:27:10.169327+0000","last_fullsized":"2023-06-10T21:26:27.781321+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"227'5891","reported_seq":6645,"reported_epoch":227,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:27:18.760701+0000","last_change":"2023-06-10T21:26:28.977278+0000","last_active":"2023-06-10T21:27:18.760701+0000","last_peered":"2023-06-10T21:27:18.760701+0000","last_clean":"2023-06-10T21:26:15.549220+0000","last_became_active":"2023-06-10T21:26:28.973820+0000","last_became_peered":"2023-06-10T21:26:28.973820+0000","last_unstale":"2023-06-10T21:27:18.760701+0000","last_undegraded":"2023-06-10T21:26:28.947847+0000","last_fullsized":"2023-06-10T21:27:18.760701+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:46.994093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340983808,"num_objects":5203,"num_object_clones":0,"num_object_copies":15609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2991,"num_objects_misplaced":4923,"num_objects_unfound":0,"num_objects_dirty":5203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5891,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":345,"num_bytes_recovered":22609920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2"],"object_location_counts":[{"shards":"2,4","objects":280},{"shards":"2,4,6","objects":2212},{"shards":"2,6","objects":2711}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":208,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.170075+0000","last_change":"2023-06-10T21:26:27.818580+0000","last_active":"2023-06-10T21:27:10.170075+0000","last_peered":"2023-06-10T21:27:10.170075+0000","last_clean":"2023-06-10T21:26:20.562258+0000","last_became_active":"2023-06-10T21:26:27.818580+0000","last_became_peered":"2023-06-10T21:26:27.818580+0000","last_unstale":"2023-06-10T21:27:10.170075+0000","last_undegraded":"2023-06-10T21:27:10.170075+0000","last_fullsized":"2023-06-10T21:26:27.783435+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":69,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388756+0000","last_change":"2023-06-10T21:24:45.166159+0000","last_active":"2023-06-10T21:27:05.388756+0000","last_peered":"2023-06-10T21:27:05.388756+0000","last_clean":"2023-06-10T21:27:05.388756+0000","last_became_active":"2023-06-10T21:24:45.147781+0000","last_became_peered":"2023-06-10T21:24:45.147781+0000","last_unstale":"2023-06-10T21:27:05.388756+0000","last_undegraded":"2023-06-10T21:27:05.388756+0000","last_fullsized":"2023-06-10T21:27:05.388756+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:25:35.150006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.385318+0000","last_change":"2023-06-10T21:26:15.749404+0000","last_active":"2023-06-10T21:27:05.385318+0000","last_peered":"2023-06-10T21:27:05.385318+0000","last_clean":"2023-06-10T21:27:05.385318+0000","last_became_active":"2023-06-10T21:26:15.748925+0000","last_became_peered":"2023-06-10T21:26:15.748925+0000","last_unstale":"2023-06-10T21:27:05.385318+0000","last_undegraded":"2023-06-10T21:27:05.385318+0000","last_fullsized":"2023-06-10T21:27:05.385318+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:13.913738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":64,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.176637+0000","last_change":"2023-06-10T21:24:45.144952+0000","last_active":"2023-06-10T21:27:10.176637+0000","last_peered":"2023-06-10T21:27:10.176637+0000","last_clean":"2023-06-10T21:27:10.176637+0000","last_became_active":"2023-06-10T21:24:45.144474+0000","last_became_peered":"2023-06-10T21:24:45.144474+0000","last_unstale":"2023-06-10T21:27:10.176637+0000","last_undegraded":"2023-06-10T21:27:10.176637+0000","last_fullsized":"2023-06-10T21:27:10.176637+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:25:09.157193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":56,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.180661+0000","last_change":"2023-06-10T21:26:27.817806+0000","last_active":"2023-06-10T21:27:10.180661+0000","last_peered":"2023-06-10T21:27:10.180661+0000","last_clean":"2023-06-10T21:26:20.308187+0000","last_became_active":"2023-06-10T21:26:27.817806+0000","last_became_peered":"2023-06-10T21:26:27.817806+0000","last_unstale":"2023-06-10T21:27:10.180661+0000","last_undegraded":"2023-06-10T21:27:10.180661+0000","last_fullsized":"2023-06-10T21:26:27.783382+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:28:39.270974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":27,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-10T21:27:10.169145+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:27:10.169145+0000","last_peered":"2023-06-10T21:27:10.169145+0000","last_clean":"2023-06-10T21:27:10.169145+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:27:10.169145+0000","last_undegraded":"2023-06-10T21:27:10.169145+0000","last_fullsized":"2023-06-10T21:27:10.169145+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":205,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165681+0000","last_change":"2023-06-10T21:26:27.782629+0000","last_active":"2023-06-10T21:27:10.165681+0000","last_peered":"2023-06-10T21:27:10.165681+0000","last_clean":"2023-06-10T21:19:00.489490+0000","last_became_active":"2023-06-10T21:26:27.782629+0000","last_became_peered":"2023-06-10T21:26:27.782629+0000","last_unstale":"2023-06-10T21:27:10.165681+0000","last_undegraded":"2023-06-10T21:27:10.165681+0000","last_fullsized":"2023-06-10T21:26:27.779384+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":49,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:10.165721+0000","last_change":"2023-06-10T21:26:27.784405+0000","last_active":"2023-06-10T21:27:10.165721+0000","last_peered":"2023-06-10T21:27:10.165721+0000","last_clean":"2023-06-10T21:25:59.679038+0000","last_became_active":"2023-06-10T21:26:27.784405+0000","last_became_peered":"2023-06-10T21:26:27.784405+0000","last_unstale":"2023-06-10T21:27:10.165721+0000","last_undegraded":"2023-06-10T21:26:27.780168+0000","last_fullsized":"2023-06-10T21:26:27.780047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-10T21:27:05.393415+0000","last_change":"2023-06-10T21:26:27.785614+0000","last_active":"2023-06-10T21:27:05.393415+0000","last_peered":"2023-06-10T21:27:05.393415+0000","last_clean":"2023-06-10T21:24:36.631470+0000","last_became_active":"2023-06-10T21:26:27.785614+0000","last_became_peered":"2023-06-10T21:26:27.785614+0000","last_unstale":"2023-06-10T21:27:05.393415+0000","last_undegraded":"2023-06-10T21:27:05.393415+0000","last_fullsized":"2023-06-10T21:26:27.776892+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:46.260297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":200,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.183014+0000","last_change":"2023-06-10T21:26:27.818384+0000","last_active":"2023-06-10T21:27:10.183014+0000","last_peered":"2023-06-10T21:27:10.183014+0000","last_clean":"2023-06-10T21:25:28.189114+0000","last_became_active":"2023-06-10T21:26:27.818384+0000","last_became_peered":"2023-06-10T21:26:27.818384+0000","last_unstale":"2023-06-10T21:27:10.183014+0000","last_undegraded":"2023-06-10T21:27:10.183014+0000","last_fullsized":"2023-06-10T21:26:27.812469+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:26:46.689756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":230,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.176687+0000","last_change":"2023-06-10T21:26:27.820844+0000","last_active":"2023-06-10T21:27:10.176687+0000","last_peered":"2023-06-10T21:27:10.176687+0000","last_clean":"2023-06-10T21:25:28.188991+0000","last_became_active":"2023-06-10T21:26:27.820844+0000","last_became_peered":"2023-06-10T21:26:27.820844+0000","last_unstale":"2023-06-10T21:27:10.176687+0000","last_undegraded":"2023-06-10T21:27:10.176687+0000","last_fullsized":"2023-06-10T21:26:27.817733+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:10.760495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":131,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-10T21:27:05.388755+0000","last_change":"2023-06-10T21:26:15.944630+0000","last_active":"2023-06-10T21:27:05.388755+0000","last_peered":"2023-06-10T21:27:05.388755+0000","last_clean":"2023-06-10T21:27:05.388755+0000","last_became_active":"2023-06-10T21:26:15.750701+0000","last_became_peered":"2023-06-10T21:26:15.750701+0000","last_unstale":"2023-06-10T21:27:05.388755+0000","last_undegraded":"2023-06-10T21:27:05.388755+0000","last_fullsized":"2023-06-10T21:27:05.388755+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:47:02.331086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":208,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-10T21:27:10.165000+0000","last_change":"2023-06-10T21:26:27.791062+0000","last_active":"2023-06-10T21:27:10.165000+0000","last_peered":"2023-06-10T21:27:10.165000+0000","last_clean":"2023-06-10T21:25:15.648591+0000","last_became_active":"2023-06-10T21:26:27.791062+0000","last_became_peered":"2023-06-10T21:26:27.791062+0000","last_unstale":"2023-06-10T21:27:10.165000+0000","last_undegraded":"2023-06-10T21:27:10.165000+0000","last_fullsized":"2023-06-10T21:26:27.779951+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5604900888,"num_objects":85525,"num_object_clones":0,"num_object_copies":256575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24105,"num_objects_misplaced":26609,"num_objects_unfound":0,"num_objects_dirty":85525,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":95983,"num_write_kb":5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9010,"num_bytes_recovered":590479360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16566988800,"data_stored":16566976584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95983,"ondisk_log_size":95983,"up":40,"acting":43,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":7738,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":78,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430563,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2074688,"kb_used_data":1574648,"kb_used_omap":0,"kb_used_meta":500032,"kb_avail":91670464,"statfs":{"total":95995035648,"available":93870555136,"internally_reserved":0,"allocated":1612439552,"data_stored":1611907446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":512032768},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:26:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":3.375,"15min":2.0419999999999998},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":42.994,"5min":53.706000000000003,"15min":53.706000000000003},"last":3.2189999999999999},{"interface":"front","average":{"1min":2.9710000000000001,"5min":3.1659999999999999,"15min":2.2189999999999999},"min":{"1min":0.32500000000000001,"5min":0.247,"15min":0.245},"max":{"1min":43.348999999999997,"5min":53.777000000000001,"15min":53.777000000000001},"last":3.246}]},{"osd":1,"last update":"Sat Jun 10 21:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":3.9729999999999999,"15min":2.5630000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4880000000000004,"5min":130.19399999999999,"15min":130.19399999999999},"last":2.4260000000000002},{"interface":"front","average":{"1min":1.6519999999999999,"5min":5.4800000000000004,"15min":3.0070000000000001},"min":{"1min":0.32000000000000001,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":4.9989999999999997,"5min":212.20599999999999,"15min":212.20599999999999},"last":0.624}]},{"osd":2,"last update":"Sat Jun 10 21:27:13 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":4.992,"15min":2.1179999999999999},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.315},"max":{"1min":115.62,"5min":160.99299999999999,"15min":160.99299999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":7.1310000000000002,"5min":7.1360000000000001,"15min":2.8100000000000001},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.309},"max":{"1min":117.776,"5min":212.13399999999999,"15min":212.13399999999999},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":5.0629999999999997,"15min":2.0619999999999998},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":14.526,"5min":63.406999999999996,"15min":63.406999999999996},"last":0.44},{"interface":"front","average":{"1min":0.83199999999999996,"5min":5.0449999999999999,"15min":2.0720000000000001},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4319999999999999,"5min":63.549999999999997,"15min":63.549999999999997},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.097,"15min":0.97199999999999998},"min":{"1min":0.311,"5min":0.26400000000000001,"15min":0.22800000000000001},"max":{"1min":2.702,"5min":8.1419999999999995,"15min":8.1419999999999995},"last":2.4129999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.887,"15min":1.347},"min":{"1min":0.29099999999999998,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":3.3610000000000002,"5min":14.657999999999999,"15min":14.657999999999999},"last":0.30299999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":3.8610000000000002,"15min":2.0920000000000001},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":0.69599999999999995,"5min":211.99799999999999,"15min":211.99799999999999},"last":3.2759999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":3.7109999999999999,"15min":1.9490000000000001},"min":{"1min":0.35499999999999998,"5min":0.315,"15min":0.30199999999999999},"max":{"1min":2.1859999999999999,"5min":212.23699999999999,"15min":212.23699999999999},"last":0.80500000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:26:39 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":1.663,"15min":1.2330000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":2.1469999999999998,"5min":10.159000000000001,"15min":10.159000000000001},"last":2.3900000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.6890000000000001,"15min":1.151},"min":{"1min":0.39600000000000002,"5min":0.374,"15min":0.32800000000000001},"max":{"1min":2.2130000000000001,"5min":10.874000000000001,"15min":10.874000000000001},"last":0.89000000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626791,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566234,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1780332,"kb_used_data":1349796,"kb_used_omap":4,"kb_used_meta":430523,"kb_avail":91964820,"statfs":{"total":95995035648,"available":94171975680,"internally_reserved":0,"allocated":1382191104,"data_stored":1381443771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":440856462},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":5.0389999999999997,"15min":3.1720000000000002},"min":{"1min":0.20699999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.0270000000000001,"5min":121.917,"15min":121.917},"last":0.371},{"interface":"front","average":{"1min":2.2130000000000001,"5min":5.8890000000000002,"15min":3.7829999999999999},"min":{"1min":0.29799999999999999,"5min":0.25,"15min":0.22600000000000001},"max":{"1min":7.0709999999999997,"5min":121.455,"15min":121.455},"last":0.307}]},{"osd":2,"last update":"Sat Jun 10 21:26:18 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.5680000000000001,"15min":1.2},"min":{"1min":0.255,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7990000000000004,"5min":18.152000000000001,"15min":18.152000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.44400000000000001,"5min":1.6140000000000001,"15min":0.85699999999999998},"min":{"1min":0.22500000000000001,"5min":0.222,"15min":0.218},"max":{"1min":1.278,"5min":12.081,"15min":12.081},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:25:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":4.2030000000000003,"15min":1.641},"min":{"1min":0.29299999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65200000000000002,"5min":118.348,"15min":118.348},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":4.9160000000000004,"15min":1.9159999999999999},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.1870000000000003,"5min":117.75,"15min":117.75},"last":5.1109999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":2.073,"15min":1.8180000000000001},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.9409999999999998,"5min":12.507999999999999,"15min":12.94},"last":1.627},{"interface":"front","average":{"1min":1.417,"5min":1.76,"15min":1.9950000000000001},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.2539999999999996,"5min":14.460000000000001,"15min":14.460000000000001},"last":1.5349999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":9.202,"15min":4.4100000000000001},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":6.9550000000000001,"5min":383.83100000000002,"15min":383.83100000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.6739999999999999,"5min":8.734,"15min":4.3360000000000003},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.9900000000000002,"5min":383.84800000000001,"15min":383.84800000000001},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":2.7269999999999999,"15min":2.262},"min":{"1min":0.443,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.9900000000000002,"5min":14.858000000000001,"15min":14.858000000000001},"last":9.5690000000000008},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3250000000000002,"15min":2.1680000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.0220000000000002,"5min":13.48,"15min":13.48},"last":10.762}]},{"osd":7,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":4.5099999999999998,"15min":3.1709999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.32600000000000001},"max":{"1min":8.0890000000000004,"5min":162.399,"15min":162.399},"last":2.5459999999999998},{"interface":"front","average":{"1min":2.0129999999999999,"5min":3.7930000000000001,"15min":2.5419999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.38200000000000001},"max":{"1min":6.2160000000000002,"5min":162.27199999999999,"15min":162.27199999999999},"last":4.4800000000000004}]}]},{"osd":0,"up_from":86,"seq":369367187655,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3991328,"kb_used_data":3480088,"kb_used_omap":5,"kb_used_meta":511226,"kb_avail":89753824,"statfs":{"total":95995035648,"available":91907915776,"internally_reserved":0,"allocated":3563610112,"data_stored":3562805855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":523496058},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:26:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":6.3140000000000001,"15min":3.8889999999999998},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.0710000000000002,"5min":175.352,"15min":175.352},"last":5.1890000000000001},{"interface":"front","average":{"1min":1.8660000000000001,"5min":6.1790000000000003,"15min":4.4279999999999999},"min":{"1min":0.23899999999999999,"5min":0.189,"15min":0.189},"max":{"1min":8.8780000000000001,"5min":175.30500000000001,"15min":175.30500000000001},"last":3.5819999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":7.1589999999999998,"5min":3.032,"15min":1.361},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":98.710999999999999,"5min":98.710999999999999,"15min":98.710999999999999},"last":4.4169999999999998},{"interface":"front","average":{"1min":7.5309999999999997,"5min":3.1480000000000001,"15min":1.3999999999999999},"min":{"1min":0.30499999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":98.772999999999996,"5min":98.772999999999996,"15min":98.772999999999996},"last":3.427}]},{"osd":3,"last update":"Sat Jun 10 21:25:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":8.7550000000000008,"15min":3.1859999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":1.524,"5min":181.886,"15min":181.886},"last":0.371},{"interface":"front","average":{"1min":0.70199999999999996,"5min":9.5440000000000005,"15min":3.4390000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9049999999999998,"5min":180.04900000000001,"15min":180.04900000000001},"last":3.8759999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":7.9580000000000002,"5min":3.2389999999999999,"15min":2.3090000000000002},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":66.915999999999997,"5min":66.915999999999997,"15min":66.915999999999997},"last":0.76200000000000001},{"interface":"front","average":{"1min":8.423,"5min":3.2280000000000002,"15min":2.0790000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.29699999999999999},"max":{"1min":66.891999999999996,"5min":66.891999999999996,"15min":66.891999999999996},"last":5.423}]},{"osd":5,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.0790000000000002,"15min":1.7789999999999999},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0380000000000003,"5min":25.545000000000002,"15min":25.545000000000002},"last":5.4610000000000003},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.0699999999999998,"15min":2.1930000000000001},"min":{"1min":0.41399999999999998,"5min":0.375,"15min":0.32400000000000001},"max":{"1min":8.7449999999999992,"5min":8.7449999999999992,"15min":17.460000000000001},"last":3.5059999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":2.1230000000000002,"15min":1.857},"min":{"1min":0.375,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.2959999999999994,"5min":25.608000000000001,"15min":25.608000000000001},"last":5.5049999999999999},{"interface":"front","average":{"1min":2.3220000000000001,"5min":2.2530000000000001,"15min":2.2730000000000001},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.7929999999999993,"5min":10.231999999999999,"15min":17.411000000000001},"last":3.4700000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.2770000000000001,"15min":2.246},"min":{"1min":0.437,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.8230000000000004,"5min":10.269,"15min":17.486999999999998},"last":3.5390000000000001},{"interface":"front","average":{"1min":1.395,"5min":1.9630000000000001,"15min":2.0190000000000001},"min":{"1min":0.39700000000000002,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":6.9720000000000004,"5min":9.734,"15min":10.867000000000001},"last":0.80700000000000005}]}]},{"osd":2,"up_from":120,"seq":515396075622,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3036896,"kb_used_data":2721880,"kb_used_omap":7,"kb_used_meta":315000,"kb_avail":90708256,"statfs":{"total":95995035648,"available":92885254144,"internally_reserved":0,"allocated":2787205120,"data_stored":2786470151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":322560081},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":11.116,"5min":3.855,"15min":1.7310000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":237.251,"5min":237.251,"15min":237.251},"last":237.251},{"interface":"front","average":{"1min":11.191000000000001,"5min":4.6070000000000002,"15min":1.978},"min":{"1min":0.20599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":232.06700000000001,"5min":232.06700000000001,"15min":232.06700000000001},"last":232.06700000000001}]},{"osd":1,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":2.1899999999999999,"15min":1.1850000000000001},"min":{"1min":0.23799999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.6880000000000006,"5min":17.718,"15min":29.486000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.163,"5min":1.8560000000000001,"15min":1.093},"min":{"1min":0.22500000000000001,"5min":0.216,"15min":0.216},"max":{"1min":8.1959999999999997,"5min":16.641999999999999,"15min":28.013999999999999},"last":0.432}]},{"osd":3,"last update":"Sat Jun 10 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":4.2549999999999999,"15min":1.6839999999999999},"min":{"1min":0.32200000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6989999999999998,"5min":96.295000000000002,"15min":96.295000000000002},"last":4.157},{"interface":"front","average":{"1min":1.0449999999999999,"5min":4.8460000000000001,"15min":1.899},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.9649999999999999,"5min":99.504999999999995,"15min":99.504999999999995},"last":2.9489999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":13.571,"5min":4.2430000000000003,"15min":1.9179999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":224.00999999999999,"5min":224.00999999999999,"15min":224.00999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":14.023,"5min":4.3860000000000001,"15min":1.9870000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":223.28399999999999,"5min":223.28399999999999,"15min":223.28399999999999},"last":0.73199999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":1.518,"15min":1.0860000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.1219999999999999,"5min":10.254,"15min":10.254},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.0659999999999998,"15min":1.2589999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2850000000000001,"5min":10.345000000000001,"15min":10.345000000000001},"last":0.76100000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.0790000000000002,"15min":1.258},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.3889999999999993,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.8879999999999999,"5min":2.3809999999999998,"15min":1.3660000000000001},"min":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8420000000000005,"5min":17.244,"15min":17.244},"last":0.79800000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.5089999999999999,"15min":1.069},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.9339999999999999,"5min":2.1789999999999998,"15min":1.3},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.82499999999999996}]}]},{"osd":3,"up_from":125,"seq":536870912085,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267612,"kb_used_data":1002452,"kb_used_omap":2,"kb_used_meta":265149,"kb_avail":92477540,"statfs":{"total":95995035648,"available":94697000960,"internally_reserved":0,"allocated":1026510848,"data_stored":1025770762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":271512629},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":2.234,"15min":0.98599999999999999},"min":{"1min":0.29399999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.0419999999999998,"5min":46.677999999999997,"15min":46.677999999999997},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.5049999999999999,"5min":4.2110000000000003,"15min":1.639},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":10.273999999999999,"5min":130.893,"15min":130.893},"last":5.5300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.6869999999999998,"15min":1.1639999999999999},"min":{"1min":0.33900000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.0030000000000001,"5min":54.112000000000002,"15min":54.112000000000002},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":3.282,"15min":1.3700000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29299999999999998},"max":{"1min":8.2810000000000006,"5min":57.814,"15min":57.814},"last":5.5609999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":2.2639999999999998,"15min":1.014},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":10.047000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":5.4859999999999998},{"interface":"front","average":{"1min":1.514,"5min":1.7989999999999999,"15min":0.86099999999999999},"min":{"1min":0.313,"5min":0.249,"15min":0.249},"max":{"1min":6.0030000000000001,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.0409999999999999,"15min":1.0629999999999999},"min":{"1min":0.39100000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.3190000000000008,"5min":13.656000000000001,"15min":13.656000000000001},"last":2.4910000000000001},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.0089999999999999,"15min":1.052},"min":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.3200000000000003,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":9.0730000000000004,"15min":3.4220000000000002},"min":{"1min":0.498,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.3580000000000005,"5min":349.47399999999999,"15min":349.47399999999999},"last":2.448},{"interface":"front","average":{"1min":2.8180000000000001,"5min":8.0099999999999998,"15min":3.0760000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.3740000000000006,"5min":349.65100000000001,"15min":349.65100000000001},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.2610000000000001,"15min":1.1479999999999999},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.9800000000000004,"5min":9.9619999999999997,"15min":9.9619999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.675,"5min":2.0339999999999998,"15min":1.1220000000000001},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":8.3339999999999996,"5min":12.835000000000001,"15min":12.835000000000001},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":5.3150000000000004,"15min":2.1789999999999998},"min":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.2230000000000008,"5min":174.44800000000001,"15min":174.44800000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.6320000000000001,"5min":3.8570000000000002,"15min":1.734},"min":{"1min":0.40400000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9450000000000003,"5min":173.18100000000001,"15min":173.18100000000001},"last":0.72999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019247,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4525440,"kb_used_data":4044280,"kb_used_omap":0,"kb_used_meta":481152,"kb_avail":89219712,"statfs":{"total":95995035648,"available":91360985088,"internally_reserved":0,"allocated":4141342720,"data_stored":4140810333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":492699648},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":1.8520000000000001,"15min":1.849},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":48.390999999999998,"5min":48.390999999999998,"15min":48.390999999999998},"last":48.390999999999998},{"interface":"front","average":{"1min":3.8010000000000002,"5min":2.29,"15min":2.2109999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23100000000000001},"max":{"1min":47.991999999999997,"5min":47.991999999999997,"15min":47.991999999999997},"last":47.991999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":5.0830000000000002,"15min":3.0510000000000002},"min":{"1min":0.34000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":15.063000000000001,"5min":182.035,"15min":182.035},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.1800000000000002,"5min":4.4390000000000001,"15min":3.024},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":15.052,"5min":181.73099999999999,"15min":181.73099999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.0659999999999998,"15min":1.0800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":22.614000000000001,"5min":22.614000000000001,"15min":22.614000000000001},"last":0.498},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.04,"15min":1.147},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":22.326000000000001,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":6.9969999999999999,"5min":7.4119999999999999,"15min":2.8420000000000001},"min":{"1min":0.498,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":59.084000000000003,"5min":103.575,"15min":103.575},"last":6.601},{"interface":"front","average":{"1min":5.8949999999999996,"5min":6.7610000000000001,"15min":2.6669999999999998},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":59.128,"5min":103.565,"15min":103.565},"last":6.6120000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.8140000000000001,"15min":1.131},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":3.399,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.046,"5min":1.7030000000000001,"15min":1.212},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.5209999999999999,"5min":11.904999999999999,"15min":11.904999999999999},"last":0.247}]},{"osd":6,"last update":"Sat Jun 10 21:26:42 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":1.782,"15min":1.284},"min":{"1min":0.35799999999999998,"5min":0.28599999999999998,"15min":0.26700000000000002},"max":{"1min":2.2530000000000001,"5min":8.2599999999999998,"15min":8.2599999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.843,"15min":1.282},"min":{"1min":0.379,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":2.448,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.32700000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:26:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":2.1150000000000002,"15min":1.484},"min":{"1min":0.35299999999999998,"5min":0.34100000000000003,"15min":0.308},"max":{"1min":3.2999999999999998,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.875,"15min":1.298},"min":{"1min":0.34499999999999997,"5min":0.33300000000000002,"15min":0.314},"max":{"1min":3.7469999999999999,"5min":9.5060000000000002,"15min":9.5060000000000002},"last":0.28000000000000003}]}]},{"osd":5,"up_from":0,"seq":154618823019,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3556315136,"data_stored":3556311064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1378746368,"data_stored":1378746368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2769420288,"data_stored":2769420288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1040711680,"data_stored":1040711680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4109570048,"data_stored":4109565976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":15204352,"data_stored":15204352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2087915520,"data_stored":2087911448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1609105408,"data_stored":1609105408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:27:25.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_5 pgp_num 16 2023-06-10T21:27:25.052 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:25.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:25.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:25.244 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:25 smithi064 ceph-mon[180297]: pgmap v808: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.8 MiB/s wr, 670 op/s; 23892/252090 objects degraded (9.478%); 25832/252090 objects misplaced (10.247%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:25.250 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:25.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:25 smithi064 ceph-mon[186706]: pgmap v808: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.8 MiB/s wr, 670 op/s; 23892/252090 objects degraded (9.478%); 25832/252090 objects misplaced (10.247%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:25.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:25.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:25.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:25 smithi152 ceph-mon[156819]: pgmap v808: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.8 MiB/s wr, 670 op/s; 23892/252090 objects degraded (9.478%); 25832/252090 objects misplaced (10.247%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:25.537 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:25.582 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:25.676 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:25.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:25.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:25.821 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:25.874 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:25.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:25.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:26.008 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:26.053 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:26.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:26.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:26.187 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:26.231 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:26.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:26.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:26.426 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 23892/252090 objects degraded (9.478%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:26 smithi152 ceph-mon[156819]: osdmap e228: 8 total, 7 up, 6 in 2023-06-10T21:27:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:26 smithi152 ceph-mon[156819]: failed to encode map e228 with expected crc 2023-06-10T21:27:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/465867273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:26 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.471 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:26.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:26.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:26.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 23892/252090 objects degraded (9.478%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:26.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[186706]: osdmap e228: 8 total, 7 up, 6 in 2023-06-10T21:27:26.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[186706]: failed to encode map e228 with expected crc 2023-06-10T21:27:26.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/465867273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 23892/252090 objects degraded (9.478%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:26.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[180297]: osdmap e228: 8 total, 7 up, 6 in 2023-06-10T21:27:26.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[180297]: failed to encode map e228 with expected crc 2023-06-10T21:27:26.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/465867273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:26 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:27:26.613 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:26.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:27:26.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:26.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:26.700 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:27:26.715 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:26.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:26.894 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:27.377 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:27.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: pgmap v809: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 23019/247038 objects degraded (9.318%); 25198/247038 objects misplaced (10.200%); 9.7 MiB/s, 155 objects/s recovering 2023-06-10T21:27:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: failed to encode map e228 with expected crc 2023-06-10T21:27:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:27:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: osdmap e229: 8 total, 7 up, 6 in 2023-06-10T21:27:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:27 smithi152 ceph-mon[156819]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:27.562 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:27.580 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:27:27.581 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:27:27.584 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: pgmap v809: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 23019/247038 objects degraded (9.318%); 25198/247038 objects misplaced (10.200%); 9.7 MiB/s, 155 objects/s recovering 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: failed to encode map e228 with expected crc 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: osdmap e229: 8 total, 7 up, 6 in 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[186706]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: pgmap v809: 89 pgs: 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 670 op/s; 23019/247038 objects degraded (9.318%); 25198/247038 objects misplaced (10.200%); 9.7 MiB/s, 155 objects/s recovering 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: failed to encode map e228 with expected crc 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: osdmap e229: 8 total, 7 up, 6 in 2023-06-10T21:27:27.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:27 smithi064 ceph-mon[180297]: failed to encode map e229 with expected crc 2023-06-10T21:27:27.639 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:27.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:27.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:27.744 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:27.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:28.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:28 smithi064 ceph-mon[180297]: failed to encode map e229 with expected crc 2023-06-10T21:27:28.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:28 smithi064 ceph-mon[186706]: failed to encode map e229 with expected crc 2023-06-10T21:27:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:28 smithi152 ceph-mon[156819]: failed to encode map e229 with expected crc 2023-06-10T21:27:28.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:28.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:28.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:28.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:28.961 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:29.505 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:29.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:29 smithi064 ceph-mon[180297]: pgmap v812: 89 pgs: 1 peering, 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 22839/242652 objects degraded (9.412%); 24506/242652 objects misplaced (10.099%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:29.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:29 smithi064 ceph-mon[186706]: pgmap v812: 89 pgs: 1 peering, 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 22839/242652 objects degraded (9.412%); 24506/242652 objects misplaced (10.099%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:29.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:29.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:29.675 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:29 smithi152 ceph-mon[156819]: pgmap v812: 89 pgs: 1 peering, 2 active+clean+remapped, 3 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 22839/242652 objects degraded (9.412%); 24506/242652 objects misplaced (10.099%); 8.8 MiB/s, 140 objects/s recovering 2023-06-10T21:27:29.681 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:29.736 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:29.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:29.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:29.884 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:29.930 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:29.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:30.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:30.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:30.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:27:30.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:27:30.606 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:30.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:30.720 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:30.876 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:30.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:31.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:31.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:31.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:31.195 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:31.270 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:31.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:31.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:31.416 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:31.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:31 smithi152 ceph-mon[156819]: pgmap v813: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 21873/238329 objects degraded (9.178%); 24238/238329 objects misplaced (10.170%); 14 MiB/s, 217 objects/s recovering 2023-06-10T21:27:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21873/238329 objects degraded (9.178%), 12 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:31.495 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:31.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:31.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:31.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:31 smithi064 ceph-mon[180297]: pgmap v813: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 21873/238329 objects degraded (9.178%); 24238/238329 objects misplaced (10.170%); 14 MiB/s, 217 objects/s recovering 2023-06-10T21:27:31.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21873/238329 objects degraded (9.178%), 12 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:31.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:31 smithi064 ceph-mon[186706]: pgmap v813: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 762 op/s; 21873/238329 objects degraded (9.178%); 24238/238329 objects misplaced (10.170%); 14 MiB/s, 217 objects/s recovering 2023-06-10T21:27:31.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21873/238329 objects degraded (9.178%), 12 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:31.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:31.641 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:31.697 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:32.230 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pgp_num to 16 2023-06-10T21:27:32.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/844690352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:32.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:32 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/844690352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:32.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:32 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/844690352' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:32 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:27:33.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:32] "GET /metrics HTTP/1.1" 200 224083 "" "Prometheus/2.33.4" 2023-06-10T21:27:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: pgmap v814: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 19955/237822 objects degraded (8.391%); 25815/237822 objects misplaced (10.855%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:27:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: osdmap e230: 8 total, 7 up, 6 in 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: pgmap v814: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 19955/237822 objects degraded (8.391%); 25815/237822 objects misplaced (10.855%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:33.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: osdmap e230: 8 total, 7 up, 6 in 2023-06-10T21:27:33.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:33.691 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:33.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: pgmap v814: 89 pgs: 1 peering, 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 35 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 19955/237822 objects degraded (8.391%); 25815/237822 objects misplaced (10.855%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:27:33.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:27:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: osdmap e230: 8 total, 7 up, 6 in 2023-06-10T21:27:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: failed to encode map e230 with expected crc 2023-06-10T21:27:33.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:33.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:33.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:33.840 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:33.921 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:33.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:33.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:34.000 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:34.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:34.105 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:34.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:34.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:34.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:34.582 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:27:34.627 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:34.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:34.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:34.696 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:34.706 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:34.725 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:34.762 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:34.780 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:34.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:34.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:34.798 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.798 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.799 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:27:34.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:34.957 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:35 smithi064 ceph-mon[180297]: pgmap v816: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 19749/233244 objects degraded (8.467%); 25405/233244 objects misplaced (10.892%); 10 MiB/s, 162 objects/s recovering 2023-06-10T21:27:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1072855362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:35 smithi064 ceph-mon[186706]: pgmap v816: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 19749/233244 objects degraded (8.467%); 25405/233244 objects misplaced (10.892%); 10 MiB/s, 162 objects/s recovering 2023-06-10T21:27:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1072855362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:35 smithi152 ceph-mon[156819]: pgmap v816: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 19749/233244 objects degraded (8.467%); 25405/233244 objects misplaced (10.892%); 10 MiB/s, 162 objects/s recovering 2023-06-10T21:27:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1072855362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:36.124 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:36.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:36.231 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:36.272 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:36.347 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:36.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:36.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 19749/233244 objects degraded (8.467%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:36.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 19749/233244 objects degraded (8.467%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:36.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:36.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 19749/233244 objects degraded (8.467%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:36.502 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:36.575 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:36.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:36.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:36.736 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:37.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:37 smithi064 ceph-mon[180297]: pgmap v817: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 18942/228282 objects degraded (8.298%); 24815/228282 objects misplaced (10.870%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:27:37.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:37 smithi064 ceph-mon[186706]: pgmap v817: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 18942/228282 objects degraded (8.298%); 24815/228282 objects misplaced (10.870%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:27:37.438 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:37.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:37.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:37.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:37 smithi152 ceph-mon[156819]: pgmap v817: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 18942/228282 objects degraded (8.298%); 24815/228282 objects misplaced (10.870%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:27:38.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:38.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:38.084 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:38.085 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:38.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:38.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:38.443 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:38.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:38.550 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:27:38.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:38.685 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:27:38.686 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:27:38.686 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:27:38.686 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:27:38.686 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:27:38.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:27:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:38.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:27:38.883 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-06-10T21:27:38.883 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:27:38.884 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 4, 5, 6, 7] 2023-06-10T21:27:38.884 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-06-10T21:27:38.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0 2023-06-10T21:27:38.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:39.007 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:39.084 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:39.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:39.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:39.266 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:27:38 smithi064 systemd[1]: Stopping Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:27:39.269 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:39 smithi152 ceph-mon[156819]: pgmap v818: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 18718/222885 objects degraded (8.398%); 23911/222885 objects misplaced (10.728%); 11 MiB/s, 180 objects/s recovering 2023-06-10T21:27:39.269 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:39.273 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:27:39.275 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:39.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:39 smithi064 ceph-mon[180297]: pgmap v818: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 18718/222885 objects degraded (8.398%); 23911/222885 objects misplaced (10.728%); 11 MiB/s, 180 objects/s recovering 2023-06-10T21:27:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:39.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:39 smithi064 ceph-mon[186706]: pgmap v818: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 18718/222885 objects degraded (8.398%); 23911/222885 objects misplaced (10.728%); 11 MiB/s, 180 objects/s recovering 2023-06-10T21:27:39.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:27:40.086 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:27:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:27:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:40.283 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[186706]: osdmap e231: 8 total, 7 up, 6 in 2023-06-10T21:27:40.284 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[186706]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.284 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[186706]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.287 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:40.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:40.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:40.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:40.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:27:40.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:40.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[180297]: osdmap e231: 8 total, 7 up, 6 in 2023-06-10T21:27:40.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[180297]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:40 smithi064 ceph-mon[180297]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.582 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:27:40 smithi064 conmon[200073]: 2023-06-10T21:27:40.572+0000 7f5a97f66700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:27:40.582 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:27:40 smithi064 conmon[200073]: 2023-06-10T21:27:40.572+0000 7f5a97f66700 -1 osd.0 231 *** Got signal Terminated *** 2023-06-10T21:27:40.582 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:27:40 smithi064 conmon[200073]: 2023-06-10T21:27:40.572+0000 7f5a97f66700 -1 osd.0 231 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:27:40.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:27:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:40 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:27:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/185111501' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:27:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:40 smithi152 ceph-mon[156819]: osdmap e231: 8 total, 7 up, 6 in 2023-06-10T21:27:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:40 smithi152 ceph-mon[156819]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:40 smithi152 ceph-mon[156819]: failed to encode map e231 with expected crc 2023-06-10T21:27:40.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:40.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:41.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:41.048 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:41.115 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:41.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:41.202 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:41.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:41.311 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:41.329 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:41.414 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:41.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:41 smithi152 ceph-mon[156819]: pgmap v820: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 17820/217758 objects degraded (8.183%); 23233/217758 objects misplaced (10.669%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:27:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:41 smithi152 ceph-mon[156819]: osd.0 marked itself down and dead 2023-06-10T21:27:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 17820/217758 objects degraded (8.183%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:41.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:41.531 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[180297]: pgmap v820: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 17820/217758 objects degraded (8.183%); 23233/217758 objects misplaced (10.669%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:27:41.531 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[180297]: osd.0 marked itself down and dead 2023-06-10T21:27:41.531 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 17820/217758 objects degraded (8.183%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:41.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[186706]: pgmap v820: 89 pgs: 3 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 36 active+clean, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 17820/217758 objects degraded (8.183%); 23233/217758 objects misplaced (10.669%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:27:41.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[186706]: osd.0 marked itself down and dead 2023-06-10T21:27:41.532 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 17820/217758 objects degraded (8.183%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:41.544 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:41.587 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:41.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:41.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:41.720 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:41.763 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:41.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:41.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:41.907 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:42.632 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:42.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[186706]: Health check update: 2 osds down (OSD_DOWN) 2023-06-10T21:27:42.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[186706]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[186706]: osdmap e232: 8 total, 6 up, 6 in 2023-06-10T21:27:42.634 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[186706]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.634 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[180297]: Health check update: 2 osds down (OSD_DOWN) 2023-06-10T21:27:42.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[180297]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[180297]: osdmap e232: 8 total, 6 up, 6 in 2023-06-10T21:27:42.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:42 smithi064 ceph-mon[180297]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:42.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:42 smithi152 ceph-mon[156819]: Health check update: 2 osds down (OSD_DOWN) 2023-06-10T21:27:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:42 smithi152 ceph-mon[156819]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:42 smithi152 ceph-mon[156819]: osdmap e232: 8 total, 6 up, 6 in 2023-06-10T21:27:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:42 smithi152 ceph-mon[156819]: failed to encode map e232 with expected crc 2023-06-10T21:27:42.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:42.768 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:42.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:42] "GET /metrics HTTP/1.1" 200 224096 "" "Prometheus/2.33.4" 2023-06-10T21:27:42.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:27:43.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:43.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:43.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:43.251 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:43.296 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:43.301 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:27:42 smithi064 bash[335925]: 0499b076219a9cf46ac69734dce2f81c65e031c1c1a976aaa3574d7c15a14f86 2023-06-10T21:27:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:43.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:43.404 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:43.430 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:43.474 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:43.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:43.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:43.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: pgmap v822: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: failed to encode map e232 with expected crc 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: osdmap e233: 8 total, 6 up, 6 in 2023-06-10T21:27:43.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[186706]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: pgmap v822: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: failed to encode map e232 with expected crc 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: osdmap e233: 8 total, 6 up, 6 in 2023-06-10T21:27:43.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:43 smithi064 ceph-mon[180297]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.612 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: pgmap v822: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 671 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:27:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: failed to encode map e232 with expected crc 2023-06-10T21:27:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: failed to encode map e233 with expected crc 2023-06-10T21:27:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: osdmap e233: 8 total, 6 up, 6 in 2023-06-10T21:27:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:43 smithi152 ceph-mon[156819]: failed to encode map e233 with expected crc 2023-06-10T21:27:44.014 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:44.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:44.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:44.150 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:44.194 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:44.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:44.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:44.430 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:44.535 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:44.569 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:44.634 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:44.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:44.766 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:44.808 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:44.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:44.938 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:44.943 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:44.981 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:44.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:44.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:45.029 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:45.060 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:45.085 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:45.087 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:45.109 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:45.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:45.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:45.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:45.170 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:27:45.181 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:45.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:45.246 INFO:teuthology.orchestra.run.smithi064.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2023-06-10T21:27:45.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:45.314 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:45.401 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:45.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:45.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:45.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:45.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[186706]: pgmap v824: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:27:45.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[186706]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[186706]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[186706]: osdmap e234: 8 total, 6 up, 6 in 2023-06-10T21:27:45.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2274049138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:45.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[180297]: pgmap v824: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:27:45.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[180297]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[180297]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[180297]: osdmap e234: 8 total, 6 up, 6 in 2023-06-10T21:27:45.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2274049138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:45.532 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:45.575 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:45.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:45 smithi152 ceph-mon[156819]: pgmap v824: 89 pgs: 2 stale+active+clean+remapped, 3 stale+active+undersized+degraded, 15 stale+active+clean, 10 stale+active+undersized, 1 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 21 active+clean, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 28 active+undersized; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 320 op/s; 17820/217122 objects degraded (8.207%); 22873/217122 objects misplaced (10.535%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:27:45.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:45 smithi152 ceph-mon[156819]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:45 smithi152 ceph-mon[156819]: failed to encode map e234 with expected crc 2023-06-10T21:27:45.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:45 smithi152 ceph-mon[156819]: osdmap e234: 8 total, 6 up, 6 in 2023-06-10T21:27:45.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2274049138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:45.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:45.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:45.753 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:45.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:45.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:45.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:45.887 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:45.929 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:45.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:46.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:46.061 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:46.104 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:46.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:46.210 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:46.236 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:46.281 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:46.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:46.387 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:46.413 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:46.455 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:46.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:46.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:46.589 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[186706]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[186706]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[186706]: osdmap e235: 8 total, 6 up, 6 in 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[186706]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 17820/217122 objects degraded (8.207%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[180297]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[180297]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[180297]: osdmap e235: 8 total, 6 up, 6 in 2023-06-10T21:27:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[180297]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 17820/217122 objects degraded (8.207%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:46.593 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:46 smithi152 ceph-mon[156819]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:46 smithi152 ceph-mon[156819]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:46 smithi152 ceph-mon[156819]: osdmap e235: 8 total, 6 up, 6 in 2023-06-10T21:27:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:46 smithi152 ceph-mon[156819]: failed to encode map e235 with expected crc 2023-06-10T21:27:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 17820/217122 objects degraded (8.207%), 11 pgs degraded, 48 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:47.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:47 smithi064 ceph-mon[186706]: pgmap v827: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 5 stale+active+clean, 5 stale+active+undersized, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 10 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 10 undersized+peered, 38 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 43373/211110 objects degraded (20.545%); 30147/211110 objects misplaced (14.280%); 7.6 MiB/s, 118 objects/s recovering 2023-06-10T21:27:47.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:47 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 7 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:27:47.321 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:47 smithi064 ceph-mon[180297]: pgmap v827: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 5 stale+active+clean, 5 stale+active+undersized, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 10 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 10 undersized+peered, 38 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 43373/211110 objects degraded (20.545%); 30147/211110 objects misplaced (14.280%); 7.6 MiB/s, 118 objects/s recovering 2023-06-10T21:27:47.321 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:47 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 7 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:27:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:47 smithi152 ceph-mon[156819]: pgmap v827: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 5 stale+active+clean, 5 stale+active+undersized, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 10 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 10 undersized+peered, 38 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 43373/211110 objects degraded (20.545%); 30147/211110 objects misplaced (14.280%); 7.6 MiB/s, 118 objects/s recovering 2023-06-10T21:27:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:47 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 7 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:27:48.041 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:48.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:48.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:48.214 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:48.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:48.345 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:48.388 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:48.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:48.425 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:27:48.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:27:48.495 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:48.521 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:48.563 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:48.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:48.698 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:48.741 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:48.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:48.872 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:48.873 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:48.915 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:48.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:49.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:49.056 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:49.206 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:49.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:49.311 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:49.337 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:49.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:49.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:49.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[180297]: pgmap v828: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 52146/209829 objects degraded (24.852%); 30147/209829 objects misplaced (14.367%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:27:49.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[186706]: pgmap v828: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 52146/209829 objects degraded (24.852%); 30147/209829 objects misplaced (14.367%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:27:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:49.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:49 smithi152 ceph-mon[156819]: pgmap v828: 89 pgs: 1 active+clean+remapped, 1 creating+peering, 2 active+recovery_wait+degraded+remapped, 9 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 405 op/s; 52146/209829 objects degraded (24.852%); 30147/209829 objects misplaced (14.367%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:27:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:27:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:27:49.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:49.511 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:49.555 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:49.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:49.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:49.688 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:49.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:49.837 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:49.864 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:49.908 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:49.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:50.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:50.049 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:50.152 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:27:50.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:50.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:50.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:50.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/826824672' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:50.334 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:50 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:50.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/826824672' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:50.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:50 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:50.337 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:50.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:27:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/826824672' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:50 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:27:51.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: pgmap v829: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 15 MiB/s, 241 objects/s recovering 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: osdmap e236: 8 total, 6 up, 6 in 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 54461/209829 objects degraded (25.955%), 21 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[186706]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: pgmap v829: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 15 MiB/s, 241 objects/s recovering 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: osdmap e236: 8 total, 6 up, 6 in 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 54461/209829 objects degraded (25.955%), 21 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:51.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:51 smithi064 ceph-mon[180297]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0.service' 2023-06-10T21:27:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: pgmap v829: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 331 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 15 MiB/s, 241 objects/s recovering 2023-06-10T21:27:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:27:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:27:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: osdmap e236: 8 total, 6 up, 6 in 2023-06-10T21:27:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 54461/209829 objects degraded (25.955%), 21 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:27:51.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:51 smithi152 ceph-mon[156819]: failed to encode map e236 with expected crc 2023-06-10T21:27:51.484 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:27:51.485 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-06-10T21:27:51.545 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:51.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:51.650 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:51.683 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:51.732 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:51.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:51.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:51.865 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:51.911 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:51.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:52.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:52.042 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:52.340 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:52 smithi064 ceph-mon[180297]: failed to encode map e236 with expected crc 2023-06-10T21:27:52.341 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:52 smithi064 ceph-mon[186706]: failed to encode map e236 with expected crc 2023-06-10T21:27:52.342 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:52.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:52.391 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:52 smithi152 ceph-mon[156819]: failed to encode map e236 with expected crc 2023-06-10T21:27:52.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:52.473 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:52.515 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:52.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:52.554 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:27:52.555 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:27:52.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:52.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:52.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:27:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:27:52] "GET /metrics HTTP/1.1" 200 224086 "" "Prometheus/2.33.4" 2023-06-10T21:27:53.257 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:53.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:53.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:53.391 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:53.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:53 smithi064 ceph-mon[186706]: pgmap v831: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 13 MiB/s, 211 objects/s recovering 2023-06-10T21:27:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:53 smithi064 ceph-mon[180297]: pgmap v831: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 13 MiB/s, 211 objects/s recovering 2023-06-10T21:27:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:53 smithi152 ceph-mon[156819]: pgmap v831: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 177 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 13 MiB/s, 211 objects/s recovering 2023-06-10T21:27:53.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:27:53.775 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:53.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:53.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:53.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:53.964 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:53.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:54.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:54.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:54.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:27:54.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:54.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:54.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:54.561 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:54.602 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:54.629 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:54.672 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:54.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:54.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:54.804 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:54.848 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:54.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:54.926 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:27:54.950 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:27:54.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:54.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:27:54.981 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:54.994 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:27:55.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:27:55.024 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:55.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:55.039 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:27:55.039 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:27:55.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:55.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:55.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:55.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:55 smithi064 ceph-mon[186706]: pgmap v832: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 9.4 MiB/s, 151 objects/s recovering 2023-06-10T21:27:55.371 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1245087613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:55.371 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:55 smithi064 ceph-mon[180297]: pgmap v832: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 9.4 MiB/s, 151 objects/s recovering 2023-06-10T21:27:55.371 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1245087613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:55.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:55 smithi152 ceph-mon[156819]: pgmap v832: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 51 op/s; 54461/209829 objects degraded (25.955%); 27296/209829 objects misplaced (13.009%); 9.4 MiB/s, 151 objects/s recovering 2023-06-10T21:27:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1245087613' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:27:55.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:55.500 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:55.544 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:55.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:55.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:55.676 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:55.719 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:55.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:55.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:55.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:55.854 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:55.897 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:55.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:56.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:56.029 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.071 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:56.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:56.205 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.247 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:56.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:56.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:56.377 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:56 smithi064 ceph-mon[180297]: Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:27:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:56 smithi064 ceph-mon[186706]: Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:27:56.419 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:56.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:56 smithi152 ceph-mon[156819]: Health check update: Reduced data availability: 11 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:27:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:56.486 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3, 0] live_osds: [1, 2, 4, 5, 6, 7] 2023-06-10T21:27:56.486 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:27:56.486 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-06-10T21:27:56.486 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:27:56.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:56.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.596 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:56.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:56.702 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:56.728 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.770 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:56.876 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:56.901 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:56.943 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:56.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:56.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:56.972 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:27:56.972 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:27:57.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:57.075 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:57.117 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:57.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:57.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:57 smithi064 ceph-mon[180297]: pgmap v833: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 42 op/s; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:57.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:57.249 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:57.292 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:57.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:57.373 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:27:57.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:27:57.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:57 smithi152 ceph-mon[156819]: pgmap v833: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 42 op/s; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:57.424 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:57.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:57 smithi064 ceph-mon[186706]: pgmap v833: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 42 op/s; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:57.462 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:27:57.463 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:27:57.463 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:27:57.463 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:27:57.463 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:27:57.467 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:27:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:57.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:27:57.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:57.609 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:58.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:58.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:58.640 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:27:58.666 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:58.708 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:27:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:58.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:27:58.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:27:58.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:27:58.845 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:58.953 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:27:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:58.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:27:58.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:27:59.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:27:59.085 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:59.128 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:27:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:59.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:27:59.236 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:27:59.262 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:59.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:27:59 smithi152 ceph-mon[156819]: pgmap v834: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:59.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:27:59 smithi064 ceph-mon[186706]: pgmap v834: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:59.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:27:59 smithi064 ceph-mon[180297]: pgmap v834: 89 pgs: 1 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 54461/209829 objects degraded (25.955%); 26064/209829 objects misplaced (12.422%); 16 MiB/s, 248 objects/s recovering 2023-06-10T21:27:59.714 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:27:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:59.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:27:59.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:27:59.843 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:27:59.886 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:27:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:27:59.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:27:59.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:00.019 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:00.061 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:00.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:00.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:00.197 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:00.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:00.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:00.467 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:00.482 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 52257/209829 objects degraded (24.905%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:00.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 52257/209829 objects degraded (24.905%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:00.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:00.566 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:00.619 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:00.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:00.679 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:00.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 52257/209829 objects degraded (24.905%), 20 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:00.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:00.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:00.888 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:00.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:00.995 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:01.021 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:01.065 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:01.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:01.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:01.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:01.196 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:01.234 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:28:01.236 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:28:01.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:01.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:01.347 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:01.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[186706]: pgmap v835: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:01.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4189222674' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[180297]: pgmap v835: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:01.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4189222674' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:01 smithi152 ceph-mon[156819]: pgmap v835: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4189222674' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:01.684 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:28:02.233 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:02.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:02.338 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:02.370 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:28:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[186706]: osdmap e237: 8 total, 6 up, 6 in 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:28:02.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:02.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:02 smithi064 ceph-mon[180297]: osdmap e237: 8 total, 6 up, 6 in 2023-06-10T21:28:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: from='client.45971 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:28:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: failed to encode map e237 with expected crc 2023-06-10T21:28:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:02 smithi152 ceph-mon[156819]: osdmap e237: 8 total, 6 up, 6 in 2023-06-10T21:28:03.039 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:03.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:03.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:02] "GET /metrics HTTP/1.1" 200 224086 "" "Prometheus/2.33.4" 2023-06-10T21:28:03.171 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:03.234 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:03.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:03.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:03.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:03.370 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:03.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:03 smithi152 ceph-mon[156819]: pgmap v836: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:28:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[180297]: pgmap v836: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:28:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[186706]: pgmap v836: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:28:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:04.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:04.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:04.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:04.372 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":835,"stamp":"2023-06-10T21:27:59.626882+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4582741926,"num_objects":69943,"num_object_clones":0,"num_object_copies":209829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52257,"num_objects_misplaced":27639,"num_objects_unfound":0,"num_objects_dirty":69943,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":111850,"num_write_kb":5810639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16711,"num_bytes_recovered":1095048732,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104438,"ondisk_log_size":104438,"up":166,"acting":171,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14879028,"kb_used_data":12290500,"kb_used_omap":20,"kb_used_meta":2588459,"kb_avail":547591884,"statfs":{"total":575970213888,"available":560734089216,"internally_reserved":0,"allocated":12585472000,"data_stored":12581328730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20850,"internal_metadata":2650582670},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2204,"num_objects_misplaced":343,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2141,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004082"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":226,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.358306+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:27:45.358306+0000","last_peered":"2023-06-10T21:27:45.358306+0000","last_clean":"2023-06-10T21:27:45.358306+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:27:45.358306+0000","last_undegraded":"2023-06-10T21:27:45.358306+0000","last_fullsized":"2023-06-10T21:27:45.358306+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316671+0000","last_change":"2023-06-10T21:26:27.787856+0000","last_active":"2023-06-10T21:27:45.316671+0000","last_peered":"2023-06-10T21:27:45.316671+0000","last_clean":"2023-06-10T21:13:57.474600+0000","last_became_active":"2023-06-10T21:26:27.787856+0000","last_became_peered":"2023-06-10T21:26:27.787856+0000","last_unstale":"2023-06-10T21:27:45.316671+0000","last_undegraded":"2023-06-10T21:27:45.316671+0000","last_fullsized":"2023-06-10T21:26:27.776169+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:29:11.586416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":156,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316089+0000","last_change":"2023-06-10T21:26:27.787641+0000","last_active":"2023-06-10T21:27:45.316089+0000","last_peered":"2023-06-10T21:27:45.316089+0000","last_clean":"2023-06-10T21:19:00.485389+0000","last_became_active":"2023-06-10T21:26:27.787641+0000","last_became_peered":"2023-06-10T21:26:27.787641+0000","last_unstale":"2023-06-10T21:27:45.316089+0000","last_undegraded":"2023-06-10T21:27:45.316089+0000","last_fullsized":"2023-06-10T21:26:27.775244+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:39:35.438222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.325415+0000","last_change":"2023-06-10T21:27:42.309768+0000","last_active":"2023-06-10T21:26:26.807370+0000","last_peered":"2023-06-10T21:27:45.325415+0000","last_clean":"2023-06-10T21:26:20.391953+0000","last_became_active":"2023-06-10T21:24:45.151049+0000","last_became_peered":"2023-06-10T21:27:42.309768+0000","last_unstale":"2023-06-10T21:27:45.325415+0000","last_undegraded":"2023-06-10T21:27:45.325415+0000","last_fullsized":"2023-06-10T21:27:42.303279+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":172,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:46.208176+0000","last_change":"2023-06-10T21:27:42.302944+0000","last_active":"2023-06-10T21:24:44.126604+0000","last_peered":"2023-06-10T21:27:46.208176+0000","last_clean":"2023-06-10T21:24:36.651687+0000","last_became_active":"2023-06-10T21:24:12.314142+0000","last_became_peered":"2023-06-10T21:27:42.302944+0000","last_unstale":"2023-06-10T21:27:46.208176+0000","last_undegraded":"2023-06-10T21:27:46.208176+0000","last_fullsized":"2023-06-10T21:27:42.300084+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:23.035575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:46.208760+0000","last_change":"2023-06-10T21:27:42.302525+0000","last_active":"2023-06-10T21:26:26.802151+0000","last_peered":"2023-06-10T21:27:46.208760+0000","last_clean":"2023-06-10T21:26:21.435727+0000","last_became_active":"2023-06-10T21:26:15.751998+0000","last_became_peered":"2023-06-10T21:27:42.302525+0000","last_unstale":"2023-06-10T21:27:46.208760+0000","last_undegraded":"2023-06-10T21:27:46.208760+0000","last_fullsized":"2023-06-10T21:27:42.301752+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:18:04.488351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":87,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:46.208735+0000","last_change":"2023-06-10T21:27:42.302190+0000","last_active":"2023-06-10T21:26:26.801428+0000","last_peered":"2023-06-10T21:27:46.208735+0000","last_clean":"2023-06-10T21:26:20.392386+0000","last_became_active":"2023-06-10T21:26:15.751324+0000","last_became_peered":"2023-06-10T21:27:42.302190+0000","last_unstale":"2023-06-10T21:27:46.208735+0000","last_undegraded":"2023-06-10T21:27:46.208735+0000","last_fullsized":"2023-06-10T21:27:42.301588+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:04.232802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":200,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316321+0000","last_change":"2023-06-10T21:26:27.792890+0000","last_active":"2023-06-10T21:27:45.316321+0000","last_peered":"2023-06-10T21:27:45.316321+0000","last_clean":"2023-06-10T21:26:05.730925+0000","last_became_active":"2023-06-10T21:26:27.792890+0000","last_became_peered":"2023-06-10T21:26:27.792890+0000","last_unstale":"2023-06-10T21:27:45.316321+0000","last_undegraded":"2023-06-10T21:27:45.316321+0000","last_fullsized":"2023-06-10T21:26:27.777122+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:51:54.472567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.325082+0000","last_change":"2023-06-10T21:26:27.786701+0000","last_active":"2023-06-10T21:27:45.325082+0000","last_peered":"2023-06-10T21:27:45.325082+0000","last_clean":"2023-06-10T21:26:20.562068+0000","last_became_active":"2023-06-10T21:26:27.786701+0000","last_became_peered":"2023-06-10T21:26:27.786701+0000","last_unstale":"2023-06-10T21:27:45.325082+0000","last_undegraded":"2023-06-10T21:27:45.325082+0000","last_fullsized":"2023-06-10T21:26:27.783028+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":184,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.318988+0000","last_change":"2023-06-10T21:27:42.313721+0000","last_active":"2023-06-10T21:27:41.324134+0000","last_peered":"2023-06-10T21:27:45.318988+0000","last_clean":"2023-06-10T21:26:05.731138+0000","last_became_active":"2023-06-10T21:26:27.786324+0000","last_became_peered":"2023-06-10T21:27:42.313721+0000","last_unstale":"2023-06-10T21:27:45.318988+0000","last_undegraded":"2023-06-10T21:27:45.318988+0000","last_fullsized":"2023-06-10T21:27:42.304448+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":212,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.319598+0000","last_change":"2023-06-10T21:26:27.785616+0000","last_active":"2023-06-10T21:27:45.319598+0000","last_peered":"2023-06-10T21:27:45.319598+0000","last_clean":"2023-06-10T21:26:20.562388+0000","last_became_active":"2023-06-10T21:26:27.785616+0000","last_became_peered":"2023-06-10T21:26:27.785616+0000","last_unstale":"2023-06-10T21:27:45.319598+0000","last_undegraded":"2023-06-10T21:26:27.781808+0000","last_fullsized":"2023-06-10T21:26:27.781691+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":186,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358136+0000","last_change":"2023-06-10T21:27:42.311034+0000","last_active":"2023-06-10T21:27:45.358136+0000","last_peered":"2023-06-10T21:27:45.358136+0000","last_clean":"2023-06-10T21:27:39.262407+0000","last_became_active":"2023-06-10T21:27:42.311034+0000","last_became_peered":"2023-06-10T21:27:42.311034+0000","last_unstale":"2023-06-10T21:27:45.358136+0000","last_undegraded":"2023-06-10T21:27:45.358136+0000","last_fullsized":"2023-06-10T21:27:42.302548+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:23:58.671971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.318077+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:27:45.318077+0000","last_peered":"2023-06-10T21:27:45.318077+0000","last_clean":"2023-06-10T21:27:45.318077+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:27:45.318077+0000","last_undegraded":"2023-06-10T21:27:45.318077+0000","last_fullsized":"2023-06-10T21:27:45.318077+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":33,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.358225+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:27:45.358225+0000","last_peered":"2023-06-10T21:27:45.358225+0000","last_clean":"2023-06-10T21:27:45.358225+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:27:45.358225+0000","last_undegraded":"2023-06-10T21:27:45.358225+0000","last_fullsized":"2023-06-10T21:27:45.358225+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":235,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.316670+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:27:45.316670+0000","last_peered":"2023-06-10T21:27:45.316670+0000","last_clean":"2023-06-10T21:27:45.316670+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:27:45.316670+0000","last_undegraded":"2023-06-10T21:27:45.316670+0000","last_fullsized":"2023-06-10T21:27:45.316670+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317945+0000","last_change":"2023-06-10T21:26:27.787457+0000","last_active":"2023-06-10T21:27:45.317945+0000","last_peered":"2023-06-10T21:27:45.317945+0000","last_clean":"2023-06-10T21:13:57.475185+0000","last_became_active":"2023-06-10T21:26:27.787457+0000","last_became_peered":"2023-06-10T21:26:27.787457+0000","last_unstale":"2023-06-10T21:27:45.317945+0000","last_undegraded":"2023-06-10T21:27:45.317945+0000","last_fullsized":"2023-06-10T21:26:27.781392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.324100+0000","last_change":"2023-06-10T21:27:42.303731+0000","last_active":"2023-06-10T21:26:26.796042+0000","last_peered":"2023-06-10T21:27:45.324100+0000","last_clean":"2023-06-10T21:26:20.391274+0000","last_became_active":"2023-06-10T21:24:45.151097+0000","last_became_peered":"2023-06-10T21:27:42.303731+0000","last_unstale":"2023-06-10T21:27:45.324100+0000","last_undegraded":"2023-06-10T21:27:45.324100+0000","last_fullsized":"2023-06-10T21:27:42.302768+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":149,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.319856+0000","last_change":"2023-06-10T21:27:42.309645+0000","last_active":"2023-06-10T21:27:41.319920+0000","last_peered":"2023-06-10T21:27:45.319856+0000","last_clean":"2023-06-10T21:26:20.562490+0000","last_became_active":"2023-06-10T21:26:27.786424+0000","last_became_peered":"2023-06-10T21:27:42.309645+0000","last_unstale":"2023-06-10T21:27:45.319856+0000","last_undegraded":"2023-06-10T21:27:45.319856+0000","last_fullsized":"2023-06-10T21:27:42.303803+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.318855+0000","last_change":"2023-06-10T21:27:42.313502+0000","last_active":"2023-06-10T21:27:45.318855+0000","last_peered":"2023-06-10T21:27:45.318855+0000","last_clean":"2023-06-10T21:27:39.268609+0000","last_became_active":"2023-06-10T21:27:42.313502+0000","last_became_peered":"2023-06-10T21:27:42.313502+0000","last_unstale":"2023-06-10T21:27:45.318855+0000","last_undegraded":"2023-06-10T21:27:45.318855+0000","last_fullsized":"2023-06-10T21:27:42.304166+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":84,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.325305+0000","last_change":"2023-06-10T21:27:42.305600+0000","last_active":"2023-06-10T21:26:26.793394+0000","last_peered":"2023-06-10T21:27:45.325305+0000","last_clean":"2023-06-10T21:26:20.391891+0000","last_became_active":"2023-06-10T21:24:45.150815+0000","last_became_peered":"2023-06-10T21:27:42.305600+0000","last_unstale":"2023-06-10T21:27:45.325305+0000","last_undegraded":"2023-06-10T21:27:45.325305+0000","last_fullsized":"2023-06-10T21:27:42.303427+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":180,"reported_epoch":235,"state":"undersized+degraded+peered","last_fresh":"2023-06-10T21:27:45.357997+0000","last_change":"2023-06-10T21:27:42.314956+0000","last_active":"2023-06-10T21:24:44.152581+0000","last_peered":"2023-06-10T21:27:45.357997+0000","last_clean":"2023-06-10T21:24:36.632232+0000","last_became_active":"2023-06-10T21:24:12.316581+0000","last_became_peered":"2023-06-10T21:27:42.314956+0000","last_unstale":"2023-06-10T21:27:45.357997+0000","last_undegraded":"2023-06-10T21:27:42.302683+0000","last_fullsized":"2023-06-10T21:27:42.302609+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:07.689994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[{"shards":"2","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317079+0000","last_change":"2023-06-10T21:26:27.790352+0000","last_active":"2023-06-10T21:27:45.317079+0000","last_peered":"2023-06-10T21:27:45.317079+0000","last_clean":"2023-06-10T21:13:57.474737+0000","last_became_active":"2023-06-10T21:26:27.790352+0000","last_became_peered":"2023-06-10T21:26:27.790352+0000","last_unstale":"2023-06-10T21:27:45.317079+0000","last_undegraded":"2023-06-10T21:27:45.317079+0000","last_fullsized":"2023-06-10T21:26:27.782802+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"233'7020","reported_seq":8736,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.314086+0000","last_change":"2023-06-10T21:27:42.334849+0000","last_active":"2023-06-10T21:27:45.314086+0000","last_peered":"2023-06-10T21:27:45.314086+0000","last_clean":"2023-06-10T21:27:41.288015+0000","last_became_active":"2023-06-10T21:27:42.334849+0000","last_became_peered":"2023-06-10T21:27:42.334849+0000","last_unstale":"2023-06-10T21:27:45.314086+0000","last_undegraded":"2023-06-10T21:27:42.305438+0000","last_fullsized":"2023-06-10T21:27:42.305231+0000","mapping_epoch":232,"log_start":"203'1128","ondisk_log_start":"203'1128","created":194,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284295168,"num_objects":4338,"num_object_clones":0,"num_object_copies":13014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7020,"num_write_kb":363456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":822,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":4338}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":236,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.317348+0000","last_change":"2023-06-10T21:27:42.316143+0000","last_active":"2023-06-10T21:27:45.317348+0000","last_peered":"2023-06-10T21:27:45.317348+0000","last_clean":"2023-06-10T21:27:39.269594+0000","last_became_active":"2023-06-10T21:27:42.316143+0000","last_became_peered":"2023-06-10T21:27:42.316143+0000","last_unstale":"2023-06-10T21:27:45.317348+0000","last_undegraded":"2023-06-10T21:27:42.306749+0000","last_fullsized":"2023-06-10T21:27:42.306637+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317052+0000","last_change":"2023-06-10T21:26:27.789518+0000","last_active":"2023-06-10T21:27:45.317052+0000","last_peered":"2023-06-10T21:27:45.317052+0000","last_clean":"2023-06-10T21:26:20.562380+0000","last_became_active":"2023-06-10T21:26:27.789518+0000","last_became_peered":"2023-06-10T21:26:27.789518+0000","last_unstale":"2023-06-10T21:27:45.317052+0000","last_undegraded":"2023-06-10T21:27:45.317052+0000","last_fullsized":"2023-06-10T21:26:27.781860+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"233'7004","reported_seq":15483,"reported_epoch":236,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:27:54.150769+0000","last_change":"2023-06-10T21:27:54.150769+0000","last_active":"2023-06-10T21:27:54.150769+0000","last_peered":"2023-06-10T21:27:54.150769+0000","last_clean":"2023-06-10T21:27:54.150769+0000","last_became_active":"2023-06-10T21:26:28.952008+0000","last_became_peered":"2023-06-10T21:26:28.952008+0000","last_unstale":"2023-06-10T21:27:54.150769+0000","last_undegraded":"2023-06-10T21:27:54.150769+0000","last_fullsized":"2023-06-10T21:27:54.150769+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:07:08.784247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4416,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7004,"num_write_kb":365440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4197,"num_bytes_recovered":275054592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4416}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":205,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.325222+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:27:45.325222+0000","last_peered":"2023-06-10T21:27:45.325222+0000","last_clean":"2023-06-10T21:27:45.325222+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:27:45.325222+0000","last_undegraded":"2023-06-10T21:27:45.325222+0000","last_fullsized":"2023-06-10T21:27:45.325222+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":214,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.313927+0000","last_change":"2023-06-10T21:26:27.790530+0000","last_active":"2023-06-10T21:27:45.313927+0000","last_peered":"2023-06-10T21:27:45.313927+0000","last_clean":"2023-06-10T21:19:00.489965+0000","last_became_active":"2023-06-10T21:26:27.790530+0000","last_became_peered":"2023-06-10T21:26:27.790530+0000","last_unstale":"2023-06-10T21:27:45.313927+0000","last_undegraded":"2023-06-10T21:27:45.313927+0000","last_fullsized":"2023-06-10T21:26:27.780661+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"231'7044","reported_seq":7105,"reported_epoch":235,"state":"undersized+degraded+peered","last_fresh":"2023-06-10T21:27:45.325683+0000","last_change":"2023-06-10T21:27:42.309680+0000","last_active":"2023-06-10T21:27:41.305749+0000","last_peered":"2023-06-10T21:27:45.325683+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:26:27.785031+0000","last_became_peered":"2023-06-10T21:27:42.309680+0000","last_unstale":"2023-06-10T21:27:45.325683+0000","last_undegraded":"2023-06-10T21:27:42.307144+0000","last_fullsized":"2023-06-10T21:27:42.307054+0000","mapping_epoch":232,"log_start":"203'900","ondisk_log_start":"203'900","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6144,"log_dups_size":0,"ondisk_log_size":6144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286916608,"num_objects":4378,"num_object_clones":0,"num_object_copies":13134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8756,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7044,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"4","objects":4378}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":221,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.319313+0000","last_change":"2023-06-10T21:26:27.785962+0000","last_active":"2023-06-10T21:27:45.319313+0000","last_peered":"2023-06-10T21:27:45.319313+0000","last_clean":"2023-06-10T21:26:20.562273+0000","last_became_active":"2023-06-10T21:26:27.785962+0000","last_became_peered":"2023-06-10T21:26:27.785962+0000","last_unstale":"2023-06-10T21:27:45.319313+0000","last_undegraded":"2023-06-10T21:26:27.782163+0000","last_fullsized":"2023-06-10T21:26:27.782058+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":103,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.324236+0000","last_change":"2023-06-10T21:27:42.316384+0000","last_active":"2023-06-10T21:27:45.324236+0000","last_peered":"2023-06-10T21:27:45.324236+0000","last_clean":"2023-06-10T21:27:39.268583+0000","last_became_active":"2023-06-10T21:27:42.316384+0000","last_became_peered":"2023-06-10T21:27:42.316384+0000","last_unstale":"2023-06-10T21:27:45.324236+0000","last_undegraded":"2023-06-10T21:27:45.324236+0000","last_fullsized":"2023-06-10T21:27:42.302432+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"231'6930","reported_seq":6992,"reported_epoch":235,"state":"undersized+degraded+peered","last_fresh":"2023-06-10T21:27:45.358342+0000","last_change":"2023-06-10T21:27:42.304780+0000","last_active":"2023-06-10T21:27:41.304314+0000","last_peered":"2023-06-10T21:27:45.358342+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:26:27.813144+0000","last_became_peered":"2023-06-10T21:27:42.304780+0000","last_unstale":"2023-06-10T21:27:45.358342+0000","last_undegraded":"2023-06-10T21:27:42.304004+0000","last_fullsized":"2023-06-10T21:27:42.303929+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6930,"log_dups_size":0,"ondisk_log_size":6930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:58:42.745075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283901952,"num_objects":4332,"num_object_clones":0,"num_object_copies":12996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8664,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6930,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[{"shards":"2","objects":4332}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":121,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:46.208637+0000","last_change":"2023-06-10T21:27:42.302207+0000","last_active":"2023-06-10T21:26:15.583040+0000","last_peered":"2023-06-10T21:27:46.208637+0000","last_clean":"2023-06-10T21:26:05.731822+0000","last_became_active":"2023-06-10T21:25:45.627031+0000","last_became_peered":"2023-06-10T21:27:42.302207+0000","last_unstale":"2023-06-10T21:27:46.208637+0000","last_undegraded":"2023-06-10T21:27:46.208637+0000","last_fullsized":"2023-06-10T21:27:42.300299+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:37:30.159295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:46.208605+0000","last_change":"2023-06-10T21:27:42.320405+0000","last_active":"2023-06-10T21:27:46.208605+0000","last_peered":"2023-06-10T21:27:46.208605+0000","last_clean":"2023-06-10T21:13:58.308459+0000","last_became_active":"2023-06-10T21:27:42.320405+0000","last_became_peered":"2023-06-10T21:27:42.320405+0000","last_unstale":"2023-06-10T21:27:46.208605+0000","last_undegraded":"2023-06-10T21:27:46.208605+0000","last_fullsized":"2023-06-10T21:27:42.302034+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:44:23.700572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"233'7124","reported_seq":7187,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:46.208429+0000","last_change":"2023-06-10T21:27:42.304807+0000","last_active":"2023-06-10T21:27:46.208429+0000","last_peered":"2023-06-10T21:27:46.208429+0000","last_clean":"2023-06-10T21:27:41.271507+0000","last_became_active":"2023-06-10T21:27:42.304807+0000","last_became_peered":"2023-06-10T21:27:42.304807+0000","last_unstale":"2023-06-10T21:27:46.208429+0000","last_undegraded":"2023-06-10T21:27:42.299853+0000","last_fullsized":"2023-06-10T21:27:42.299750+0000","mapping_epoch":232,"log_start":"203'900","ondisk_log_start":"203'900","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6224,"log_dups_size":0,"ondisk_log_size":6224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:50:57.992685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4396,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7124,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":4396}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":22,"reported_epoch":235,"state":"undersized+degraded+peered","last_fresh":"2023-06-10T21:27:45.314615+0000","last_change":"2023-06-10T21:27:42.310245+0000","last_active":"2023-06-10T21:27:41.308146+0000","last_peered":"2023-06-10T21:27:45.314615+0000","last_clean":"2023-06-10T20:59:27.427105+0000","last_became_active":"2023-06-10T20:59:25.411214+0000","last_became_peered":"2023-06-10T21:27:42.310245+0000","last_unstale":"2023-06-10T21:27:45.314615+0000","last_undegraded":"2023-06-10T21:27:42.306819+0000","last_fullsized":"2023-06-10T21:27:42.306741+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[{"shards":"4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":199,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.314227+0000","last_change":"2023-06-10T21:26:27.790578+0000","last_active":"2023-06-10T21:27:45.314227+0000","last_peered":"2023-06-10T21:27:45.314227+0000","last_clean":"2023-06-10T21:26:05.731741+0000","last_became_active":"2023-06-10T21:26:27.790578+0000","last_became_peered":"2023-06-10T21:26:27.790578+0000","last_unstale":"2023-06-10T21:27:45.314227+0000","last_undegraded":"2023-06-10T21:27:45.314227+0000","last_fullsized":"2023-06-10T21:26:27.779863+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"233'6786","reported_seq":8515,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.358449+0000","last_change":"2023-06-10T21:26:27.820034+0000","last_active":"2023-06-10T21:27:45.358449+0000","last_peered":"2023-06-10T21:27:45.358449+0000","last_clean":"2023-06-10T21:26:26.761025+0000","last_became_active":"2023-06-10T21:26:27.820034+0000","last_became_peered":"2023-06-10T21:26:27.820034+0000","last_unstale":"2023-06-10T21:27:45.358449+0000","last_undegraded":"2023-06-10T21:26:27.817850+0000","last_fullsized":"2023-06-10T21:26:27.812347+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6786,"log_dups_size":0,"ondisk_log_size":6786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:28:47.168875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6786,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54198272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":4270}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":89,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.314524+0000","last_change":"2023-06-10T21:26:27.785142+0000","last_active":"2023-06-10T21:27:45.314524+0000","last_peered":"2023-06-10T21:27:45.314524+0000","last_clean":"2023-06-10T21:13:57.477830+0000","last_became_active":"2023-06-10T21:26:27.785142+0000","last_became_peered":"2023-06-10T21:26:27.785142+0000","last_unstale":"2023-06-10T21:27:45.314524+0000","last_undegraded":"2023-06-10T21:27:45.314524+0000","last_fullsized":"2023-06-10T21:26:27.782073+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":224,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.358618+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:27:45.358618+0000","last_peered":"2023-06-10T21:27:45.358618+0000","last_clean":"2023-06-10T21:27:45.358618+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:27:45.358618+0000","last_undegraded":"2023-06-10T21:27:45.358618+0000","last_fullsized":"2023-06-10T21:27:45.358618+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"233'7008","reported_seq":12406,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.370080+0000","last_change":"2023-06-10T21:27:45.367487+0000","last_active":"2023-06-10T21:27:45.370080+0000","last_peered":"2023-06-10T21:27:45.370080+0000","last_clean":"2023-06-10T21:27:45.370080+0000","last_became_active":"2023-06-10T21:27:45.353438+0000","last_became_peered":"2023-06-10T21:27:45.353438+0000","last_unstale":"2023-06-10T21:27:45.370080+0000","last_undegraded":"2023-06-10T21:27:45.370080+0000","last_fullsized":"2023-06-10T21:27:45.370080+0000","mapping_epoch":234,"log_start":"200'700","ondisk_log_start":"200'700","created":194,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7008,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3133,"num_bytes_recovered":205324288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":210,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.358038+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:27:45.358038+0000","last_peered":"2023-06-10T21:27:45.358038+0000","last_clean":"2023-06-10T21:27:45.358038+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:27:45.358038+0000","last_undegraded":"2023-06-10T21:27:45.358038+0000","last_fullsized":"2023-06-10T21:27:45.358038+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.314382+0000","last_change":"2023-06-10T21:26:27.784827+0000","last_active":"2023-06-10T21:27:45.314382+0000","last_peered":"2023-06-10T21:27:45.314382+0000","last_clean":"2023-06-10T21:13:57.670486+0000","last_became_active":"2023-06-10T21:26:27.784827+0000","last_became_peered":"2023-06-10T21:26:27.784827+0000","last_unstale":"2023-06-10T21:27:45.314382+0000","last_undegraded":"2023-06-10T21:27:45.314382+0000","last_fullsized":"2023-06-10T21:26:27.780178+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"233'6974","reported_seq":7037,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.318580+0000","last_change":"2023-06-10T21:27:42.313119+0000","last_active":"2023-06-10T21:27:45.318580+0000","last_peered":"2023-06-10T21:27:45.318580+0000","last_clean":"2023-06-10T21:27:41.268268+0000","last_became_active":"2023-06-10T21:27:42.313119+0000","last_became_peered":"2023-06-10T21:27:42.313119+0000","last_unstale":"2023-06-10T21:27:45.318580+0000","last_undegraded":"2023-06-10T21:27:42.303848+0000","last_fullsized":"2023-06-10T21:27:42.303732+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6974,"log_dups_size":0,"ondisk_log_size":6974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285999104,"num_objects":4364,"num_object_clones":0,"num_object_copies":13092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4364,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6974,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4364}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317178+0000","last_change":"2023-06-10T21:26:27.788939+0000","last_active":"2023-06-10T21:27:45.317178+0000","last_peered":"2023-06-10T21:27:45.317178+0000","last_clean":"2023-06-10T21:13:57.474197+0000","last_became_active":"2023-06-10T21:26:27.788939+0000","last_became_peered":"2023-06-10T21:26:27.788939+0000","last_unstale":"2023-06-10T21:27:45.317178+0000","last_undegraded":"2023-06-10T21:27:45.317178+0000","last_fullsized":"2023-06-10T21:26:27.781734+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":126,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":157,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316824+0000","last_change":"2023-06-10T21:26:27.784584+0000","last_active":"2023-06-10T21:27:45.316824+0000","last_peered":"2023-06-10T21:27:45.316824+0000","last_clean":"2023-06-10T21:19:10.700014+0000","last_became_active":"2023-06-10T21:26:27.784584+0000","last_became_peered":"2023-06-10T21:26:27.784584+0000","last_unstale":"2023-06-10T21:27:45.316824+0000","last_undegraded":"2023-06-10T21:27:45.316824+0000","last_fullsized":"2023-06-10T21:26:27.776138+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:41:43.266239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"233'6989","reported_seq":7045,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.317700+0000","last_change":"2023-06-10T21:27:42.308142+0000","last_active":"2023-06-10T21:27:45.317700+0000","last_peered":"2023-06-10T21:27:45.317700+0000","last_clean":"2023-06-10T21:27:41.281666+0000","last_became_active":"2023-06-10T21:27:42.308142+0000","last_became_peered":"2023-06-10T21:27:42.308142+0000","last_unstale":"2023-06-10T21:27:45.317700+0000","last_undegraded":"2023-06-10T21:27:42.302619+0000","last_fullsized":"2023-06-10T21:27:42.302503+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290521088,"num_objects":4433,"num_object_clones":0,"num_object_copies":13299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6989,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":4433}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":96,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317829+0000","last_change":"2023-06-10T21:27:42.325643+0000","last_active":"2023-06-10T21:27:45.317829+0000","last_peered":"2023-06-10T21:27:45.317829+0000","last_clean":"2023-06-10T21:09:15.147600+0000","last_became_active":"2023-06-10T21:27:42.325643+0000","last_became_peered":"2023-06-10T21:27:42.325643+0000","last_unstale":"2023-06-10T21:27:45.317829+0000","last_undegraded":"2023-06-10T21:27:45.317829+0000","last_fullsized":"2023-06-10T21:27:42.302582+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358501+0000","last_change":"2023-06-10T21:26:27.820466+0000","last_active":"2023-06-10T21:27:45.358501+0000","last_peered":"2023-06-10T21:27:45.358501+0000","last_clean":"2023-06-10T21:26:05.731923+0000","last_became_active":"2023-06-10T21:26:27.820466+0000","last_became_peered":"2023-06-10T21:26:27.820466+0000","last_unstale":"2023-06-10T21:27:45.358501+0000","last_undegraded":"2023-06-10T21:27:45.358501+0000","last_fullsized":"2023-06-10T21:26:27.812254+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:20:50.370098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:46.208246+0000","last_change":"2023-06-10T21:27:42.304439+0000","last_active":"2023-06-10T21:27:46.208246+0000","last_peered":"2023-06-10T21:27:46.208246+0000","last_clean":"2023-06-10T20:59:27.427609+0000","last_became_active":"2023-06-10T21:27:42.304439+0000","last_became_peered":"2023-06-10T21:27:42.304439+0000","last_unstale":"2023-06-10T21:27:46.208246+0000","last_undegraded":"2023-06-10T21:27:46.208246+0000","last_fullsized":"2023-06-10T21:27:42.299342+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:19.876073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"233'6985","reported_seq":9268,"reported_epoch":235,"state":"active+undersized+remapped","last_fresh":"2023-06-10T21:27:45.358109+0000","last_change":"2023-06-10T21:27:42.306954+0000","last_active":"2023-06-10T21:27:45.358109+0000","last_peered":"2023-06-10T21:27:45.358109+0000","last_clean":"2023-06-10T21:27:41.254432+0000","last_became_active":"2023-06-10T21:27:42.306954+0000","last_became_peered":"2023-06-10T21:27:42.306954+0000","last_unstale":"2023-06-10T21:27:45.358109+0000","last_undegraded":"2023-06-10T21:27:45.358109+0000","last_fullsized":"2023-06-10T21:27:42.302899+0000","mapping_epoch":232,"log_start":"200'800","ondisk_log_start":"200'800","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:57:47.151958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285540352,"num_objects":4357,"num_object_clones":0,"num_object_copies":13071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8714,"num_objects_unfound":0,"num_objects_dirty":4357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6985,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1096,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":4357}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":100,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:46.208340+0000","last_change":"2023-06-10T21:27:42.300114+0000","last_active":"2023-06-10T21:26:26.814921+0000","last_peered":"2023-06-10T21:27:46.208340+0000","last_clean":"2023-06-10T21:26:21.407543+0000","last_became_active":"2023-06-10T21:26:15.747276+0000","last_became_peered":"2023-06-10T21:27:42.300114+0000","last_unstale":"2023-06-10T21:27:46.208340+0000","last_undegraded":"2023-06-10T21:27:46.208340+0000","last_fullsized":"2023-06-10T21:27:42.299612+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:33.451542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":205,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.314268+0000","last_change":"2023-06-10T21:27:42.320570+0000","last_active":"2023-06-10T21:27:45.314268+0000","last_peered":"2023-06-10T21:27:45.314268+0000","last_clean":"2023-06-10T21:26:05.729089+0000","last_became_active":"2023-06-10T21:27:42.320570+0000","last_became_peered":"2023-06-10T21:27:42.320570+0000","last_unstale":"2023-06-10T21:27:45.314268+0000","last_undegraded":"2023-06-10T21:27:45.314268+0000","last_fullsized":"2023-06-10T21:27:42.304971+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":57,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358660+0000","last_change":"2023-06-10T21:26:27.812850+0000","last_active":"2023-06-10T21:27:45.358660+0000","last_peered":"2023-06-10T21:27:45.358660+0000","last_clean":"2023-06-10T21:26:20.308366+0000","last_became_active":"2023-06-10T21:26:27.812850+0000","last_became_peered":"2023-06-10T21:26:27.812850+0000","last_unstale":"2023-06-10T21:27:45.358660+0000","last_undegraded":"2023-06-10T21:27:45.358660+0000","last_fullsized":"2023-06-10T21:26:27.783047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:47:49.320329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"233'6901","reported_seq":8660,"reported_epoch":236,"state":"active+clean","last_fresh":"2023-06-10T21:27:50.145451+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:27:50.145451+0000","last_peered":"2023-06-10T21:27:50.145451+0000","last_clean":"2023-06-10T21:27:50.145451+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:27:50.145451+0000","last_undegraded":"2023-06-10T21:27:50.145451+0000","last_fullsized":"2023-06-10T21:27:50.145451+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6901,"log_dups_size":0,"ondisk_log_size":6901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283705344,"num_objects":4329,"num_object_clones":0,"num_object_copies":12987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6901,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":62,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.358284+0000","last_change":"2023-06-10T21:27:42.304922+0000","last_active":"2023-06-10T21:27:41.309461+0000","last_peered":"2023-06-10T21:27:45.358284+0000","last_clean":"2023-06-10T21:26:20.307980+0000","last_became_active":"2023-06-10T21:26:27.818621+0000","last_became_peered":"2023-06-10T21:27:42.304922+0000","last_unstale":"2023-06-10T21:27:45.358284+0000","last_undegraded":"2023-06-10T21:27:45.358284+0000","last_fullsized":"2023-06-10T21:27:42.303524+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:42.678679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":66,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358168+0000","last_change":"2023-06-10T21:27:42.315035+0000","last_active":"2023-06-10T21:27:45.358168+0000","last_peered":"2023-06-10T21:27:45.358168+0000","last_clean":"2023-06-10T21:27:39.262443+0000","last_became_active":"2023-06-10T21:27:42.315035+0000","last_became_peered":"2023-06-10T21:27:42.315035+0000","last_unstale":"2023-06-10T21:27:45.358168+0000","last_undegraded":"2023-06-10T21:27:45.358168+0000","last_fullsized":"2023-06-10T21:27:42.303622+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:01.998754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316797+0000","last_change":"2023-06-10T21:27:42.310407+0000","last_active":"2023-06-10T21:27:45.316797+0000","last_peered":"2023-06-10T21:27:45.316797+0000","last_clean":"2023-06-10T21:24:36.627027+0000","last_became_active":"2023-06-10T21:27:42.310407+0000","last_became_peered":"2023-06-10T21:27:42.310407+0000","last_unstale":"2023-06-10T21:27:45.316797+0000","last_undegraded":"2023-06-10T21:27:45.316797+0000","last_fullsized":"2023-06-10T21:27:42.302946+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.b","version":"233'6948","reported_seq":13319,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:46.208550+0000","last_change":"2023-06-10T21:27:42.320410+0000","last_active":"2023-06-10T21:27:46.208550+0000","last_peered":"2023-06-10T21:27:46.208550+0000","last_clean":"2023-06-10T21:27:41.281077+0000","last_became_active":"2023-06-10T21:27:42.320410+0000","last_became_peered":"2023-06-10T21:27:42.320410+0000","last_unstale":"2023-06-10T21:27:46.208550+0000","last_undegraded":"2023-06-10T21:27:42.301904+0000","last_fullsized":"2023-06-10T21:27:42.301836+0000","mapping_epoch":232,"log_start":"200'800","ondisk_log_start":"200'800","created":194,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:34:24.105093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286785536,"num_objects":4376,"num_object_clones":0,"num_object_copies":13128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6948,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3150,"num_bytes_recovered":206438400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":4376}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":230,"reported_epoch":236,"state":"active+undersized","last_fresh":"2023-06-10T21:27:50.146045+0000","last_change":"2023-06-10T21:26:27.819291+0000","last_active":"2023-06-10T21:27:50.146045+0000","last_peered":"2023-06-10T21:27:50.146045+0000","last_clean":"2023-06-10T21:26:20.308048+0000","last_became_active":"2023-06-10T21:26:27.819291+0000","last_became_peered":"2023-06-10T21:26:27.819291+0000","last_unstale":"2023-06-10T21:27:50.146045+0000","last_undegraded":"2023-06-10T21:27:50.146045+0000","last_fullsized":"2023-06-10T21:26:27.818075+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:00:43.956182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":514,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.316903+0000","last_change":"2023-06-10T21:27:42.314354+0000","last_active":"2023-06-10T21:27:45.316903+0000","last_peered":"2023-06-10T21:27:45.316903+0000","last_clean":"2023-06-10T21:27:39.268542+0000","last_became_active":"2023-06-10T21:27:42.314354+0000","last_became_peered":"2023-06-10T21:27:42.314354+0000","last_unstale":"2023-06-10T21:27:45.316903+0000","last_undegraded":"2023-06-10T21:27:42.302346+0000","last_fullsized":"2023-06-10T21:27:42.302221+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":189,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.313862+0000","last_change":"2023-06-10T21:26:27.785005+0000","last_active":"2023-06-10T21:27:45.313862+0000","last_peered":"2023-06-10T21:27:45.313862+0000","last_clean":"2023-06-10T21:26:05.731398+0000","last_became_active":"2023-06-10T21:26:27.785005+0000","last_became_peered":"2023-06-10T21:26:27.785005+0000","last_unstale":"2023-06-10T21:27:45.313862+0000","last_undegraded":"2023-06-10T21:27:45.313862+0000","last_fullsized":"2023-06-10T21:26:27.779673+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":41,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.319736+0000","last_change":"2023-06-10T21:27:42.345642+0000","last_active":"2023-06-10T21:27:45.319736+0000","last_peered":"2023-06-10T21:27:45.319736+0000","last_clean":"2023-06-10T21:27:39.269550+0000","last_became_active":"2023-06-10T21:27:42.345642+0000","last_became_peered":"2023-06-10T21:27:42.345642+0000","last_unstale":"2023-06-10T21:27:45.319736+0000","last_undegraded":"2023-06-10T21:27:45.319736+0000","last_fullsized":"2023-06-10T21:27:42.303517+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"233'7143","reported_seq":7211,"reported_epoch":235,"state":"active+recovery_wait+undersized+degraded","last_fresh":"2023-06-10T21:27:45.358008+0000","last_change":"2023-06-10T21:27:42.330269+0000","last_active":"2023-06-10T21:27:45.358008+0000","last_peered":"2023-06-10T21:27:45.358008+0000","last_clean":"2023-06-10T21:27:41.295603+0000","last_became_active":"2023-06-10T21:27:42.317942+0000","last_became_peered":"2023-06-10T21:27:42.317942+0000","last_unstale":"2023-06-10T21:27:45.358008+0000","last_undegraded":"2023-06-10T21:27:42.303621+0000","last_fullsized":"2023-06-10T21:27:42.302899+0000","mapping_epoch":232,"log_start":"203'1000","ondisk_log_start":"203'1000","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6143,"log_dups_size":0,"ondisk_log_size":6143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:13:45.907289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285802496,"num_objects":4361,"num_object_clones":0,"num_object_copies":13083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4361,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7143,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":4361}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":246,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316774+0000","last_change":"2023-06-10T21:27:42.314762+0000","last_active":"2023-06-10T21:27:45.316774+0000","last_peered":"2023-06-10T21:27:45.316774+0000","last_clean":"2023-06-10T21:27:39.268448+0000","last_became_active":"2023-06-10T21:27:42.314762+0000","last_became_peered":"2023-06-10T21:27:42.314762+0000","last_unstale":"2023-06-10T21:27:45.316774+0000","last_undegraded":"2023-06-10T21:27:45.316774+0000","last_fullsized":"2023-06-10T21:27:42.302610+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.314139+0000","last_change":"2023-06-10T21:27:42.309154+0000","last_active":"2023-06-10T21:26:26.791033+0000","last_peered":"2023-06-10T21:27:45.314139+0000","last_clean":"2023-06-10T21:26:20.391145+0000","last_became_active":"2023-06-10T21:24:45.139551+0000","last_became_peered":"2023-06-10T21:27:42.309154+0000","last_unstale":"2023-06-10T21:27:45.314139+0000","last_undegraded":"2023-06-10T21:27:45.314139+0000","last_fullsized":"2023-06-10T21:27:42.305647+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":56,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358352+0000","last_change":"2023-06-10T21:27:42.315039+0000","last_active":"2023-06-10T21:27:45.358352+0000","last_peered":"2023-06-10T21:27:45.358352+0000","last_clean":"2023-06-10T21:25:28.175746+0000","last_became_active":"2023-06-10T21:27:42.315039+0000","last_became_peered":"2023-06-10T21:27:42.315039+0000","last_unstale":"2023-06-10T21:27:45.358352+0000","last_undegraded":"2023-06-10T21:27:45.358352+0000","last_fullsized":"2023-06-10T21:27:42.304408+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:35.796244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"233'7003","reported_seq":7578,"reported_epoch":235,"state":"active+undersized+remapped","last_fresh":"2023-06-10T21:27:45.314355+0000","last_change":"2023-06-10T21:27:42.307650+0000","last_active":"2023-06-10T21:27:45.314355+0000","last_peered":"2023-06-10T21:27:45.314355+0000","last_clean":"2023-06-10T21:27:41.266678+0000","last_became_active":"2023-06-10T21:27:42.307650+0000","last_became_peered":"2023-06-10T21:27:42.307650+0000","last_unstale":"2023-06-10T21:27:45.314355+0000","last_undegraded":"2023-06-10T21:27:45.314355+0000","last_fullsized":"2023-06-10T21:27:42.304970+0000","mapping_epoch":232,"log_start":"203'1118","ondisk_log_start":"203'1118","created":194,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291242008,"num_objects":4445,"num_object_clones":0,"num_object_copies":13335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8890,"num_objects_unfound":0,"num_objects_dirty":4445,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7003,"num_write_kb":366273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":251,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4445}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.319462+0000","last_change":"2023-06-10T21:27:42.345653+0000","last_active":"2023-06-10T21:27:45.319462+0000","last_peered":"2023-06-10T21:27:45.319462+0000","last_clean":"2023-06-10T21:27:39.269301+0000","last_became_active":"2023-06-10T21:27:42.345653+0000","last_became_peered":"2023-06-10T21:27:42.345653+0000","last_unstale":"2023-06-10T21:27:45.319462+0000","last_undegraded":"2023-06-10T21:27:45.319462+0000","last_fullsized":"2023-06-10T21:27:42.303194+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":66,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358530+0000","last_change":"2023-06-10T21:27:42.310782+0000","last_active":"2023-06-10T21:27:45.358530+0000","last_peered":"2023-06-10T21:27:45.358530+0000","last_clean":"2023-06-10T21:27:39.263504+0000","last_became_active":"2023-06-10T21:27:42.310782+0000","last_became_peered":"2023-06-10T21:27:42.310782+0000","last_unstale":"2023-06-10T21:27:45.358530+0000","last_undegraded":"2023-06-10T21:27:45.358530+0000","last_fullsized":"2023-06-10T21:27:42.303295+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:43:30.398711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":52,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316457+0000","last_change":"2023-06-10T21:27:42.321780+0000","last_active":"2023-06-10T21:27:45.316457+0000","last_peered":"2023-06-10T21:27:45.316457+0000","last_clean":"2023-06-10T21:25:40.344921+0000","last_became_active":"2023-06-10T21:27:42.321780+0000","last_became_peered":"2023-06-10T21:27:42.321780+0000","last_unstale":"2023-06-10T21:27:45.316457+0000","last_undegraded":"2023-06-10T21:27:45.316457+0000","last_fullsized":"2023-06-10T21:27:42.303141+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.e","version":"233'6913","reported_seq":7533,"reported_epoch":235,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:27:45.358505+0000","last_change":"2023-06-10T21:27:26.580630+0000","last_active":"2023-06-10T21:27:45.358505+0000","last_peered":"2023-06-10T21:27:45.358505+0000","last_clean":"2023-06-10T21:27:45.358505+0000","last_became_active":"2023-06-10T21:26:28.807447+0000","last_became_peered":"2023-06-10T21:26:28.807447+0000","last_unstale":"2023-06-10T21:27:45.358505+0000","last_undegraded":"2023-06-10T21:27:45.358505+0000","last_fullsized":"2023-06-10T21:27:45.358505+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:20.877709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287244288,"num_objects":4383,"num_object_clones":0,"num_object_copies":13149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4383,"num_objects_unfound":0,"num_objects_dirty":4383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6913,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":4383}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":50,"reported_epoch":235,"state":"undersized+degraded+peered","last_fresh":"2023-06-10T21:27:45.316936+0000","last_change":"2023-06-10T21:27:42.310787+0000","last_active":"2023-06-10T21:27:41.322100+0000","last_peered":"2023-06-10T21:27:45.316936+0000","last_clean":"2023-06-10T20:59:27.427422+0000","last_became_active":"2023-06-10T21:26:27.787375+0000","last_became_peered":"2023-06-10T21:27:42.310787+0000","last_unstale":"2023-06-10T21:27:45.316936+0000","last_undegraded":"2023-06-10T21:27:42.306037+0000","last_fullsized":"2023-06-10T21:27:42.305944+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":142,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.316833+0000","last_change":"2023-06-10T21:26:27.788119+0000","last_active":"2023-06-10T21:27:45.316833+0000","last_peered":"2023-06-10T21:27:45.316833+0000","last_clean":"2023-06-10T20:58:33.354602+0000","last_became_active":"2023-06-10T21:26:27.788119+0000","last_became_peered":"2023-06-10T21:26:27.788119+0000","last_unstale":"2023-06-10T21:27:45.316833+0000","last_undegraded":"2023-06-10T21:27:45.316833+0000","last_fullsized":"2023-06-10T21:26:27.776800+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:41.776787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":154,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.317291+0000","last_change":"2023-06-10T21:27:42.309578+0000","last_active":"2023-06-10T21:27:41.323287+0000","last_peered":"2023-06-10T21:27:45.317291+0000","last_clean":"2023-06-10T21:24:36.651653+0000","last_became_active":"2023-06-10T21:26:27.787854+0000","last_became_peered":"2023-06-10T21:27:42.309578+0000","last_unstale":"2023-06-10T21:27:45.317291+0000","last_undegraded":"2023-06-10T21:27:45.317291+0000","last_fullsized":"2023-06-10T21:27:42.303433+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"233'6817","reported_seq":11091,"reported_epoch":236,"state":"active+recovering+degraded+remapped","last_fresh":"2023-06-10T21:27:58.813958+0000","last_change":"2023-06-10T21:27:54.151227+0000","last_active":"2023-06-10T21:27:58.813958+0000","last_peered":"2023-06-10T21:27:58.813958+0000","last_clean":"2023-06-10T21:26:15.549220+0000","last_became_active":"2023-06-10T21:26:28.973820+0000","last_became_peered":"2023-06-10T21:26:28.973820+0000","last_unstale":"2023-06-10T21:27:58.813958+0000","last_undegraded":"2023-06-10T21:26:28.947847+0000","last_fullsized":"2023-06-10T21:27:58.813958+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6817,"log_dups_size":0,"ondisk_log_size":6817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:46.994093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4277,"num_objects_misplaced":1236,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6817,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2100,"num_bytes_recovered":137625600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,4,6","objects":3041},{"shards":"2,6","objects":1236}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":217,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317208+0000","last_change":"2023-06-10T21:26:27.818580+0000","last_active":"2023-06-10T21:27:45.317208+0000","last_peered":"2023-06-10T21:27:45.317208+0000","last_clean":"2023-06-10T21:26:20.562258+0000","last_became_active":"2023-06-10T21:26:27.818580+0000","last_became_peered":"2023-06-10T21:26:27.818580+0000","last_unstale":"2023-06-10T21:27:45.317208+0000","last_undegraded":"2023-06-10T21:27:45.317208+0000","last_fullsized":"2023-06-10T21:26:27.783435+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":32,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:46.208691+0000","last_change":"2023-06-10T21:27:42.320339+0000","last_active":"2023-06-10T21:27:46.208691+0000","last_peered":"2023-06-10T21:27:46.208691+0000","last_clean":"2023-06-10T20:59:28.808370+0000","last_became_active":"2023-06-10T21:27:42.320339+0000","last_became_peered":"2023-06-10T21:27:42.320339+0000","last_unstale":"2023-06-10T21:27:46.208691+0000","last_undegraded":"2023-06-10T21:27:42.300987+0000","last_fullsized":"2023-06-10T21:27:42.300877+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:14:53.764531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.318431+0000","last_change":"2023-06-10T21:27:42.313948+0000","last_active":"2023-06-10T21:27:45.318431+0000","last_peered":"2023-06-10T21:27:45.318431+0000","last_clean":"2023-06-10T21:25:40.220854+0000","last_became_active":"2023-06-10T21:27:42.313948+0000","last_became_peered":"2023-06-10T21:27:42.313948+0000","last_unstale":"2023-06-10T21:27:45.318431+0000","last_undegraded":"2023-06-10T21:27:45.318431+0000","last_fullsized":"2023-06-10T21:27:42.303041+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":12,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.317609+0000","last_change":"2023-06-10T21:27:42.313053+0000","last_active":"2023-06-10T21:27:45.317609+0000","last_peered":"2023-06-10T21:27:45.317609+0000","last_clean":"2023-06-10T20:59:24.384447+0000","last_became_active":"2023-06-10T21:27:42.313053+0000","last_became_peered":"2023-06-10T21:27:42.313053+0000","last_unstale":"2023-06-10T21:27:45.317609+0000","last_undegraded":"2023-06-10T21:27:45.317609+0000","last_fullsized":"2023-06-10T21:27:42.304644+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":65,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358256+0000","last_change":"2023-06-10T21:26:27.817806+0000","last_active":"2023-06-10T21:27:45.358256+0000","last_peered":"2023-06-10T21:27:45.358256+0000","last_clean":"2023-06-10T21:26:20.308187+0000","last_became_active":"2023-06-10T21:26:27.817806+0000","last_became_peered":"2023-06-10T21:26:27.817806+0000","last_unstale":"2023-06-10T21:27:45.358256+0000","last_undegraded":"2023-06-10T21:27:45.358256+0000","last_fullsized":"2023-06-10T21:26:27.783382+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:28:39.270974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":36,"reported_epoch":235,"state":"active+clean","last_fresh":"2023-06-10T21:27:45.317607+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:27:45.317607+0000","last_peered":"2023-06-10T21:27:45.317607+0000","last_clean":"2023-06-10T21:27:45.317607+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:27:45.317607+0000","last_undegraded":"2023-06-10T21:27:45.317607+0000","last_fullsized":"2023-06-10T21:27:45.317607+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":214,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.314413+0000","last_change":"2023-06-10T21:26:27.782629+0000","last_active":"2023-06-10T21:27:45.314413+0000","last_peered":"2023-06-10T21:27:45.314413+0000","last_clean":"2023-06-10T21:19:00.489490+0000","last_became_active":"2023-06-10T21:26:27.782629+0000","last_became_peered":"2023-06-10T21:26:27.782629+0000","last_unstale":"2023-06-10T21:27:45.314413+0000","last_undegraded":"2023-06-10T21:27:45.314413+0000","last_fullsized":"2023-06-10T21:26:27.779384+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.314626+0000","last_change":"2023-06-10T21:26:27.784405+0000","last_active":"2023-06-10T21:27:45.314626+0000","last_peered":"2023-06-10T21:27:45.314626+0000","last_clean":"2023-06-10T21:25:59.679038+0000","last_became_active":"2023-06-10T21:26:27.784405+0000","last_became_peered":"2023-06-10T21:26:27.784405+0000","last_unstale":"2023-06-10T21:27:45.314626+0000","last_undegraded":"2023-06-10T21:26:27.780168+0000","last_fullsized":"2023-06-10T21:26:27.780047+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":218,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.314345+0000","last_change":"2023-06-10T21:27:42.309456+0000","last_active":"2023-06-10T21:24:44.144630+0000","last_peered":"2023-06-10T21:27:45.314345+0000","last_clean":"2023-06-10T21:24:36.631470+0000","last_became_active":"2023-06-10T21:24:12.316438+0000","last_became_peered":"2023-06-10T21:27:42.309456+0000","last_unstale":"2023-06-10T21:27:45.314345+0000","last_undegraded":"2023-06-10T21:27:45.314345+0000","last_fullsized":"2023-06-10T21:27:42.306289+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":209,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.358677+0000","last_change":"2023-06-10T21:26:27.818384+0000","last_active":"2023-06-10T21:27:45.358677+0000","last_peered":"2023-06-10T21:27:45.358677+0000","last_clean":"2023-06-10T21:25:28.189114+0000","last_became_active":"2023-06-10T21:26:27.818384+0000","last_became_peered":"2023-06-10T21:26:27.818384+0000","last_unstale":"2023-06-10T21:27:45.358677+0000","last_undegraded":"2023-06-10T21:27:45.358677+0000","last_fullsized":"2023-06-10T21:26:27.812469+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:26:46.689756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":244,"reported_epoch":235,"state":"undersized+peered","last_fresh":"2023-06-10T21:27:45.358035+0000","last_change":"2023-06-10T21:27:42.304731+0000","last_active":"2023-06-10T21:27:41.300747+0000","last_peered":"2023-06-10T21:27:45.358035+0000","last_clean":"2023-06-10T21:25:28.188991+0000","last_became_active":"2023-06-10T21:26:27.820844+0000","last_became_peered":"2023-06-10T21:27:42.304731+0000","last_unstale":"2023-06-10T21:27:45.358035+0000","last_undegraded":"2023-06-10T21:27:45.358035+0000","last_fullsized":"2023-06-10T21:27:42.303381+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:00:08.777898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":68,"reported_epoch":235,"state":"active+undersized+degraded","last_fresh":"2023-06-10T21:27:45.359695+0000","last_change":"2023-06-10T21:27:42.312352+0000","last_active":"2023-06-10T21:27:45.359695+0000","last_peered":"2023-06-10T21:27:45.359695+0000","last_clean":"2023-06-10T21:25:28.176076+0000","last_became_active":"2023-06-10T21:27:42.312352+0000","last_became_peered":"2023-06-10T21:27:42.312352+0000","last_unstale":"2023-06-10T21:27:45.359695+0000","last_undegraded":"2023-06-10T21:27:42.306845+0000","last_fullsized":"2023-06-10T21:27:42.306741+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:02.670313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":217,"reported_epoch":235,"state":"active+undersized","last_fresh":"2023-06-10T21:27:45.313904+0000","last_change":"2023-06-10T21:26:27.791062+0000","last_active":"2023-06-10T21:27:45.313904+0000","last_peered":"2023-06-10T21:27:45.313904+0000","last_clean":"2023-06-10T21:25:15.648591+0000","last_became_active":"2023-06-10T21:26:27.791062+0000","last_became_peered":"2023-06-10T21:26:27.791062+0000","last_unstale":"2023-06-10T21:27:45.313904+0000","last_undegraded":"2023-06-10T21:27:45.313904+0000","last_fullsized":"2023-06-10T21:26:27.779951+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":58,"acting":58,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4582146072,"num_objects":69919,"num_object_clones":0,"num_object_copies":209757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":52235,"num_objects_misplaced":27639,"num_objects_unfound":0,"num_objects_dirty":69919,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":111589,"num_write_kb":5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16683,"num_bytes_recovered":1093271552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14368387072,"data_stored":14368374856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104243,"ondisk_log_size":104243,"up":30,"acting":35,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6076,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19667,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":62,"acting":62,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":2,"acting":2,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":14,"acting":14,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430571,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922952,"kb_used_data":1407552,"kb_used_omap":0,"kb_used_meta":515392,"kb_avail":91822200,"statfs":{"total":95995035648,"available":94025932800,"internally_reserved":0,"allocated":1441333248,"data_stored":1440798328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":527761408},"hb_peers":[1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:22 2023","interfaces":[{"interface":"back","average":{"1min":6.5860000000000003,"5min":3.7469999999999999,"15min":2.4009999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":100.319,"5min":100.319,"15min":100.319},"last":0.27500000000000002},{"interface":"front","average":{"1min":6.7949999999999999,"5min":3.673,"15min":2.5339999999999998},"min":{"1min":0.249,"5min":0.247,"15min":0.245},"max":{"1min":100.19,"5min":100.19,"15min":100.19},"last":0.245}]},{"osd":1,"last update":"Sat Jun 10 21:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":3.9729999999999999,"15min":2.5630000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.22900000000000001},"max":{"1min":6.4880000000000004,"5min":130.19399999999999,"15min":130.19399999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.6519999999999999,"5min":5.4800000000000004,"15min":3.0070000000000001},"min":{"1min":0.32000000000000001,"5min":0.28499999999999998,"15min":0.28100000000000003},"max":{"1min":4.9989999999999997,"5min":212.20599999999999,"15min":212.20599999999999},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:27:13 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":4.992,"15min":2.1179999999999999},"min":{"1min":0.33000000000000002,"5min":0.317,"15min":0.315},"max":{"1min":115.62,"5min":160.99299999999999,"15min":160.99299999999999},"last":0.432},{"interface":"front","average":{"1min":7.1310000000000002,"5min":7.1360000000000001,"15min":2.8100000000000001},"min":{"1min":0.38700000000000001,"5min":0.309,"15min":0.309},"max":{"1min":117.776,"5min":212.13399999999999,"15min":212.13399999999999},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":5.0629999999999997,"15min":2.0619999999999998},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":14.526,"5min":63.406999999999996,"15min":63.406999999999996},"last":0.44},{"interface":"front","average":{"1min":0.83199999999999996,"5min":5.0449999999999999,"15min":2.0720000000000001},"min":{"1min":0.312,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.4319999999999999,"5min":63.549999999999997,"15min":63.549999999999997},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:27:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.121,"15min":0.98199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.22800000000000001},"max":{"1min":3.1819999999999999,"5min":8.1419999999999995,"15min":8.1419999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.6679999999999999,"15min":1.3260000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.202,"5min":14.657999999999999,"15min":14.657999999999999},"last":0.27600000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":4.0910000000000002,"15min":2.1709999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":12.048,"5min":211.99799999999999,"15min":211.99799999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.98099999999999998,"5min":3.6299999999999999,"15min":1.958},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.3010000000000002,"5min":212.23699999999999,"15min":212.23699999999999},"last":0.30199999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:27:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":1.899,"15min":1.3109999999999999},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.29199999999999998},"max":{"1min":13.073,"5min":13.073,"15min":13.073},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.6000000000000001,"15min":1.127},"min":{"1min":0.38800000000000001,"5min":0.374,"15min":0.32800000000000001},"max":{"1min":3.8239999999999998,"5min":10.874000000000001,"15min":10.874000000000001},"last":0.36499999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626799,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566241,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1554124,"kb_used_data":1127684,"kb_used_omap":4,"kb_used_meta":426427,"kb_avail":92191028,"statfs":{"total":95995035648,"available":94403612672,"internally_reserved":0,"allocated":1154748416,"data_stored":1153976677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":436662158},"hb_peers":[2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":5.0389999999999997,"15min":3.1720000000000002},"min":{"1min":0.20699999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.0270000000000001,"5min":121.917,"15min":121.917},"last":1.2849999999999999},{"interface":"front","average":{"1min":2.2130000000000001,"5min":5.8890000000000002,"15min":3.7829999999999999},"min":{"1min":0.29799999999999999,"5min":0.25,"15min":0.22600000000000001},"max":{"1min":7.0709999999999997,"5min":121.455,"15min":121.455},"last":1.3120000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:27:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.4649999999999999,"15min":1.304},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.0830000000000002,"5min":18.152000000000001,"15min":18.152000000000001},"last":102.09399999999999},{"interface":"front","average":{"1min":2.3809999999999998,"5min":1.899,"15min":0.98399999999999999},"min":{"1min":0.26900000000000002,"5min":0.222,"15min":0.218},"max":{"1min":8.0559999999999992,"5min":12.081,"15min":12.081},"last":102.077}]},{"osd":3,"last update":"Sat Jun 10 21:25:36 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":4.2030000000000003,"15min":1.641},"min":{"1min":0.29299999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65200000000000002,"5min":118.348,"15min":118.348},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.2030000000000001,"5min":4.9160000000000004,"15min":1.9159999999999999},"min":{"1min":0.34300000000000003,"5min":0.224,"15min":0.224},"max":{"1min":7.1870000000000003,"5min":117.75,"15min":117.75},"last":5.1109999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":2.073,"15min":1.8180000000000001},"min":{"1min":0.32100000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.9409999999999998,"5min":12.507999999999999,"15min":12.94},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.417,"5min":1.76,"15min":1.9950000000000001},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.2539999999999996,"5min":14.460000000000001,"15min":14.460000000000001},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":9.202,"15min":4.4100000000000001},"min":{"1min":0.432,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":6.9550000000000001,"5min":383.83100000000002,"15min":383.83100000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.6739999999999999,"5min":8.734,"15min":4.3360000000000003},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.9900000000000002,"5min":383.84800000000001,"15min":383.84800000000001},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":2.7269999999999999,"15min":2.262},"min":{"1min":0.443,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.9900000000000002,"5min":14.858000000000001,"15min":14.858000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.3250000000000002,"15min":2.1680000000000001},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.0220000000000002,"5min":13.48,"15min":13.48},"last":0.63}]},{"osd":7,"last update":"Sat Jun 10 21:27:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":4.5099999999999998,"15min":3.1709999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.32600000000000001},"max":{"1min":8.0890000000000004,"5min":162.399,"15min":162.399},"last":0.505},{"interface":"front","average":{"1min":2.0129999999999999,"5min":3.7930000000000001,"15min":2.5419999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.38200000000000001},"max":{"1min":6.2160000000000002,"5min":162.27199999999999,"15min":162.27199999999999},"last":0.45400000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187659,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3507076,"kb_used_data":2969596,"kb_used_omap":5,"kb_used_meta":537466,"kb_avail":90238076,"statfs":{"total":95995035648,"available":92403789824,"internally_reserved":0,"allocated":3040866304,"data_stored":3040057020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":550365818},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:26:58 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":6.3140000000000001,"15min":3.8889999999999998},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":3.0710000000000002,"5min":175.352,"15min":175.352},"last":0.26200000000000001},{"interface":"front","average":{"1min":1.8660000000000001,"5min":6.1790000000000003,"15min":4.4279999999999999},"min":{"1min":0.23899999999999999,"5min":0.189,"15min":0.189},"max":{"1min":8.8780000000000001,"5min":175.30500000000001,"15min":175.30500000000001},"last":3.194}]},{"osd":2,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":7.1589999999999998,"5min":3.032,"15min":1.361},"min":{"1min":0.28000000000000003,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":98.710999999999999,"5min":98.710999999999999,"15min":98.710999999999999},"last":1.236},{"interface":"front","average":{"1min":7.5309999999999997,"5min":3.1480000000000001,"15min":1.3999999999999999},"min":{"1min":0.30499999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":98.772999999999996,"5min":98.772999999999996,"15min":98.772999999999996},"last":3.3639999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:25:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":8.7550000000000008,"15min":3.1859999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":1.524,"5min":181.886,"15min":181.886},"last":0.371},{"interface":"front","average":{"1min":0.70199999999999996,"5min":9.5440000000000005,"15min":3.4390000000000001},"min":{"1min":0.28399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9049999999999998,"5min":180.04900000000001,"15min":180.04900000000001},"last":3.8759999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":7.9580000000000002,"5min":3.2389999999999999,"15min":2.3090000000000002},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":66.915999999999997,"5min":66.915999999999997,"15min":66.915999999999997},"last":1.28},{"interface":"front","average":{"1min":8.423,"5min":3.2280000000000002,"15min":2.0790000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.29699999999999999},"max":{"1min":66.891999999999996,"5min":66.891999999999996,"15min":66.891999999999996},"last":0.496}]},{"osd":5,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.0790000000000002,"15min":1.7789999999999999},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.0380000000000003,"5min":25.545000000000002,"15min":25.545000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.0699999999999998,"15min":2.1930000000000001},"min":{"1min":0.41399999999999998,"5min":0.375,"15min":0.32400000000000001},"max":{"1min":8.7449999999999992,"5min":8.7449999999999992,"15min":17.460000000000001},"last":3.331}]},{"osd":6,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":2.1230000000000002,"15min":1.857},"min":{"1min":0.375,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.2959999999999994,"5min":25.608000000000001,"15min":25.608000000000001},"last":2.8679999999999999},{"interface":"front","average":{"1min":2.3220000000000001,"5min":2.2530000000000001,"15min":2.2730000000000001},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.7929999999999993,"5min":10.231999999999999,"15min":17.411000000000001},"last":3.254}]},{"osd":7,"last update":"Sat Jun 10 21:27:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.2770000000000001,"15min":2.246},"min":{"1min":0.437,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.8230000000000004,"5min":10.269,"15min":17.486999999999998},"last":3.2919999999999998},{"interface":"front","average":{"1min":1.395,"5min":1.9630000000000001,"15min":2.0190000000000001},"min":{"1min":0.39700000000000002,"5min":0.38300000000000001,"15min":0.36799999999999999},"max":{"1min":6.9720000000000004,"5min":9.734,"15min":10.867000000000001},"last":1.349}]}]},{"osd":2,"up_from":120,"seq":515396075630,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2570104,"kb_used_data":2224752,"kb_used_omap":7,"kb_used_meta":345336,"kb_avail":91175048,"statfs":{"total":95995035648,"available":93363249152,"internally_reserved":0,"allocated":2278146048,"data_stored":2277394008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":353624145},"hb_peers":[1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":11.116,"5min":3.855,"15min":1.7310000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":237.251,"5min":237.251,"15min":237.251},"last":0.45200000000000001},{"interface":"front","average":{"1min":11.191000000000001,"5min":4.6070000000000002,"15min":1.978},"min":{"1min":0.20599999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":232.06700000000001,"5min":232.06700000000001,"15min":232.06700000000001},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":2.1899999999999999,"15min":1.1850000000000001},"min":{"1min":0.23799999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":8.6880000000000006,"5min":17.718,"15min":29.486000000000001},"last":0.255},{"interface":"front","average":{"1min":1.163,"5min":1.8560000000000001,"15min":1.093},"min":{"1min":0.22500000000000001,"5min":0.216,"15min":0.216},"max":{"1min":8.1959999999999997,"5min":16.641999999999999,"15min":28.013999999999999},"last":0.30299999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:25:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":4.2549999999999999,"15min":1.6839999999999999},"min":{"1min":0.32200000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.6989999999999998,"5min":96.295000000000002,"15min":96.295000000000002},"last":4.157},{"interface":"front","average":{"1min":1.0449999999999999,"5min":4.8460000000000001,"15min":1.899},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.9649999999999999,"5min":99.504999999999995,"15min":99.504999999999995},"last":2.9489999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":13.571,"5min":4.2430000000000003,"15min":1.9179999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":224.00999999999999,"5min":224.00999999999999,"15min":224.00999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":14.023,"5min":4.3860000000000001,"15min":1.9870000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":223.28399999999999,"5min":223.28399999999999,"15min":223.28399999999999},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2709999999999999,"5min":1.518,"15min":1.0860000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.1219999999999999,"5min":10.254,"15min":10.254},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.0659999999999998,"15min":1.2589999999999999},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":7.2850000000000001,"5min":10.345000000000001,"15min":10.345000000000001},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.0790000000000002,"15min":1.258},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.3889999999999993,"5min":9.4949999999999992,"15min":9.4949999999999992},"last":0.627},{"interface":"front","average":{"1min":1.8879999999999999,"5min":2.3809999999999998,"15min":1.3660000000000001},"min":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.8420000000000005,"5min":17.244,"15min":17.244},"last":0.623}]},{"osd":7,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.5089999999999999,"15min":1.069},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.9329999999999998,"5min":9.9329999999999998,"15min":9.9329999999999998},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.9339999999999999,"5min":2.1789999999999998,"15min":1.3},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":9.9079999999999995,"5min":9.9079999999999995,"15min":9.9079999999999995},"last":0.66600000000000004}]}]},{"osd":3,"up_from":125,"seq":536870912085,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267612,"kb_used_data":1002452,"kb_used_omap":2,"kb_used_meta":265149,"kb_avail":92477540,"statfs":{"total":95995035648,"available":94697000960,"internally_reserved":0,"allocated":1026510848,"data_stored":1025770762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":271512629},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":2.234,"15min":0.98599999999999999},"min":{"1min":0.29399999999999998,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":8.0419999999999998,"5min":46.677999999999997,"15min":46.677999999999997},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.5049999999999999,"5min":4.2110000000000003,"15min":1.639},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":10.273999999999999,"5min":130.893,"15min":130.893},"last":5.5300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.6869999999999998,"15min":1.1639999999999999},"min":{"1min":0.33900000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.0030000000000001,"5min":54.112000000000002,"15min":54.112000000000002},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.2759999999999998,"5min":3.282,"15min":1.3700000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29299999999999998},"max":{"1min":8.2810000000000006,"5min":57.814,"15min":57.814},"last":5.5609999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":2.2639999999999998,"15min":1.014},"min":{"1min":0.27600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":10.047000000000001,"5min":12.726000000000001,"15min":12.726000000000001},"last":5.4859999999999998},{"interface":"front","average":{"1min":1.514,"5min":1.7989999999999999,"15min":0.86099999999999999},"min":{"1min":0.313,"5min":0.249,"15min":0.249},"max":{"1min":6.0030000000000001,"5min":9.8000000000000007,"15min":9.8000000000000007},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":2.0409999999999999,"15min":1.0629999999999999},"min":{"1min":0.39100000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.3190000000000008,"5min":13.656000000000001,"15min":13.656000000000001},"last":2.4910000000000001},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.0089999999999999,"15min":1.052},"min":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.3200000000000003,"5min":8.4800000000000004,"15min":8.4800000000000004},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":9.0730000000000004,"15min":3.4220000000000002},"min":{"1min":0.498,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.3580000000000005,"5min":349.47399999999999,"15min":349.47399999999999},"last":2.448},{"interface":"front","average":{"1min":2.8180000000000001,"5min":8.0099999999999998,"15min":3.0760000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":8.3740000000000006,"5min":349.65100000000001,"15min":349.65100000000001},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":2.2610000000000001,"15min":1.1479999999999999},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.9800000000000004,"5min":9.9619999999999997,"15min":9.9619999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.675,"5min":2.0339999999999998,"15min":1.1220000000000001},"min":{"1min":0.373,"5min":0.313,"15min":0.313},"max":{"1min":8.3339999999999996,"5min":12.835000000000001,"15min":12.835000000000001},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:25:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1909999999999998,"5min":5.3150000000000004,"15min":2.1789999999999998},"min":{"1min":0.38300000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.2230000000000008,"5min":174.44800000000001,"15min":174.44800000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.6320000000000001,"5min":3.8570000000000002,"15min":1.734},"min":{"1min":0.40400000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.9450000000000003,"5min":173.18100000000001,"15min":173.18100000000001},"last":0.72999999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019255,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4057160,"kb_used_data":3558464,"kb_used_omap":0,"kb_used_meta":498688,"kb_avail":89687992,"statfs":{"total":95995035648,"available":91840503808,"internally_reserved":0,"allocated":3643867136,"data_stored":3643331935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":510656512},"hb_peers":[1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":1.8520000000000001,"15min":1.849},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22},"max":{"1min":48.390999999999998,"5min":48.390999999999998,"15min":48.390999999999998},"last":0.27700000000000002},{"interface":"front","average":{"1min":3.8010000000000002,"5min":2.29,"15min":2.2109999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23100000000000001},"max":{"1min":47.991999999999997,"5min":47.991999999999997,"15min":47.991999999999997},"last":0.30399999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:27:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":5.0830000000000002,"15min":3.0510000000000002},"min":{"1min":0.34000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":15.063000000000001,"5min":182.035,"15min":182.035},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.1800000000000002,"5min":4.4390000000000001,"15min":3.024},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":15.052,"5min":181.73099999999999,"15min":181.73099999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0249999999999999,"5min":2.0659999999999998,"15min":1.0800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":22.614000000000001,"5min":22.614000000000001,"15min":22.614000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.04,"15min":1.147},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":22.326000000000001,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:25:39 2023","interfaces":[{"interface":"back","average":{"1min":6.9969999999999999,"5min":7.4119999999999999,"15min":2.8420000000000001},"min":{"1min":0.498,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":59.084000000000003,"5min":103.575,"15min":103.575},"last":6.601},{"interface":"front","average":{"1min":5.8949999999999996,"5min":6.7610000000000001,"15min":2.6669999999999998},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":59.128,"5min":103.565,"15min":103.565},"last":6.6120000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:27:17 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.8140000000000001,"15min":1.131},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":3.399,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.046,"5min":1.7030000000000001,"15min":1.212},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.5209999999999999,"5min":11.904999999999999,"15min":11.904999999999999},"last":0.30399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:27:44 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.476,"15min":1.2430000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.024,"5min":7.5090000000000003,"15min":8.2599999999999998},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.6679999999999999,"15min":1.2529999999999999},"min":{"1min":0.32700000000000001,"5min":0.309,"15min":0.30399999999999999},"max":{"1min":3.4220000000000002,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.373}]},{"osd":7,"last update":"Sat Jun 10 21:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.927,"15min":1.415},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.9000000000000004,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.7290000000000001,"15min":1.3129999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.0800000000000001,"5min":9.5060000000000002,"15min":9.5060000000000002},"last":0.40100000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823026,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3009613824,"data_stored":3009609752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1151729664,"data_stored":1151729664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2275737600,"data_stored":2275737600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1040711680,"data_stored":1040711680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3652063232,"data_stored":3652059160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":13303808,"data_stored":13303808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1786646528,"data_stored":1786642456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1438580736,"data_stored":1438580736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:28:04.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_5 pgp_num 16 2023-06-10T21:28:04.380 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:04.415 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:04.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:04.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:04 smithi064 ceph-mon[180297]: failed to encode map e237 with expected crc 2023-06-10T21:28:04.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:04 smithi064 ceph-mon[186706]: failed to encode map e237 with expected crc 2023-06-10T21:28:04.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:04.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:04 smithi152 ceph-mon[156819]: failed to encode map e237 with expected crc 2023-06-10T21:28:04.546 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:04.586 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:04.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:04.620 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:28:04.620 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:28:04.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:28:04.620 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13232 10.9 10:9012193c:::benchmark_data_smithi064_312713_object13231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e201)", 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 139.35063577400001, 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7512189839999999, 2023-06-10T21:28:04.621 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.622 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.622 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.622 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.622 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.623 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13232 2023-06-10T21:28:04.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.623 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:28:04.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:28:04.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264126+0000", 2023-06-10T21:28:04.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:28:04.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264155+0000", 2023-06-10T21:28:04.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8109000000000001e-05 2023-06-10T21:28:04.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264178+0000", 2023-06-10T21:28:04.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3536999999999998e-05 2023-06-10T21:28:04.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264189+0000", 2023-06-10T21:28:04.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0995000000000001e-05 2023-06-10T21:28:04.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264267+0000", 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7470000000000002e-05 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264279+0000", 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2045000000000001e-05 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614347+0000", 2023-06-10T21:28:04.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35006879200000002 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614373+0000", 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5544000000000001e-05 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.003258+0000", 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3888854589999999 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.004667+0000", 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014090960000000001 2023-06-10T21:28:04.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015232+0000", 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010564163999999999 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015348+0000", 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115995 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.637 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13262 10.9 10:9b8ca29e:::benchmark_data_smithi064_312713_object13261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:28:04.638 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:28:04.638 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 139.33331512000001, 2023-06-10T21:28:04.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7392644879999999, 2023-06-10T21:28:04.638 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.639 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.639 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.639 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.639 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13262 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:28:04.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281448+0000", 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281480+0000", 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2228999999999999e-05 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281490+0000", 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8079999999999999e-06 2023-06-10T21:28:04.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281496+0000", 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7790000000000001e-06 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281565+0000", 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8689999999999995e-05 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281579+0000", 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4499e-05 2023-06-10T21:28:04.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614432+0000", 2023-06-10T21:28:04.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33285274500000001 2023-06-10T21:28:04.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614439+0000", 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.007692+0000", 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.393252581 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.009132+0000", 2023-06-10T21:28:04.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014407669999999999 2023-06-10T21:28:04.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:28:04.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.011401+0000", 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002268927 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015111+0000", 2023-06-10T21:28:04.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037098830000000002 2023-06-10T21:28:04.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015417+0000", 2023-06-10T21:28:04.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030584399999999999 2023-06-10T21:28:04.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020055+0000", 2023-06-10T21:28:04.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046377069999999996 2023-06-10T21:28:04.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020100+0000", 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5522999999999998e-05 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020714+0000", 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00061340799999999997 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13275 10.9 10:9047ac4e:::benchmark_data_smithi064_312713_object13274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:28:04.655 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:28:04.656 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 139.32542395300001, 2023-06-10T21:28:04.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7361003699999999, 2023-06-10T21:28:04.656 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.656 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.656 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13275 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:28:04.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289339+0000", 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:04.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289384+0000", 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5176000000000003e-05 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289404+0000", 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0273999999999999e-05 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289414+0000", 2023-06-10T21:28:04.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028e-05 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289471+0000", 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6205999999999997e-05 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289480+0000", 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2690000000000005e-06 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614466+0000", 2023-06-10T21:28:04.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32498670400000002 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614474+0000", 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8569999999999995e-06 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015439+0000", 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.400964885 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015551+0000", 2023-06-10T21:28:04.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111486 2023-06-10T21:28:04.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:28:04.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015728+0000", 2023-06-10T21:28:04.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000177412 2023-06-10T21:28:04.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020768+0000", 2023-06-10T21:28:04.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050401889999999996 2023-06-10T21:28:04.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020809+0000", 2023-06-10T21:28:04.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1056000000000001e-05 2023-06-10T21:28:04.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024793+0000", 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003983444 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024844+0000", 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1477000000000002e-05 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025441+0000", 2023-06-10T21:28:04.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00059653199999999999 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54555 10.7 10:e3cbd15b:::benchmark_data_smithi064_312713_object54554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:28:04.672 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.810124372000004, 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1739721809999999, 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.673 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54555 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804638+0000", 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804665+0000", 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6137000000000001e-05 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804675+0000", 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0899e-05 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804691+0000", 2023-06-10T21:28:04.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5661000000000001e-05 2023-06-10T21:28:04.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804758+0000", 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6879999999999997e-05 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804769+0000", 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1028e-05 2023-06-10T21:28:04.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818008+0000", 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0132391670000001 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818017+0000", 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3829999999999992e-06 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.955590+0000", 2023-06-10T21:28:04.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375735200000001 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.956353+0000", 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000762807 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978548+0000", 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022194875999999999 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978612+0000", 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4449999999999994e-05 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54557 10.f 10:f1e22a0f:::benchmark_data_smithi064_312713_object54556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:28:04.684 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:28:04.685 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.810039301000003, 2023-06-10T21:28:04.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1819947270000002, 2023-06-10T21:28:04.685 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.685 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.686 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.686 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.686 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.687 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54557 2023-06-10T21:28:04.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.687 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804724+0000", 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804750+0000", 2023-06-10T21:28:04.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5415999999999999e-05 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804759+0000", 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5969999999999998e-06 2023-06-10T21:28:04.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804767+0000", 2023-06-10T21:28:04.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:28:04.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804793+0000", 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5993e-05 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804800+0000", 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1019999999999999e-06 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818039+0000", 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013239368 2023-06-10T21:28:04.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818047+0000", 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5510000000000002e-06 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.977925+0000", 2023-06-10T21:28:04.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159878645 2023-06-10T21:28:04.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978506+0000", 2023-06-10T21:28:04.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058025500000000001 2023-06-10T21:28:04.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986600+0000", 2023-06-10T21:28:04.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080940130000000006 2023-06-10T21:28:04.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986720+0000", 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120254 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.700 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54569 10.3 10:c761f176:::benchmark_data_smithi064_312713_object54568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.809223058000001, 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0199541459999999, 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.701 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54569 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.702 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805540+0000", 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:04.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805563+0000", 2023-06-10T21:28:04.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2124e-05 2023-06-10T21:28:04.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805572+0000", 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2550000000000003e-06 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805580+0000", 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7929999999999998e-06 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805621+0000", 2023-06-10T21:28:04.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1746999999999999e-05 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805631+0000", 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3880000000000006e-06 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822642+0000", 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0170114130000001 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822728+0000", 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6032999999999994e-05 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822844+0000", 2023-06-10T21:28:04.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011603300000000001 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.824327+0000", 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014828739999999999 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825354+0000", 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001026875 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825416+0000", 2023-06-10T21:28:04.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1940999999999995e-05 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825496+0000", 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9591000000000004e-05 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.713 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54574 10.5 10:a853286f:::benchmark_data_smithi064_312713_object54573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.808392284999996, 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0150581569999999, 2023-06-10T21:28:04.714 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54574 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.715 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806370+0000", 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:04.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806397+0000", 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6996999999999999e-05 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806413+0000", 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6099999999999998e-05 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806422+0000", 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.833e-06 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809434+0000", 2023-06-10T21:28:04.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030114529999999999 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809442+0000", 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1799999999999996e-06 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818062+0000", 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0086199140000001 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818198+0000", 2023-06-10T21:28:04.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013567700000000001 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0", 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818376+0000", 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000178813 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.820582+0000", 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022051589999999999 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821290+0000", 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070819899999999996 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821344+0000", 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4304e-05 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821430+0000", 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6263999999999999e-05 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54575 10.e 10:708b6ab1:::benchmark_data_smithi064_312713_object54574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.799038163999995, 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0012291179999999, 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.727 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54575 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:28:04.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815725+0000", 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815752+0000", 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7001000000000001e-05 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815769+0000", 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7203000000000001e-05 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815779+0000", 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6020000000000006e-06 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820630+0000", 2023-06-10T21:28:04.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048513219999999999 2023-06-10T21:28:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820642+0000", 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2081e-05 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817919+0000", 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99727731900000005 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817927+0000", 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7549999999999994e-06 2023-06-10T21:28:04.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810577+0000", 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99265023100000005 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810664+0000", 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6879999999999995e-05 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810771+0000", 2023-06-10T21:28:04.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106632 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816578+0000", 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058069870000000004 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816627+0000", 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8791000000000001e-05 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816709+0000", 2023-06-10T21:28:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2793000000000005e-05 2023-06-10T21:28:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816771+0000", 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1524000000000003e-05 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816955+0000", 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018453799999999999 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.743 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54576 10.f 10:f052f063:::benchmark_data_smithi064_312713_object54575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.793945124999993, 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1709000860000001, 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.744 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54576 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:28:04.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820817+0000", 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820849+0000", 2023-06-10T21:28:04.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1776000000000002e-05 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820866+0000", 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6892999999999999e-05 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820875+0000", 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5739999999999996e-06 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820944+0000", 2023-06-10T21:28:04.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9340999999999999e-05 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820957+0000", 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3226e-05 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818068+0000", 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99711057199999997 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818077+0000", 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7480000000000005e-06 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978558+0000", 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1604804230000001 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979614+0000", 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001056162 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979766+0000", 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015153399999999999 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.984553+0000", 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047875230000000001 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986763+0000", 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022097850000000001 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990555+0000", 2023-06-10T21:28:04.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037920319999999999 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990588+0000", 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3130999999999998e-05 2023-06-10T21:28:04.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.991719+0000", 2023-06-10T21:28:04.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001131331 2023-06-10T21:28:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.758 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54580 10.e 10:70306020:::benchmark_data_smithi064_312713_object54579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.785955842000007, 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9886503549999999, 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.759 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54580 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828807+0000", 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828835+0000", 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7953000000000002e-05 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828857+0000", 2023-06-10T21:28:04.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2255999999999998e-05 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828867+0000", 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9520000000000006e-06 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828928+0000", 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1260999999999995e-05 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828938+0000", 2023-06-10T21:28:04.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0195e-05 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817952+0000", 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98901349999999999 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817959+0000", 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3690000000000001e-06 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812558+0000", 2023-06-10T21:28:04.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.994598918 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812679+0000", 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012119499999999999 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812787+0000", 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010732 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816636+0000", 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038490880000000001 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817109+0000", 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047353 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817227+0000", 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011818 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817283+0000", 2023-06-10T21:28:04.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5247000000000001e-05 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817459+0000", 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017632800000000001 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.772 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54582 10.e 10:78d4d384:::benchmark_data_smithi064_312713_object54581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 96.825365765000001, 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028370658, 2023-06-10T21:28:04.773 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54582 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.774 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789397+0000", 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789430+0000", 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3229999999999999e-05 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813125+0000", 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023694667999999999 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813150+0000", 2023-06-10T21:28:04.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5534999999999999e-05 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817984+0000", 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048335940000000001 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817990+0000", 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8869999999999997e-06 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814293+0000", 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99630294900000005 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814412+0000", 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011934099999999999 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814525+0000", 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011297400000000001 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816642+0000", 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021171509999999998 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817165+0000", 2023-06-10T21:28:04.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052269299999999996 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817520+0000", 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035547299999999998 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817583+0000", 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090999999999999e-05 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817769+0000", 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018677799999999999 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93470 10.f 10:f601cf75:::benchmark_data_smithi064_312713_object2714:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:28:04.786 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.497457799000003, 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.39052785000000001, 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.787 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93470 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117305+0000", 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117310+0000", 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2850000000000002e-06 2023-06-10T21:28:04.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117320+0000", 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2159999999999995e-06 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117327+0000", 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2470000000000002e-06 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.132977+0000", 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015650496 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.133683+0000", 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070519099999999998 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166028+0000", 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032345157999999999 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166124+0000", 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5873999999999997e-05 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166229+0000", 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010504499999999999 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172050+0000", 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058210919999999999 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.175754+0000", 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037038639999999999 2023-06-10T21:28:04.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.506456+0000", 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33070211799999999 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.506511+0000", 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5271e-05 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507834+0000", 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013234049999999999 2023-06-10T21:28:04.798 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93479 10.f 10:f21736d6:::benchmark_data_smithi064_312713_object2723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.494647524999998, 2023-06-10T21:28:04.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.38120153600000001, 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93479 2023-06-10T21:28:04.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120115+0000", 2023-06-10T21:28:04.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120123+0000", 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6690000000000006e-06 2023-06-10T21:28:04.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120136+0000", 2023-06-10T21:28:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3159000000000001e-05 2023-06-10T21:28:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120146+0000", 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.133759+0000", 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013613142999999999 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.134418+0000", 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065979000000000005 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.162952+0000", 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028533912000000002 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.163054+0000", 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101817 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.163196+0000", 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014156100000000001 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.170963+0000", 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077668759999999998 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172078+0000", 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001115436 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.369431+0000", 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19735330300000001 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.369467+0000", 2023-06-10T21:28:04.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5976999999999998e-05 2023-06-10T21:28:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.501318+0000", 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13185106799999999 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.813 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93515 10.7 10:e0fa7a28:::benchmark_data_smithi064_312713_object2759:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.444850738, 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35528036099999999, 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93515 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:28:04.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:28:04.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169913+0000", 2023-06-10T21:28:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169916+0000", 2023-06-10T21:28:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7259999999999998e-06 2023-06-10T21:28:04.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169922+0000", 2023-06-10T21:28:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6049999999999997e-06 2023-06-10T21:28:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169927+0000", 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9960000000000001e-06 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.171491+0000", 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001564735 2023-06-10T21:28:04.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172025+0000", 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053375000000000004 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505189+0000", 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.333163863 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505285+0000", 2023-06-10T21:28:04.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6084999999999999e-05 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505417+0000", 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013150400000000001 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.514118+0000", 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087012889999999992 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.518148+0000", 2023-06-10T21:28:04.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040298829999999997 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.523254+0000", 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051061709999999996 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.523312+0000", 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8529000000000002e-05 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.525194+0000", 2023-06-10T21:28:04.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018816250000000001 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93518 10.f 10:f6658ccc:::benchmark_data_smithi064_312713_object2762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:28:04.829 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 50.442678946999997, 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.353940329, 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.830 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93518 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172084+0000", 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172090+0000", 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1209999999999998e-06 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172100+0000", 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0366999999999999e-05 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172107+0000", 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4460000000000004e-06 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172140+0000", 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2665e-05 2023-06-10T21:28:04.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172649+0000", 2023-06-10T21:28:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050871100000000002 2023-06-10T21:28:04.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507880+0000", 2023-06-10T21:28:04.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33523090900000002 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507987+0000", 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010741 2023-06-10T21:28:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.508127+0000", 2023-06-10T21:28:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140283 2023-06-10T21:28:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.514508+0000", 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063808509999999999 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.518298+0000", 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00378926 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.524434+0000", 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061361949999999997 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.524481+0000", 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7307e-05 2023-06-10T21:28:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.526026+0000", 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015447740000000001 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:97037 10.f 10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 45.103576279999999, 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35586957400000002, 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 97037 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511190+0000", 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9340000000000002e-06 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511198+0000", 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0910000000000004e-06 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511203+0000", 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.412e-06 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511240+0000", 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7057000000000001e-05 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:28:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511951+0000", 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071124399999999998 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839245+0000", 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32729350600000001 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839346+0000", 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010158 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839487+0000", 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014055299999999999 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.845225+0000", 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057377060000000004 2023-06-10T21:28:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.845297+0000", 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2552000000000006e-05 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.865475+0000", 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020177835000000002 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.865540+0000", 2023-06-10T21:28:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5288000000000002e-05 2023-06-10T21:28:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.867058+0000", 2023-06-10T21:28:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001517432 2023-06-10T21:28:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111470 10.a 10:51c5c507:::benchmark_data_smithi064_312713_object20714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.302466708000001, 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0089041089999999, 2023-06-10T21:28:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111470 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312296+0000", 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312303+0000", 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4030000000000004e-06 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312316+0000", 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3351e-05 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312327+0000", 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0559e-05 2023-06-10T21:28:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.315119+0000", 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027926600000000002 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.315131+0000", 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2006e-05 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.321118+0000", 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005986448 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.321202+0000", 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4065000000000002e-05 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111477 10.8 10:1f95a548:::benchmark_data_smithi064_312713_object20721:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.302430083000001, 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0239321720000001, 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111477 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312337+0000", 2023-06-10T21:28:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5159999999999999e-06 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312345+0000", 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5569999999999998e-06 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312351+0000", 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0390000000000001e-06 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312451+0000", 2023-06-10T21:28:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9871000000000007e-05 2023-06-10T21:28:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312462+0000", 2023-06-10T21:28:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1076e-05 2023-06-10T21:28:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330040+0000", 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.017578477 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330149+0000", 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010885199999999999 2023-06-10T21:28:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336202+0000", 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006053353 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336267+0000", 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4151000000000003e-05 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111482 10.5 10.2e274505 (undecoded) ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.295239368000001, 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001240895, 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111482 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:28:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319524+0000", 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319532+0000", 2023-06-10T21:28:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1629999999999996e-06 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319546+0000", 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4496e-05 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319555+0000", 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2e-06 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319609+0000", 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3497e-05 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319622+0000", 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325e-05 2023-06-10T21:28:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.320740+0000", 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001117627 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.320766+0000", 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6193000000000001e-05 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111485 10.8 10:16819874:::benchmark_data_smithi064_312713_object20729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.285894874, 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0079469780000001, 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111485 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:28:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:28:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328868+0000", 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328874+0000", 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3410000000000002e-06 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328886+0000", 2023-06-10T21:28:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2594e-05 2023-06-10T21:28:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328893+0000", 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5359999999999998e-06 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.339898+0000", 2023-06-10T21:28:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011005173 2023-06-10T21:28:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:28:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.339912+0000", 2023-06-10T21:28:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4032999999999999e-05 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330385+0000", 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99047314200000003 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330481+0000", 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6095999999999996e-05 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2023-06-10T21:28:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330564+0000", 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2441000000000006e-05 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.335813+0000", 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005249592 2023-06-10T21:28:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336705+0000", 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00089199799999999997 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:28:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336749+0000", 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3569000000000002e-05 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336817+0000", 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7805000000000006e-05 2023-06-10T21:28:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:04.900 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:28:04.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:04.902 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.903 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.904 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:28:04.932 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:04.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:04.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:04.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:05.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:05.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:05.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:05 smithi064 ceph-mon[180297]: pgmap v838: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:05.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3500985150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:05 smithi064 ceph-mon[186706]: pgmap v838: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3500985150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:05 smithi152 ceph-mon[156819]: pgmap v838: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 27639/209829 objects misplaced (13.172%); 13 MiB/s, 214 objects/s recovering 2023-06-10T21:28:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3500985150' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:05.886 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:05.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:05.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:05.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:28:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:05.948 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:05.985 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:05.992 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:06.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:06.115 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:06.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:06.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:06.246 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:06.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:28:06.292 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:06.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:06.298 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:06.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:06.423 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:06.517 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:06.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:06.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:06.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:06.648 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:06.741 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:06.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:06.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:06.872 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:06.966 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:06.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:07.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:07.102 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:07.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:07.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:07.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:07.360 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:07 smithi064 ceph-mon[180297]: pgmap v839: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:07.361 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:07 smithi064 ceph-mon[186706]: pgmap v839: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:07.394 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:07.443 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:07.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:07.447 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:07.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:07.577 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:07.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:07.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:07 smithi152 ceph-mon[156819]: pgmap v839: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:07.783 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:07.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:07.864 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:07.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:07.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:07.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:28:07.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:07.998 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:08.044 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:08.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:08.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:08.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:08.176 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:08.219 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:08.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:08.240 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:08.240 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:08.246 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 10 pgp_num to 16 2023-06-10T21:28:08.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:08.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:28:08.386 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:08.483 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:08.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:08.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:28:08.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-10T21:28:08.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:08.616 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:09.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:28:09.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:09.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:09.136 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:09.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:09.196 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:09.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:09.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:09.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:09.356 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:09.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[186706]: pgmap v840: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:09.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[186706]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[186706]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:28:09.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[186706]: osdmap e238: 8 total, 6 up, 6 in 2023-06-10T21:28:09.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[180297]: pgmap v840: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:09.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[180297]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[180297]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:28:09.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:09 smithi064 ceph-mon[180297]: osdmap e238: 8 total, 6 up, 6 in 2023-06-10T21:28:09.384 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:09 smithi152 ceph-mon[156819]: pgmap v840: 89 pgs: 2 active+clean+remapped, 1 active+recovering+degraded+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 52257/209829 objects degraded (24.905%); 26458/209829 objects misplaced (12.609%); 13 MiB/s, 208 objects/s recovering 2023-06-10T21:28:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:09 smithi152 ceph-mon[156819]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:09 smithi152 ceph-mon[156819]: failed to encode map e238 with expected crc 2023-06-10T21:28:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/439572426' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-10T21:28:09.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:09 smithi152 ceph-mon[156819]: osdmap e238: 8 total, 6 up, 6 in 2023-06-10T21:28:09.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:09.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:09.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:09.669 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:10.013 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:10.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:10.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:10.146 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:10.252 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:28:10.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:10.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:10.450 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:10.478 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:10.538 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:10.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:10.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:10.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:10.674 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:10.825 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:10.825 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:11.124 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:11.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:11.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:11.258 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:11.337 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:11.355 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:11.373 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:11.391 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:11.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: pgmap v842: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:11.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 47980/209829 objects degraded (22.866%), 19 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:11.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:11.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: failed to encode map e239 with expected crc 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[186706]: osdmap e239: 8 total, 6 up, 6 in 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: pgmap v842: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 47980/209829 objects degraded (22.866%), 19 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:11.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: failed to encode map e239 with expected crc 2023-06-10T21:28:11.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:11.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:11 smithi064 ceph-mon[180297]: osdmap e239: 8 total, 6 up, 6 in 2023-06-10T21:28:11.409 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:11.426 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:11.426 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.427 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:28:11.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: pgmap v842: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:11.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 47980/209829 objects degraded (22.866%), 19 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:11.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:11.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: failed to encode map e239 with expected crc 2023-06-10T21:28:11.539 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4173619398' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:11.540 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:11 smithi152 ceph-mon[156819]: osdmap e239: 8 total, 6 up, 6 in 2023-06-10T21:28:11.551 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:11.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:11.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:28:11.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:11.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:11.690 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:11.739 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:11.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:11.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:11.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:11.873 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:11.973 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:12.020 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:12.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:12.108 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:12.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:12.155 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:12.200 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:12.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:12.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:12.337 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2607641137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[186706]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[186706]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[186706]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2607641137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[180297]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[180297]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:12 smithi064 ceph-mon[180297]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2607641137' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:12 smithi152 ceph-mon[156819]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:12 smithi152 ceph-mon[156819]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:12 smithi152 ceph-mon[156819]: failed to encode map e239 with expected crc 2023-06-10T21:28:12.581 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:12.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:12.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:12.713 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:12.754 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:12.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:12.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:12.884 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:13.063 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:13.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:13.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:12] "GET /metrics HTTP/1.1" 200 224072 "" "Prometheus/2.33.4" 2023-06-10T21:28:13.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:13.184 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:28:13.202 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:13.296 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:13.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:13.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:13.428 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:13.471 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:13.472 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:13 smithi152 ceph-mon[156819]: pgmap v844: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:13.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:13.508 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:28:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:28:13.508 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166807 9.4 9:2480dc05:::benchmark_data_smithi064_247471_object57819:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 268.18048580999999, 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040999767, 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.509 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166807 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:28:13.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322453+0000", 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9229999999999996e-06 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322460+0000", 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9490000000000008e-06 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322470+0000", 2023-06-10T21:28:13.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2830000000000004e-06 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322478+0000", 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003841133 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326319+0000", 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9380000000000004e-06 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326329+0000", 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0051517590000001 2023-06-10T21:28:13.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331480+0000", 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1434000000000006e-05 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331572+0000", 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031794995 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363367+0000", 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8282999999999995e-05 2023-06-10T21:28:13.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363455+0000", 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040999767 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166814 9.4 9:21a2952b:::benchmark_data_smithi064_247471_object57826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 268.18044949099999, 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0414135449999999, 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.519 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166814 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:13.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322491+0000", 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.658e-06 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322495+0000", 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9670000000000004e-06 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322502+0000", 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.096e-06 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322507+0000", 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038619230000000002 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326369+0000", 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7579999999999998e-06 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326377+0000", 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0052419239999999 2023-06-10T21:28:13.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331619+0000", 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6457000000000002e-05 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331716+0000", 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3530999999999996e-05 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331809+0000", 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023985312000000002 2023-06-10T21:28:13.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355795+0000", 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079406909999999997 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363735+0000", 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1652999999999997e-05 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363767+0000", 2023-06-10T21:28:13.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5206000000000001e-05 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363812+0000", 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2964e-05 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363905+0000", 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0414135449999999 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.531 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166820 9.4 9:2fd87fc4:::benchmark_data_smithi064_247471_object57832:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 268.17825946099998, 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.039453033, 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.532 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166820 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:28:13.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324680+0000", 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4710000000000004e-06 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324687+0000", 2023-06-10T21:28:13.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6809999999999993e-06 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324696+0000", 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9989999999999999e-06 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324701+0000", 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017058240000000001 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326407+0000", 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9110000000000004e-06 2023-06-10T21:28:13.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326414+0000", 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.008438868 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.334853+0000", 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9625999999999998e-05 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.334953+0000", 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8136999999999999e-05 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.335051+0000", 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020754058999999998 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355805+0000", 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081369440000000001 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363942+0000", 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4404000000000002e-05 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364006+0000", 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1557999999999998e-05 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364048+0000", 2023-06-10T21:28:13.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6891000000000005e-05 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364135+0000", 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.039453033 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.543 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166852 9.4 9:2c2b0a22:::benchmark_data_smithi064_247471_object57864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 268.12065605800001, 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98200426799999996, 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166852 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:28:13.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382283+0000", 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5140000000000007e-06 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382293+0000", 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5099999999999999e-05 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382308+0000", 2023-06-10T21:28:13.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4240000000000004e-06 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382315+0000", 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1654000000000001e-05 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382397+0000", 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1015999999999999e-05 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382408+0000", 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95501303800000004 2023-06-10T21:28:13.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337421+0000", 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0987000000000003e-05 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337492+0000", 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0113999999999998e-05 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337582+0000", 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018232515000000001 2023-06-10T21:28:13.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355815+0000", 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081570289999999997 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363972+0000", 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019824 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364170+0000", 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6148e-05 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364206+0000", 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3392000000000004e-05 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364289+0000", 2023-06-10T21:28:13.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98200426799999996 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166875 9.4 9:2e6a97c5:::benchmark_data_smithi064_247471_object57887:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.556 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.96646544700002, 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82800355000000003, 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.557 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166875 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536473+0000", 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4477999999999999e-05 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536488+0000", 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7987000000000002e-05 2023-06-10T21:28:13.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536516+0000", 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9879999999999999e-06 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536526+0000", 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4812e-05 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536610+0000", 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2381999999999999e-05 2023-06-10T21:28:13.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536623+0000", 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.803186336 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339809+0000", 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143893 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339953+0000", 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146976 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.340100+0000", 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015724806000000001 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355825+0000", 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084980560000000004 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364323+0000", 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8036999999999999e-05 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364351+0000", 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7922e-05 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364389+0000", 2023-06-10T21:28:13.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0325e-05 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364479+0000", 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82800355000000003 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.568 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166881 9.4 9:2065f492:::benchmark_data_smithi064_247471_object57893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.95180185599997, 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81353793299999999, 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.569 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166881 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:28:13.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551137+0000", 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4239999999999993e-06 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551146+0000", 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6155e-05 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551162+0000", 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8860000000000004e-06 2023-06-10T21:28:13.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551170+0000", 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0584000000000001e-05 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551231+0000", 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0097e-05 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551241+0000", 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79565823499999999 2023-06-10T21:28:13.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346899+0000", 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010809700000000001 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.347007+0000", 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2709999999999999e-05 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.347090+0000", 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016244689 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363334+0000", 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001185237 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364520+0000", 2023-06-10T21:28:13.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0880000000000002e-05 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364550+0000", 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8509e-05 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364589+0000", 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8195000000000005e-05 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364677+0000", 2023-06-10T21:28:13.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81353793299999999 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166885 9.4 9:2f426375:::benchmark_data_smithi064_247471_object57897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:28:13.581 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.946937859, 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80886061200000003, 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166885 2023-06-10T21:28:13.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556001+0000", 2023-06-10T21:28:13.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.028e-05 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556012+0000", 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1024999999999999e-05 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556033+0000", 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0660000000000004e-06 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556041+0000", 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0553e-05 2023-06-10T21:28:13.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556111+0000", 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2969999999999999e-05 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556124+0000", 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79310488000000001 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349229+0000", 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110064 2023-06-10T21:28:13.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349339+0000", 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130178 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349469+0000", 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013871082 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363340+0000", 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001378514 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364719+0000", 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7698000000000001e-05 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364747+0000", 2023-06-10T21:28:13.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5630999999999998e-05 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364782+0000", 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1612000000000004e-05 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364864+0000", 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80886061200000003 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166889 9.4 9:208e17d2:::benchmark_data_smithi064_247471_object57901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.94193114199999, 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81055013600000003, 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166889 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:28:13.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561008+0000", 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9509999999999995e-06 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561017+0000", 2023-06-10T21:28:13.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6741000000000001e-05 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561034+0000", 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5610000000000004e-06 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561041+0000", 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8444999999999999e-05 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561109+0000", 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2872e-05 2023-06-10T21:28:13.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561122+0000", 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79092483199999997 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352047+0000", 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111666 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352159+0000", 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130674 2023-06-10T21:28:13.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352289+0000", 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011159099 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363448+0000", 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014472140000000001 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364896+0000", 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064686250000000004 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371364+0000", 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4372e-05 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371429+0000", 2023-06-10T21:28:13.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131564 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371560+0000", 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81055013600000003 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.605 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166904 9.4 9:2d035138:::benchmark_data_smithi064_247471_object57916:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.91491722699999, 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78375932999999998, 2023-06-10T21:28:13.606 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166904 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588022+0000", 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0492999999999999e-05 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588032+0000", 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0548e-05 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588053+0000", 2023-06-10T21:28:13.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8210000000000002e-06 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588061+0000", 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2191000000000001e-05 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588123+0000", 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1460000000000002e-06 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588132+0000", 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76775603000000003 2023-06-10T21:28:13.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355888+0000", 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014207900000000001 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.356030+0000", 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016228099999999999 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.356193+0000", 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072615379999999997 2023-06-10T21:28:13.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363454+0000", 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014683350000000001 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364922+0000", 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067034099999999999 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371626+0000", 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8415000000000002e-05 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371684+0000", 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8995999999999999e-05 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371783+0000", 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78375932999999998 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166915 9.4 9:2e5c8b28:::benchmark_data_smithi064_247471_object57927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:28:13.618 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 267.87153657200002, 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74064215499999997, 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.619 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166915 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631403+0000", 2023-06-10T21:28:13.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9249999999999995e-06 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631410+0000", 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4515000000000001e-05 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631425+0000", 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8109999999999999e-06 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631431+0000", 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8529000000000002e-05 2023-06-10T21:28:13.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631479+0000", 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4639999999999995e-06 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631489+0000", 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72782496200000002 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359314+0000", 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112879 2023-06-10T21:28:13.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359427+0000", 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013150099999999999 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359558+0000", 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051280220000000003 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364686+0000", 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071560779999999997 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371842+0000", 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4267000000000001e-05 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371896+0000", 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4815999999999999e-05 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371941+0000", 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010541500000000001 2023-06-10T21:28:13.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.372047+0000", 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74064215499999997 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:209425 9.4 9:2f96b396:::benchmark_data_smithi064_247471_object100437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 193.05051735200001, 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.237427307, 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.631 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 209425 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452422+0000", 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1479999999999995e-06 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452431+0000", 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8559e-05 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452449+0000", 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9000000000000006e-06 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452457+0000", 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0895e-05 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452508+0000", 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105846 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452614+0000", 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121283 2023-06-10T21:28:13.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452735+0000", 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044428790000000003 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.457178+0000", 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114129 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.457292+0000", 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.232459255 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689752+0000", 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5896999999999997e-05 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689808+0000", 2023-06-10T21:28:13.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3510999999999997e-05 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689851+0000", 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.237427307 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.641 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(10.f 227/220 [PushOp(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head, version: 200'380, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head@200'380, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 53.988789584999999, 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001, 2023-06-10T21:28:13.642 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514151+0000", 2023-06-10T21:28:13.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5096999999999998e-05 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514206+0000", 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5574e-05 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514222+0000", 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0900000000000005e-06 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514230+0000", 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036878129999999999 2023-06-10T21:28:13.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517918+0000", 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0910000000000003e-06 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517922+0000", 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.231370187 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.749292+0000", 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001 2023-06-10T21:28:13.648 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:96995 10.f e227/220 10:f95a89c6:::benchmark_data_smithi064_312713_object6239:head v 227'5921, mlcod=200'373)", 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 53.984474691999999, 2023-06-10T21:28:13.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001, 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1389999999999998e-06 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518473+0000", 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0293e-05 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518483+0000", 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6609999999999997e-06 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518489+0000", 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23332091099999999 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751810+0000", 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9099999999999999e-05 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751839+0000", 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027593894000000001 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779433+0000", 2023-06-10T21:28:13.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1985999999999999e-05 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779455+0000", 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.656 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:97001 10.f e227/220 10:fd7a027e:::benchmark_data_smithi064_312713_object6245:head v 227'5922, mlcod=200'373)", 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 53.980445193000001, 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002, 2023-06-10T21:28:13.657 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522495+0000", 2023-06-10T21:28:13.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5910000000000001e-06 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522502+0000", 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22114999699999999 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743652+0000", 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4776999999999999e-05 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743667+0000", 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013856040999999999 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757523+0000", 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8875999999999999e-05 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757551+0000", 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021906202999999999 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779458+0000", 2023-06-10T21:28:13.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9106e-05 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779477+0000", 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.664 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111479 10.6 10:610f9587:::benchmark_data_smithi064_312713_object20723:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.190588335000001, 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999, 2023-06-10T21:28:13.665 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111479 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:28:13.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312351+0000", 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1720000000000004e-06 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312358+0000", 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312370+0000", 2023-06-10T21:28:13.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4580000000000004e-06 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312378+0000", 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024553309999999998 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314833+0000", 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314842+0000", 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0343133339999999 2023-06-10T21:28:13.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.349155+0000", 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001791864 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350947+0000", 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111480 10.d 10:bb138473:::benchmark_data_smithi064_312713_object20724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:28:13.673 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.190555031999999, 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999, 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.674 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111480 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.802e-06 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312388+0000", 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9600000000000002e-06 2023-06-10T21:28:13.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312392+0000", 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7259999999999998e-06 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312395+0000", 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7097000000000001e-05 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312432+0000", 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5759999999999996e-06 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312438+0000", 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.019375897 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331814+0000", 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6889000000000003e-05 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331881+0000", 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999 2023-06-10T21:28:13.681 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111483 10.d 10:b3a132fc:::benchmark_data_smithi064_312713_object20727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.181628594000003, 2023-06-10T21:28:13.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817, 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111483 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.683 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321311+0000", 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1680000000000003e-06 2023-06-10T21:28:13.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321316+0000", 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2384999999999999e-05 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321329+0000", 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9739999999999999e-06 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321335+0000", 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069082699999999998 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322026+0000", 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2101e-05 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322038+0000", 2023-06-10T21:28:13.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.009867013 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331905+0000", 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2904000000000005e-05 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331978+0000", 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039655499999999999 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 6", 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.332374+0000", 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011245184 2023-06-10T21:28:13.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343619+0000", 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7795999999999998e-05 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343707+0000", 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0745000000000003e-05 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343758+0000", 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9522e-05 2023-06-10T21:28:13.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343857+0000", 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111488 10.4 10.41ecfd74 (undecoded) ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.171208131999997, 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999, 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:13.694 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111488 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331731+0000", 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7428e-05 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:28:13.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4331999999999999e-05 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331763+0000", 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5490000000000003e-06 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331769+0000", 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9897e-05 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331819+0000", 2023-06-10T21:28:13.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0000000000000002e-06 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331825+0000", 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0023298279999999 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334155+0000", 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0281e-05 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334185+0000", 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999 2023-06-10T21:28:13.701 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111491 10.6 10:69475ff3:::benchmark_data_smithi064_312713_object20735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.152995881000002, 2023-06-10T21:28:13.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693, 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111491 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.703 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349943+0000", 2023-06-10T21:28:13.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5360000000000004e-06 2023-06-10T21:28:13.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349951+0000", 2023-06-10T21:28:13.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4100999999999999e-05 2023-06-10T21:28:13.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349965+0000", 2023-06-10T21:28:13.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2029999999999997e-06 2023-06-10T21:28:13.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349971+0000", 2023-06-10T21:28:13.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4274000000000002e-05 2023-06-10T21:28:13.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350035+0000", 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9600000000000006e-06 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350044+0000", 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000890976 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350935+0000", 2023-06-10T21:28:13.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051541900000000003 2023-06-10T21:28:13.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351451+0000", 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029125 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:28:13.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351742+0000", 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098297039999999999 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.361571+0000", 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025176170000000002 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364089+0000", 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5660000000000001e-05 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364125+0000", 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4188000000000001e-05 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364209+0000", 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693 2023-06-10T21:28:13.716 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111494 10.6 10:60c1325c:::benchmark_data_smithi064_312713_object20738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:28:13.717 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 32.136768289000003, 2023-06-10T21:28:13.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013, 2023-06-10T21:28:13.718 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:13.718 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:13.718 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:13.718 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:13.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:13.719 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111494 2023-06-10T21:28:13.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.719 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:13.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:13.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:28:13.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:13.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366171+0000", 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9200000000000001e-06 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366177+0000", 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:28:13.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366190+0000", 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9599999999999997e-06 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366196+0000", 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6909000000000001e-05 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366243+0000", 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4109999999999996e-06 2023-06-10T21:28:13.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366251+0000", 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.989107077 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:13.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355358+0000", 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102029 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355460+0000", 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2402000000000001e-05 2023-06-10T21:28:13.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355553+0000", 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086816740000000003 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:13.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364234+0000", 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003252605 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367487+0000", 2023-06-10T21:28:13.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8502999999999999e-05 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367546+0000", 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7332999999999998e-05 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:13.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367643+0000", 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:13.731 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:13.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:13.733 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:13.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:13 smithi064 ceph-mon[180297]: pgmap v844: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:13.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:13 smithi064 ceph-mon[186706]: pgmap v844: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 9.7 MiB/s, 154 objects/s recovering 2023-06-10T21:28:13.759 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:13.802 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:13.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:13.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:13.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:13.931 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:14.064 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3, 0] live_osds: [1, 2, 4, 5, 6, 7] 2023-06-10T21:28:14.064 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:28:14.064 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-06-10T21:28:14.064 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:28:14.065 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:28:14.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-06-10T21:28:14.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:28:14.285 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:14.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:14.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:28:14.393 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:14.422 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:14.425 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:14.469 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:14.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:14.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:14.601 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:14.696 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:14.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:14.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:14.711 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:14.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:14.828 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:15.123 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:15.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:15.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:15.254 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:15 smithi152 ceph-mon[156819]: pgmap v845: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:15.458 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:15 smithi064 ceph-mon[186706]: pgmap v845: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:15.459 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:15 smithi064 ceph-mon[180297]: pgmap v845: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:15.505 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:15.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:15.607 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:15.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:15.641 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:15.941 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:15.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:15.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:16.034 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:16.034 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:28:16.034 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:16.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:16.075 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:16.175 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:16.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:16.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:16.263 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:28:16.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:16.309 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:16.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:16.413 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:16.426 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:16.426 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:16.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:16.468 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:16.501 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:16.518 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:16.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:16.522 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:16.522 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:28:16.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:16.651 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:16.963 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:16.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:16.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:17.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:17.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:17.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: pgmap v846: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:17.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:17.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: osdmap e240: 8 total, 6 up, 6 in 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3283822958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:28:17.269 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: pgmap v846: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: osdmap e240: 8 total, 6 up, 6 in 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3283822958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:17.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:28:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:17.414 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:17.424 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:17.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:17.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:17.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:17.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: pgmap v846: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%); 440 KiB/s, 6 objects/s recovering 2023-06-10T21:28:17.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:17.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: failed to encode map e240 with expected crc 2023-06-10T21:28:17.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2786563385' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:17.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: osdmap e240: 8 total, 6 up, 6 in 2023-06-10T21:28:17.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3283822958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:17.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:28:17.555 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:17.707 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:17.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:17.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:17.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:17.841 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:17.959 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:17.960 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:17.989 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:18.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:18.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:18.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:18.134 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:18.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:18.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:18.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:18.319 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[186706]: failed to encode map e240 with expected crc 2023-06-10T21:28:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[180297]: failed to encode map e240 with expected crc 2023-06-10T21:28:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:18 smithi152 ceph-mon[156819]: failed to encode map e240 with expected crc 2023-06-10T21:28:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:18.882 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:28:18.883 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:18.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:18.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:19.035 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:19.080 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:19.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:19.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:19.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:19.214 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:19.258 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:19.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:19.286 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:28:19.311 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:19 smithi064 ceph-mon[186706]: pgmap v848: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:19.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:19 smithi064 ceph-mon[180297]: pgmap v848: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:19.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:19.391 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:19.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:28:19.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:19.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:19.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:19.570 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:19.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:19.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:19 smithi152 ceph-mon[156819]: pgmap v848: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:19.784 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:19.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:19.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:19.920 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:20.014 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:20.064 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:20.165 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:20.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:20.296 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:20.998 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:21.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:21.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:21.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:21.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:21.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:21.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:21.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:21.375 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[180297]: pgmap v849: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[186706]: pgmap v849: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.469 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:21.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:21.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:21.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:21.637 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:21.638 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:21.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:21.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:21 smithi152 ceph-mon[156819]: pgmap v849: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:21.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:21.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:21.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:21.792 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:21.835 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:21.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:21.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:21.962 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [3, 0] live_osds: [1, 2, 4, 5, 6, 7] 2023-06-10T21:28:21.963 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2023-06-10T21:28:21.963 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2023-06-10T21:28:21.963 DEBUG:teuthology.orchestra.run.smithi064:osd.0> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0.service 2023-06-10T21:28:21.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl start ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0 2023-06-10T21:28:21.967 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:21.996 INFO:journalctl@ceph.osd.0.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T21:28:22.300 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:22 smithi064 systemd[1]: Starting Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:28:22.302 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:22.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:22.310 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:22.312 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:28:22.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:22.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:22.436 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:22.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:28:22.465 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:22.616 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:22.628 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:22.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:22.730 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:22.764 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:22.910 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:22.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:23.014 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:23.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:23.053 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:22] "GET /metrics HTTP/1.1" 200 224028 "" "Prometheus/2.33.4" 2023-06-10T21:28:23.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:23.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:23.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:23.427 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:23.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:23.466 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:23.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:23.486 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:23.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: pgmap v850: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:23.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:23.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.499 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:23.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: osdmap e241: 8 total, 6 up, 6 in 2023-06-10T21:28:23.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.500 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.502 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:23.504 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:23.522 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:23.522 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.523 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:28:23.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: pgmap v850: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:23.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:23.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: osdmap e241: 8 total, 6 up, 6 in 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: pgmap v850: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:23.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1289092583' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: osdmap e241: 8 total, 6 up, 6 in 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: failed to encode map e241 with expected crc 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.543 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:23.648 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:23.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:23.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:23.779 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:24.041 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:24.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:24.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:24.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:24.176 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:24.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:24.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:24.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:24.356 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:24.404 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:24.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:24.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:24.537 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:24.582 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:24.583 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:28:24.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:28:24.583 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:28:24.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109295 9.9 e141/127 9:9b3a09da:::benchmark_data_smithi064_247471_object307:head v 141'6783, mlcod=141'6781)", 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67912869700001, 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998, 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.584 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:28:24.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898799+0000", 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898798+0000", 2023-06-10T21:28:24.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2339999999999997e-06 2023-06-10T21:28:24.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898805+0000", 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5519999999999993e-06 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898815+0000", 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2270000000000001e-06 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898820+0000", 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4044e-05 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898864+0000", 2023-06-10T21:28:24.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9573e-05 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.898884+0000", 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35487087899999997 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253755+0000", 2023-06-10T21:28:24.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3262000000000001e-05 2023-06-10T21:28:24.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253788+0000", 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35498897499999998 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109297 9.5 e141/127 9:a2279825:::benchmark_data_smithi064_247471_object309:head v 141'6825, mlcod=141'6823)", 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67871187700001, 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998, 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.594 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899216+0000", 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899215+0000", 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8650000000000003e-06 2023-06-10T21:28:24.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899222+0000", 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899232+0000", 2023-06-10T21:28:24.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6350000000000003e-06 2023-06-10T21:28:24.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899237+0000", 2023-06-10T21:28:24.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0815000000000001e-05 2023-06-10T21:28:24.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899278+0000", 2023-06-10T21:28:24.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1216000000000002e-05 2023-06-10T21:28:24.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.899299+0000", 2023-06-10T21:28:24.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.349632361 2023-06-10T21:28:24.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248932+0000", 2023-06-10T21:28:24.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100882 2023-06-10T21:28:24.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249032+0000", 2023-06-10T21:28:24.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34981669199999998 2023-06-10T21:28:24.606 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.607 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.608 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109299 9.7 9:e2e5e039:::benchmark_data_smithi064_247471_object311:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:28:24.608 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:28:24.608 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67579792200002, 2023-06-10T21:28:24.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998, 2023-06-10T21:28:24.609 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.609 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.610 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.610 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.610 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109299 2023-06-10T21:28:24.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.611 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:28:24.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902130+0000", 2023-06-10T21:28:24.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:24.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902128+0000", 2023-06-10T21:28:24.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9909999999999996e-06 2023-06-10T21:28:24.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902135+0000", 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0420000000000008e-06 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902144+0000", 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6470000000000003e-06 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902150+0000", 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6499000000000003e-05 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902197+0000", 2023-06-10T21:28:24.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011322100000000001 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012462399999999999 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902434+0000", 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046768010000000004 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907111+0000", 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6680000000000004e-06 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907118+0000", 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34198863299999999 2023-06-10T21:28:24.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249107+0000", 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102996 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249210+0000", 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4842000000000002e-05 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249254+0000", 2023-06-10T21:28:24.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34712465399999998 2023-06-10T21:28:24.623 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.623 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109302 9.a 9:593e78fe:::benchmark_data_smithi064_247471_object314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:28:24.624 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.675616472, 2023-06-10T21:28:24.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999, 2023-06-10T21:28:24.625 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.625 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109302 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902311+0000", 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902310+0000", 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5360000000000003e-06 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902315+0000", 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.669e-06 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902322+0000", 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9060000000000004e-06 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.902325+0000", 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074886199999999999 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903074+0000", 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010393800000000001 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903178+0000", 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8347000000000001e-05 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903247+0000", 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.345750055 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248997+0000", 2023-06-10T21:28:24.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000337282 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249334+0000", 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6821e-05 2023-06-10T21:28:24.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249361+0000", 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7002000000000002e-05 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249408+0000", 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8299e-05 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249446+0000", 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34713491099999999 2023-06-10T21:28:24.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109305 9.6 9:68c28d89:::benchmark_data_smithi064_247471_object317:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:28:24.636 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67487445699999, 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997, 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109305 2023-06-10T21:28:24.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903053+0000", 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903052+0000", 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:28:24.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903058+0000", 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0450000000000006e-06 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903067+0000", 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8790000000000002e-06 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903071+0000", 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8942e-05 2023-06-10T21:28:24.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903120+0000", 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100547 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903221+0000", 2023-06-10T21:28:24.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000109324 2023-06-10T21:28:24.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:28:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.903330+0000", 2023-06-10T21:28:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34556609100000002 2023-06-10T21:28:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248896+0000", 2023-06-10T21:28:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042370840000000003 2023-06-10T21:28:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253133+0000", 2023-06-10T21:28:24.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3943000000000006e-05 2023-06-10T21:28:24.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253217+0000", 2023-06-10T21:28:24.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3278999999999997e-05 2023-06-10T21:28:24.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253261+0000", 2023-06-10T21:28:24.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3470000000000001e-05 2023-06-10T21:28:24.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253314+0000", 2023-06-10T21:28:24.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35026092599999997 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.651 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109296 9.9 e141/127 9:9d3096da:::benchmark_data_smithi064_247471_object308:head v 141'6784, mlcod=141'6781)", 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67316417799998, 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001, 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904763+0000", 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3180000000000003e-06 2023-06-10T21:28:24.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904769+0000", 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0399999999999993e-06 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904777+0000", 2023-06-10T21:28:24.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6299999999999997e-06 2023-06-10T21:28:24.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904782+0000", 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115028 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904897+0000", 2023-06-10T21:28:24.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8039000000000001e-05 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904915+0000", 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34887546699999999 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253790+0000", 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5639999999999999e-05 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.253806+0000", 2023-06-10T21:28:24.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34904265600000001 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109303 9.b e141/127 9:d16859ff:::benchmark_data_smithi064_247471_object315:head v 141'6884, mlcod=141'6882)", 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:28:24.660 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67313429799998, 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885, 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904793+0000", 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.456e-06 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904797+0000", 2023-06-10T21:28:24.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9980000000000003e-06 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904802+0000", 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0400000000000001e-06 2023-06-10T21:28:24.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904805+0000", 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34422613600000002 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249031+0000", 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7886000000000001e-05 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249089+0000", 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078164589999999996 2023-06-10T21:28:24.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256905+0000", 2023-06-10T21:28:24.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2179999999999999e-05 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.256947+0000", 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352153885 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.669 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109304 9.5 e141/127 9:a38538a9:::benchmark_data_smithi064_247471_object316:head v 141'6826, mlcod=141'6823)", 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67311299800002, 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202, 2023-06-10T21:28:24.670 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:28:24.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904815+0000", 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904814+0000", 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8880000000000002e-06 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904820+0000", 2023-06-10T21:28:24.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.994e-06 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904825+0000", 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.965e-06 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.904828+0000", 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002189528 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907018+0000", 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3083e-05 2023-06-10T21:28:24.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907031+0000", 2023-06-10T21:28:24.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34204052800000001 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249071+0000", 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7423000000000003e-05 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249119+0000", 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.344304202 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35400.0:109300 9.9 e141/127 9:94ae4f7e:::benchmark_data_smithi064_247471_object312:head v 141'6785, mlcod=141'6781)", 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.67012396400003, 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001, 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907804+0000", 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907803+0000", 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.772e-06 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.907812+0000", 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34109469599999998 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248906+0000", 2023-06-10T21:28:24.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1025999999999997e-05 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248937+0000", 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016277199999999999 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249100+0000", 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4561999999999997e-05 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249155+0000", 2023-06-10T21:28:24.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083045289999999997 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257459+0000", 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8495000000000002e-05 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257508+0000", 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34970407100000001 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.686 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:109307 9.f 9:f866cb96:::benchmark_data_smithi064_247471_object319:head [delete] snapc 0=[] ondisk+write+known_if_redirected e141)", 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.66953775299999, 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092, 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 109307 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908390+0000", 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:24.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908388+0000", 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4800000000000004e-06 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.908396+0000", 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.34054057399999998 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248936+0000", 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1486000000000001e-05 2023-06-10T21:28:24.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248958+0000", 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032944700000000002 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249287+0000", 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101111 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249388+0000", 2023-06-10T21:28:24.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123366 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249512+0000", 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077532640000000002 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257265+0000", 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045560400000000002 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.257721+0000", 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003373905 2023-06-10T21:28:24.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261094+0000", 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6616000000000002e-05 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261141+0000", 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6924000000000001e-05 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.261178+0000", 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.352788092 2023-06-10T21:28:24.697 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.35400.0:109299 9.7 e141/127 ondisk, result = 0)", 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 361.66657389599999, 2023-06-10T21:28:24.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999, 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911354+0000", 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911353+0000", 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7229999999999998e-06 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:22.911359+0000", 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33752210999999999 2023-06-10T21:28:24.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248882+0000", 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6460000000000001e-05 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.248908+0000", 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015431399999999999 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249062+0000", 2023-06-10T21:28:24.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2586999999999999e-05 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249085+0000", 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017446900000000001 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:22:23.249259+0000", 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33790566599999999 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.705 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137750 9.f 9:fe4ed14c:::benchmark_data_smithi064_247471_object28762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 322.14941405600001, 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768, 2023-06-10T21:28:24.706 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137750 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.707 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428514+0000", 2023-06-10T21:28:24.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428513+0000", 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.377e-06 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428516+0000", 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1229999999999997e-06 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428523+0000", 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8249999999999998e-06 2023-06-10T21:28:24.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.428526+0000", 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00081994200000000004 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429346+0000", 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2603999999999996e-05 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429439+0000", 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104776 2023-06-10T21:28:24.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.429544+0000", 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049570079999999997 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.434501+0000", 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018088150000000001 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436310+0000", 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30228024799999997 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738590+0000", 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2488999999999998e-05 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738642+0000", 2023-06-10T21:28:24.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8994e-05 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738681+0000", 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31016768 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.717 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137758 9.7 9:e64fca9d:::benchmark_data_smithi064_247471_object28770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 322.14154981399997, 2023-06-10T21:28:24.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999, 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137758 2023-06-10T21:28:24.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436378+0000", 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436377+0000", 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2279999999999998e-06 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436382+0000", 2023-06-10T21:28:24.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4789999999999996e-06 2023-06-10T21:28:24.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436390+0000", 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6550000000000003e-06 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436395+0000", 2023-06-10T21:28:24.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5500000000000001e-05 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436441+0000", 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105098 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436546+0000", 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011236899999999999 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436658+0000", 2023-06-10T21:28:24.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30189442999999999 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738552+0000", 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000176961 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738729+0000", 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5865999999999997e-05 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738785+0000", 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5071999999999999e-05 2023-06-10T21:28:24.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738820+0000", 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1973e-05 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738852+0000", 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30247453699999999 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.730 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137759 9.7 9:e8bc7c80:::benchmark_data_smithi064_247471_object28771:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 322.14152155699998, 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001, 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137759 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436406+0000", 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8289999999999999e-06 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436409+0000", 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5740000000000003e-06 2023-06-10T21:28:24.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436414+0000", 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7810000000000005e-06 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.436422+0000", 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004238187 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440660+0000", 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117327 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440777+0000", 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012919200000000001 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:28:24.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.440907+0000", 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.297783668 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738690+0000", 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8073999999999999e-05 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738758+0000", 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000156365 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738915+0000", 2023-06-10T21:28:24.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6863000000000003e-05 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738952+0000", 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8658999999999997e-05 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.738990+0000", 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30258411800000001 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.741 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:137770 9.7 9:e3c55b93:::benchmark_data_smithi064_247471_object28782:head [delete] snapc 0=[] ondisk+write+known_if_redirected e152)", 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 322.13292288399998, 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009, 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.742 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 137770 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445005+0000", 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445004+0000", 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.569e-06 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445008+0000", 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1970000000000001e-06 2023-06-10T21:28:24.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445014+0000", 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9210000000000002e-06 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.445018+0000", 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28887405700000002 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.733892+0000", 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146957 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734039+0000", 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012832300000000001 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.734167+0000", 2023-06-10T21:28:24.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049619210000000002 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.739129+0000", 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047693650000000002 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.743899+0000", 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011874 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744017+0000", 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8343e-05 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744076+0000", 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9201e-05 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:02.744115+0000", 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29911009 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199845 9.f 9:ff71ad38:::benchmark_data_smithi064_247471_object90857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 223.053536498, 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001, 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:24.753 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199845 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524391+0000", 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524389+0000", 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524398+0000", 2023-06-10T21:28:24.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2369e-05 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524410+0000", 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8360000000000003e-06 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.524417+0000", 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084813649999999994 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532898+0000", 2023-06-10T21:28:24.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114187 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533013+0000", 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118496 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.533131+0000", 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050821640000000001 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.538213+0000", 2023-06-10T21:28:24.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087860630000000002 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.546999+0000", 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28852924899999999 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835529+0000", 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4122999999999998e-05 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835573+0000", 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9298999999999998e-05 2023-06-10T21:28:24.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.835612+0000", 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31122091800000001 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201393 9.5 9:a576e4a0:::benchmark_data_smithi064_247471_object92405:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 220.45487755600001, 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001, 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.764 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201393 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123050+0000", 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123049+0000", 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7429999999999999e-06 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123055+0000", 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6000000000000007e-06 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123064+0000", 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9880000000000001e-06 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123070+0000", 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019396100000000001 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123263+0000", 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018801169999999999 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125144+0000", 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000161557 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.125305+0000", 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023599202 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148904+0000", 2023-06-10T21:28:24.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6900000000000004e-06 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148914+0000", 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593960899999999 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124854+0000", 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2537999999999999e-05 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124866+0000", 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0018160970000001 2023-06-10T21:28:24.773 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201395 9.5 9:a9ebfe85:::benchmark_data_smithi064_247471_object92407:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 220.45458277200001, 2023-06-10T21:28:24.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999, 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201395 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.775 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123345+0000", 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123344+0000", 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0849999999999996e-06 2023-06-10T21:28:24.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123349+0000", 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.354e-06 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123357+0000", 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4389999999999999e-06 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.123363+0000", 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011364407 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134727+0000", 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118065 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.134845+0000", 2023-06-10T21:28:24.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000154458 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.135000+0000", 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013944756000000001 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148945+0000", 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9670000000000007e-06 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.148954+0000", 2023-06-10T21:28:24.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97593688599999995 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124890+0000", 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2764e-05 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124903+0000", 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0015583109999999 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.784 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201403 9.5 9.ddfae635 (undecoded) ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 220.444477581, 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997, 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.785 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201403 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133450+0000", 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9119999999999998e-06 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133453+0000", 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5379999999999999e-06 2023-06-10T21:28:24.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133459+0000", 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0280000000000002e-06 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.133463+0000", 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020130169999999999 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153593+0000", 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4200000000000004e-06 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.153600+0000", 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97132188100000005 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124922+0000", 2023-06-10T21:28:24.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2189999999999998e-06 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124929+0000", 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99147873399999997 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.792 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:201415 9.5 9.ce5b1865 (undecoded) ondisk+write+known_if_redirected e172)", 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 220.411425103, 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998, 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:24.793 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 201415 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166503+0000", 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:24.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166501+0000", 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0360000000000003e-06 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166507+0000", 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8352999999999999e-05 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166526+0000", 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7720000000000002e-06 2023-06-10T21:28:24.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166531+0000", 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5688e-05 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166576+0000", 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8590000000000001e-06 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:44.166582+0000", 2023-06-10T21:28:24.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95836468200000002 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124947+0000", 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3220000000000004e-06 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:24.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:45.124954+0000", 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95845161499999998 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:24.801 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:24.802 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:24.802 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:24.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/981952367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:24.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:28:24.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:28:24.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:24.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/981952367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:24.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:28:24.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:28:24.804 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:24.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:24.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/981952367' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:24.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:28:24.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:28:24.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:24.806 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:24.807 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:24.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:24.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-10T21:28:24.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-10T21:28:24.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-10T21:28:24.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-10T21:28:24.908 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2023-06-10T21:28:24.909 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-10T21:28:24.909 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:24 smithi064 bash[347375]: --> ceph-volume raw activate successful for osd ID: 0 2023-06-10T21:28:24.937 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:25.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:28:25.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:28:25.263 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:25.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:25.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:25.396 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:25.439 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:25.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:25.493 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:25.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:25.568 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:25 smithi152 ceph-mon[156819]: pgmap v852: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:25.571 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:25.621 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:25.671 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:25.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:25.743 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:25 smithi064 ceph-mon[180297]: pgmap v852: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:25.743 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:25 smithi064 ceph-mon[186706]: pgmap v852: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:25.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:25.801 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:25.846 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:25.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:25.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:25.977 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:26.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:28:26.157 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:25 smithi064 ceph-osd[348311]: -- 172.21.15.64:0/4166023382 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55be990344e0 con 0x55be99007000 2023-06-10T21:28:26.158 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:26 smithi064 bash[348176]: 8f85d9362d65f1adcef68ddeb88d835f35595eb1ddb5c97bc5612409b184281b 2023-06-10T21:28:26.158 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:26 smithi064 systemd[1]: Started Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:28:26.222 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:26.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:26.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:26.355 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:26.657 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:26 smithi064 conmon[348287]: 2023-06-10T21:28:26.412+0000 7f84aec26640 -1 Falling back to public interface 2023-06-10T21:28:26.743 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:26.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:26.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:28:26.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:26.882 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:26.888 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:27.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:27.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:27.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:27.096 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:27.096 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:28:27.097 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:28:27.097 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:27.141 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:27.173 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:27.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:27.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:27.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:27.455 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:27.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:27 smithi064 ceph-mon[186706]: pgmap v853: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:27.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:27 smithi064 ceph-mon[180297]: pgmap v853: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:27.754 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:27.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:27.829 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:27 smithi152 ceph-mon[156819]: pgmap v853: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:27.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:28:27.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:27.918 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:27.961 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:27.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:28.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:28.098 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:28.142 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:28.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:28.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:28.194 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:28.248 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:28.262 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:28.281 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:28.453 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:28.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:28.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:28.467 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:28:28.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:28.585 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:28.680 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:28.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:28.709 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:28.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:28.813 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:28.909 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:28.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:29.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:29.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:29.072 INFO:teuthology.orchestra.run.smithi064.stderr:no valid command found; 10 closest matches: 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:0 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:1 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:2 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:abort 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:assert 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs debug_inject_read_zeros 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs files list 2023-06-10T21:28:29.073 INFO:teuthology.orchestra.run.smithi064.stderr:bluefs stats 2023-06-10T21:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:bluestore bluefs device info [] 2023-06-10T21:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:config diff 2023-06-10T21:28:29.074 INFO:teuthology.orchestra.run.smithi064.stderr:admin_socket: invalid command 2023-06-10T21:28:29.095 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:29.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:29.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:29.230 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:29.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: pgmap v854: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:29.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:29.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:29.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: osdmap e242: 8 total, 6 up, 6 in 2023-06-10T21:28:29.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[186706]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: pgmap v854: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:29.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: osdmap e242: 8 total, 6 up, 6 in 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:29 smithi064 ceph-mon[180297]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.638 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: pgmap v854: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:29.638 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2679779863' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: osdmap e242: 8 total, 6 up, 6 in 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:29 smithi152 ceph-mon[156819]: failed to encode map e242 with expected crc 2023-06-10T21:28:29.789 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:29.789 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:28:29.789 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:28:29.789 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:28:29.789 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:28:30.159 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-0, ['dump_ops_in_flight'] 2023-06-10T21:28:30.182 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:30.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:30.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:30.316 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:30.537 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:30.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:30.602 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:28:30.641 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:30.676 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:30.721 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:30.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:30.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:30.828 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:30.856 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:30.901 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:30.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:30.968 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:30.976 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:30.995 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:31.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:31.013 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:31.032 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:31.033 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:31.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:31.070 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:31.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.0 2023-06-10T21:28:31.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.071 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:31.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:28:31.081 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:31.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:31.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:31.212 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:31.254 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:31.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:31.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:31.391 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:31.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:31.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:31.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:31.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:31.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:31 smithi064 ceph-mon[186706]: pgmap v856: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/50245321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:31 smithi064 ceph-mon[180297]: pgmap v856: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/50245321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:31.783 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:31.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:31.858 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:31.887 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:31.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:31 smithi152 ceph-mon[156819]: pgmap v856: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:31.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/50245321' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:31.916 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:32.011 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:32.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:32.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:32.145 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:32.240 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:32.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:32.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:32.373 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:32.480 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:32.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:32.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:32.485 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:32 smithi152 ceph-mon[156819]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.485 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:32 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.505 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:32.506 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:32.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:32.613 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:32 smithi064 ceph-mon[180297]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:32 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:32 smithi064 ceph-mon[186706]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:32 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-10T21:28:32.661 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:32 smithi064 conmon[348287]: 2023-06-10T21:28:32.300+0000 7f84aec26640 -1 osd.0 231 log_to_monitors true 2023-06-10T21:28:32.661 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:28:32 smithi064 conmon[348287]: 2023-06-10T21:28:32.494+0000 7f849f891700 -1 osd.0 231 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:28:32.757 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:32.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:32.795 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:28:32.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:32.888 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:33.112 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:33.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:33.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:33.155 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:32] "GET /metrics HTTP/1.1" 200 224028 "" "Prometheus/2.33.4" 2023-06-10T21:28:33.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:28:33.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:33.245 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:33.288 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:33.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:33.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:33.424 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:33.471 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:33.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:33.560 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:33.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:33.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: pgmap v857: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:33.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: osdmap e243: 8 total, 6 up, 6 in 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:33.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: pgmap v857: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: osdmap e243: 8 total, 6 up, 6 in 2023-06-10T21:28:33.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:33.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:33.618 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: pgmap v857: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: failed to encode map e243 with expected crc 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: osdmap e243: 8 total, 6 up, 6 in 2023-06-10T21:28:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: from='osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:33.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:28:34.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:34.485 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: Health check update: 1 osds down (OSD_DOWN) 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228] boot 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: osdmap e244: 8 total, 7 up, 6 in 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.486 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[186706]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:34.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:34.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:34.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:34.684 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:34.782 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:34.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: Health check update: 1 osds down (OSD_DOWN) 2023-06-10T21:28:34.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228] boot 2023-06-10T21:28:34.785 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: osdmap e244: 8 total, 7 up, 6 in 2023-06-10T21:28:34.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:28:34.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.786 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:34 smithi064 ceph-mon[180297]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:34.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:34.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:34.915 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: Health check update: 1 osds down (OSD_DOWN) 2023-06-10T21:28:34.915 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: osd.0 [v2:172.21.15.64:6802/1109161228,v1:172.21.15.64:6803/1109161228] boot 2023-06-10T21:28:34.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: osdmap e244: 8 total, 7 up, 6 in 2023-06-10T21:28:34.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-10T21:28:34.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.916 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.917 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:34 smithi152 ceph-mon[156819]: failed to encode map e244 with expected crc 2023-06-10T21:28:34.919 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:34.966 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:34.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:35.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:35.101 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:35.117 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:35.117 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:35.144 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:35.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:35.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:28:35.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:35.274 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:35.369 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:35.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:35.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:35.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:35.493 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:28:35.500 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:35.577 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:35.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:35.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: pgmap v860: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:35.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.589 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.590 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: osdmap e245: 8 total, 7 up, 6 in 2023-06-10T21:28:35.590 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.590 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.590 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.590 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:35.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:35.724 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:35.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: pgmap v860: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:35.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.852 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: osdmap e245: 8 total, 7 up, 6 in 2023-06-10T21:28:35.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:35.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: pgmap v860: 89 pgs: 3 active+clean+remapped, 10 active+clean, 2 active+undersized+remapped, 5 undersized+degraded+peered, 14 active+undersized+degraded, 15 undersized+peered, 40 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 47980/209829 objects degraded (22.866%); 30680/209829 objects misplaced (14.621%) 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: osdmap e245: 8 total, 7 up, 6 in 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: failed to encode map e245 with expected crc 2023-06-10T21:28:35.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:28:35.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:28:36.148 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:36.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:36.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:36.299 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:36.349 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:36.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:36.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:36.455 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:36.457 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:36.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:36.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:36.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:36.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: osdmap e246: 8 total, 7 up, 6 in 2023-06-10T21:28:36.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:36 smithi152 ceph-mon[156819]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.858 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:36.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:36.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: osdmap e246: 8 total, 7 up, 6 in 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[186706]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:36.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/793276215' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:28:36.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: osdmap e246: 8 total, 7 up, 6 in 2023-06-10T21:28:36.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:36 smithi064 ceph-mon[180297]: failed to encode map e246 with expected crc 2023-06-10T21:28:36.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:37.024 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:37.102 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:37.172 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:37.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:37.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:37.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:37.359 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:37.438 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:37.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:37.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:37.615 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:28:37.632 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:37.657 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:37 smithi152 ceph-mon[156819]: pgmap v863: 89 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 6 peering, 3 active+clean+remapped, 18 active+clean, 3 undersized+degraded+peered, 11 active+undersized+degraded, 10 undersized+peered, 35 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 30522/209817 objects degraded (14.547%); 13076/209817 objects misplaced (6.232%); 0 B/s, 1 objects/s recovering 2023-06-10T21:28:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:37 smithi152 ceph-mon[156819]: failed to encode map e246 with expected crc 2023-06-10T21:28:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:37 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 30522/209817 objects degraded (14.547%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:37.696 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:37.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:37.732 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:37.734 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[180297]: pgmap v863: 89 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 6 peering, 3 active+clean+remapped, 18 active+clean, 3 undersized+degraded+peered, 11 active+undersized+degraded, 10 undersized+peered, 35 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 30522/209817 objects degraded (14.547%); 13076/209817 objects misplaced (6.232%); 0 B/s, 1 objects/s recovering 2023-06-10T21:28:37.734 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[180297]: failed to encode map e246 with expected crc 2023-06-10T21:28:37.735 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 30522/209817 objects degraded (14.547%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:37.735 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[186706]: pgmap v863: 89 pgs: 1 active+recovery_wait+degraded, 2 remapped+peering, 6 peering, 3 active+clean+remapped, 18 active+clean, 3 undersized+degraded+peered, 11 active+undersized+degraded, 10 undersized+peered, 35 active+undersized; 4.3 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 0 op/s; 30522/209817 objects degraded (14.547%); 13076/209817 objects misplaced (6.232%); 0 B/s, 1 objects/s recovering 2023-06-10T21:28:37.735 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[186706]: failed to encode map e246 with expected crc 2023-06-10T21:28:37.736 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:37 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 30522/209817 objects degraded (14.547%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:37.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:37.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:37.767 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:37.808 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:37.838 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:37.845 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.846 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.847 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.847 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:28:37.885 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:38.417 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:38.418 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:28:38.418 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:28:38.418 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:28:38.418 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:38.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3955019267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:38.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3955019267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:38.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3955019267' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:38.959 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:28:39.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:28:39.644 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:39.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:39.674 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:39 smithi152 ceph-mon[156819]: pgmap v864: 89 pgs: 1 active+recovery_wait+degraded, 6 peering, 5 active+clean+remapped, 33 active+clean, 8 active+undersized+degraded, 36 active+undersized; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 8541/207345 objects degraded (4.119%); 21640/207345 objects misplaced (10.437%); 0 B/s, 11 objects/s recovering 2023-06-10T21:28:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:39 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive) 2023-06-10T21:28:39.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:39.833 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:39.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:39 smithi064 ceph-mon[186706]: pgmap v864: 89 pgs: 1 active+recovery_wait+degraded, 6 peering, 5 active+clean+remapped, 33 active+clean, 8 active+undersized+degraded, 36 active+undersized; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 8541/207345 objects degraded (4.119%); 21640/207345 objects misplaced (10.437%); 0 B/s, 11 objects/s recovering 2023-06-10T21:28:39.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:39 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive) 2023-06-10T21:28:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:39 smithi064 ceph-mon[180297]: pgmap v864: 89 pgs: 1 active+recovery_wait+degraded, 6 peering, 5 active+clean+remapped, 33 active+clean, 8 active+undersized+degraded, 36 active+undersized; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 8541/207345 objects degraded (4.119%); 21640/207345 objects misplaced (10.437%); 0 B/s, 11 objects/s recovering 2023-06-10T21:28:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:39 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive) 2023-06-10T21:28:39.942 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:28:39.977 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:39.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:40.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:40.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:40.154 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:40.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:40.280 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:28:40.280 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:28:40.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.280 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115934 10.4 10:252f37b2:::benchmark_data_smithi064_312713_object25178:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.280 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.275441+0000", 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001768386, 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017925619999999999, 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.281 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115934 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275441+0000", 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275441+0000", 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275440+0000", 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275447+0000", 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8920000000000003e-06 2023-06-10T21:28:40.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275459+0000", 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2465e-05 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275468+0000", 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4530000000000002e-06 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275519+0000", 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1576999999999998e-05 2023-06-10T21:28:40.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275666+0000", 2023-06-10T21:28:40.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146689 2023-06-10T21:28:40.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275822+0000", 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155363 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115937 10.2 10:45802cdf:::benchmark_data_smithi064_312713_object25181:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.275492+0000", 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001717812, 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017972489999999999, 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.289 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115937 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275492+0000", 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275492+0000", 2023-06-10T21:28:40.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275492+0000", 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275495+0000", 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2780000000000002e-06 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275502+0000", 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5989999999999996e-06 2023-06-10T21:28:40.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275508+0000", 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6509999999999998e-06 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275586+0000", 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8288999999999995e-05 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275711+0000", 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012455999999999999 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275829+0000", 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118522 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115938 10.8 10:1b029891:::benchmark_data_smithi064_312713_object25182:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.275521+0000", 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001688356, 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018114730000000001, 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.297 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115938 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275521+0000", 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275521+0000", 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275521+0000", 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275524+0000", 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9289999999999998e-06 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275531+0000", 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0319999999999998e-06 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275537+0000", 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.733e-06 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275581+0000", 2023-06-10T21:28:40.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4094000000000002e-05 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275689+0000", 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108223 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275821+0000", 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013184399999999999 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115928 10.b 10:de283f4f:::benchmark_data_smithi064_312713_object25172:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.265642+0000", 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011567568, 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011744884000000001, 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.305 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115928 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265642+0000", 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265642+0000", 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265641+0000", 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265647+0000", 2023-06-10T21:28:40.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5820000000000001e-06 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265660+0000", 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3295000000000001e-05 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265672+0000", 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1999e-05 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.267783+0000", 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021109219999999999 2023-06-10T21:28:40.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.267944+0000", 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016064200000000001 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:28:40.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.268084+0000", 2023-06-10T21:28:40.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140018 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.274567+0000", 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064836720000000002 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.274714+0000", 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014658999999999999 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.314 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115929 10.d 10:b33f910e:::benchmark_data_smithi064_312713_object25173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.265689+0000", 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011521201999999999, 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011763312999999999, 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115929 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265689+0000", 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265689+0000", 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265688+0000", 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265692+0000", 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9689999999999996e-06 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265700+0000", 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8280000000000003e-06 2023-06-10T21:28:40.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265706+0000", 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0680000000000002e-06 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.269519+0000", 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038130640000000001 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.269632+0000", 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112549 2023-06-10T21:28:40.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.269745+0000", 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113156 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:40.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.272640+0000", 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028951789999999999 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.275321+0000", 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026814909999999998 2023-06-10T21:28:40.323 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115930 10.8 10:1c29468d:::benchmark_data_smithi064_312713_object25174:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.265718+0000", 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011491517999999999, 2023-06-10T21:28:40.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011781163000000001, 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115930 2023-06-10T21:28:40.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265718+0000", 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265718+0000", 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265718+0000", 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:40.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265721+0000", 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8849999999999999e-06 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265727+0000", 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0499999999999997e-06 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265733+0000", 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6779999999999997e-06 2023-06-10T21:28:40.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265870+0000", 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013795100000000001 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265980+0000", 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109861 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.266089+0000", 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108208 2023-06-10T21:28:40.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.272530+0000", 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064413370000000001 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.274514+0000", 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019842789999999998 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.333 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:115931 10.b 10:d6a3ba6a:::benchmark_data_smithi064_312713_object25175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e246)", 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:40.265747+0000", 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011463209, 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011829279, 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:28:40.334 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 115931 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265747+0000", 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265747+0000", 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265746+0000", 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265749+0000", 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8229999999999999e-06 2023-06-10T21:28:40.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265758+0000", 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9080000000000001e-06 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.265764+0000", 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6269999999999997e-06 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:28:40.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.271121+0000", 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053570309999999999 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.271237+0000", 2023-06-10T21:28:40.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116632 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.271368+0000", 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013041800000000001 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.274809+0000", 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034409800000000002 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:40.274880+0000", 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1667000000000003e-05 2023-06-10T21:28:40.342 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 7 2023-06-10T21:28:40.343 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:28:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:40.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:40.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:40.499 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:41.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:41.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:41.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:41.201 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:41.344 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:41.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:41.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:41.501 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:41 smithi152 ceph-mon[156819]: pgmap v865: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 10 objects/s recovering 2023-06-10T21:28:41.523 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:41.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:28:41.709 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:41.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:41.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:41.860 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:41.905 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:41 smithi064 ceph-mon[180297]: pgmap v865: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 10 objects/s recovering 2023-06-10T21:28:41.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:41 smithi064 ceph-mon[186706]: pgmap v865: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 10 objects/s recovering 2023-06-10T21:28:41.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:41.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:41.920 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:28:41.920 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:28:41.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:28:41.920 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20066 9.d 9:b754690b:::benchmark_data_smithi064_247471_object20065:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.93154883699998, 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999, 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.921 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20066 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983876+0000", 2023-06-10T21:28:41.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.983875+0000", 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019127799999999999 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984066+0000", 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2920000000000001e-06 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984075+0000", 2023-06-10T21:28:41.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9660000000000001e-06 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984081+0000", 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8265999999999999e-05 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984159+0000", 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121333 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984281+0000", 2023-06-10T21:28:41.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144637 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984425+0000", 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014625199999999999 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985888+0000", 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024362569999999998 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988324+0000", 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29050814800000002 2023-06-10T21:28:41.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278832+0000", 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5896999999999998e-05 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278878+0000", 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7633999999999999e-05 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278916+0000", 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29504017199999999 2023-06-10T21:28:41.931 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20069 9.3 9:c33031bf:::benchmark_data_smithi064_247471_object20068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:28:41.932 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.93108118599997, 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999, 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20069 2023-06-10T21:28:41.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984343+0000", 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.984342+0000", 2023-06-10T21:28:41.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012432649999999999 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985585+0000", 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6175e-05 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985602+0000", 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8390000000000001e-06 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985609+0000", 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1567999999999998e-05 2023-06-10T21:28:41.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985660+0000", 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013685800000000001 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985797+0000", 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016983999999999999 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985967+0000", 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026175999999999999 2023-06-10T21:28:41.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988584+0000", 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012783530000000001 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989863+0000", 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28834436000000002 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278207+0000", 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:28:41.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278244+0000", 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8973e-05 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278293+0000", 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394963499999999 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.943 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20072 9.3 9:ce7b60fb:::benchmark_data_smithi064_247471_object20071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.92980081000002, 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998, 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.944 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20072 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:28:41.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985624+0000", 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985623+0000", 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7206e-05 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985650+0000", 2023-06-10T21:28:41.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7629e-05 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985668+0000", 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985674+0000", 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001225133 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986899+0000", 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012472500000000001 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987024+0000", 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000152232 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987176+0000", 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014119930000000001 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988588+0000", 2023-06-10T21:28:41.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017869019999999999 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990375+0000", 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28840961799999998 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278785+0000", 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9700000000000002e-05 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278834+0000", 2023-06-10T21:28:41.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8235999999999998e-05 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.278872+0000", 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29324878599999998 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.954 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20074 9.3 9:cf5d96ce:::benchmark_data_smithi064_247471_object20073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.929744572, 2023-06-10T21:28:41.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001, 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20074 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.956 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985680+0000", 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985679+0000", 2023-06-10T21:28:41.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9383e-05 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985709+0000", 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985716+0000", 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.048e-06 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.985720+0000", 2023-06-10T21:28:41.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002102862 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987822+0000", 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4449000000000004e-05 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987907+0000", 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000134979 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988042+0000", 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00064599799999999995 2023-06-10T21:28:41.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988688+0000", 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001729993 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990418+0000", 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28863366400000001 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279052+0000", 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3007999999999998e-05 2023-06-10T21:28:41.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279105+0000", 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8399000000000003e-05 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.279143+0000", 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29346308300000001 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20077 9.d 9:bf31cec4:::benchmark_data_smithi064_247471_object20076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.966 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.92855992199998, 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002, 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.967 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20077 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986865+0000", 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986863+0000", 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1553000000000001e-05 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986895+0000", 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5759999999999995e-06 2023-06-10T21:28:41.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986905+0000", 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7969999999999997e-06 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986910+0000", 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1326000000000001e-05 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.986982+0000", 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012606299999999999 2023-06-10T21:28:41.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987108+0000", 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014393300000000001 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.987252+0000", 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001079055 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988331+0000", 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117989 2023-06-10T21:28:41.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988449+0000", 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29228185899999998 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280731+0000", 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4913000000000002e-05 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280776+0000", 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8541000000000003e-05 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.280814+0000", 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29394951400000002 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:20079 9.d 9:b60e4f03:::benchmark_data_smithi064_247471_object20078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 461.926432207, 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267, 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.978 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 20079 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988992+0000", 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:41.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.988991+0000", 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9812000000000001e-05 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989071+0000", 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1182000000000001e-05 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989082+0000", 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1869999999999996e-06 2023-06-10T21:28:41.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989087+0000", 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6656999999999998e-05 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989124+0000", 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013583000000000001 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989260+0000", 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9172999999999995e-05 2023-06-10T21:28:41.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.989349+0000", 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00098969899999999992 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990339+0000", 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017819999999999999 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:20:59.990517+0000", 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29222891400000001 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282746+0000", 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8776000000000001e-05 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282805+0000", 2023-06-10T21:28:41.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4026000000000002e-05 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:21:00.282849+0000", 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.293856267 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.988 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199854 9.3 9:c5f79f4f:::benchmark_data_smithi064_247471_object90866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.37898911400001, 2023-06-10T21:28:41.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362, 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199854 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.990 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:41.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536435+0000", 2023-06-10T21:28:41.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:28:41.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:41.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536433+0000", 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7489999999999998e-06 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536441+0000", 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5624000000000002e-05 2023-06-10T21:28:41.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536457+0000", 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5969999999999997e-06 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536463+0000", 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107921 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536571+0000", 2023-06-10T21:28:41.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011766500000000001 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536689+0000", 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5592000000000003e-05 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.536774+0000", 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0037939639999999999 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540568+0000", 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012242509 2023-06-10T21:28:41.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552811+0000", 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.296973082 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849784+0000", 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6795999999999997e-05 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849831+0000", 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4078e-05 2023-06-10T21:28:41.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.849875+0000", 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.313439362 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.000 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199859 9.3 9:ca245c36:::benchmark_data_smithi064_247471_object90871:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.37451727300001, 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001, 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.001 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199859 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:28:42.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540907+0000", 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.915e-06 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540910+0000", 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9649999999999996e-06 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540916+0000", 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7190000000000001e-06 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540920+0000", 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9440999999999997e-05 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540960+0000", 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111699 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541072+0000", 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053361700000000001 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541605+0000", 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011243814 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552849+0000", 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039226670000000003 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556772+0000", 2023-06-10T21:28:42.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29326851399999998 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850040+0000", 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7537999999999999e-05 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850078+0000", 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4517999999999997e-05 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850112+0000", 2023-06-10T21:28:42.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30920499800000001 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199860 9.d 9:bf36e690:::benchmark_data_smithi064_247471_object90872:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:28:42.012 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.374492584, 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999, 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.013 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199860 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540932+0000", 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540931+0000", 2023-06-10T21:28:42.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8059999999999999e-06 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540934+0000", 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2140000000000001e-06 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540940+0000", 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7560000000000001e-06 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.540944+0000", 2023-06-10T21:28:42.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003930256 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544874+0000", 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122638 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.544997+0000", 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012774499999999999 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545125+0000", 2023-06-10T21:28:42.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046761019999999997 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549801+0000", 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00087457099999999999 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.550675+0000", 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29424700599999998 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844922+0000", 2023-06-10T21:28:42.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6332000000000002e-05 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.844979+0000", 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3044000000000003e-05 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.845022+0000", 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30408984999999999 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199861 9.3 9:c4d624c7:::benchmark_data_smithi064_247471_object90873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.37432702199999, 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997, 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.024 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199861 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:28:42.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541097+0000", 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9880000000000004e-06 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541102+0000", 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6309999999999992e-06 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541110+0000", 2023-06-10T21:28:42.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3319999999999999e-06 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541115+0000", 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045389230000000003 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545654+0000", 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010809799999999999 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545762+0000", 2023-06-10T21:28:42.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000124232 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.545887+0000", 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069982400000000002 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552885+0000", 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003894041 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556779+0000", 2023-06-10T21:28:42.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29336341300000002 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850142+0000", 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3971999999999999e-05 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850176+0000", 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3324999999999997e-05 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850210+0000", 2023-06-10T21:28:42.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30911225599999997 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199863 9.3 9:c858996c:::benchmark_data_smithi064_247471_object90875:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.035 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.37426685200001, 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986, 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.036 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199863 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541158+0000", 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541157+0000", 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0770000000000001e-06 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541161+0000", 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3150000000000003e-06 2023-06-10T21:28:42.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541168+0000", 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0239999999999997e-06 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.541173+0000", 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084016230000000004 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549575+0000", 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011957599999999999 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549695+0000", 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013125600000000001 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:42.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549826+0000", 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070024570000000001 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556828+0000", 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036556430000000001 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560484+0000", 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28975610600000001 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850240+0000", 2023-06-10T21:28:42.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2453999999999997e-05 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850283+0000", 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3065999999999998e-05 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850316+0000", 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.309157986 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199865 9.3 9:cef2937d:::benchmark_data_smithi064_247471_object90877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.36578634599999, 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124, 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.047 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199865 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549638+0000", 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549637+0000", 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549642+0000", 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7441000000000001e-05 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549660+0000", 2023-06-10T21:28:42.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4330000000000001e-06 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.549666+0000", 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003246578 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.552913+0000", 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000139326 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553052+0000", 2023-06-10T21:28:42.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147138 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.553199+0000", 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036693580000000002 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.054 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556869+0000", 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003640038 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560509+0000", 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28983566300000002 2023-06-10T21:28:42.055 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850344+0000", 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3887999999999998e-05 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850378+0000", 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1239999999999999e-05 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850409+0000", 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30077124 2023-06-10T21:28:42.057 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199870 9.3 9:cea33e0b:::benchmark_data_smithi064_247471_object90882:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:28:42.058 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 240.358802736, 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003, 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199870 2023-06-10T21:28:42.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556622+0000", 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556620+0000", 2023-06-10T21:28:42.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.57e-06 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556626+0000", 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5829999999999996e-06 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.062 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556635+0000", 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1599999999999997e-06 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556641+0000", 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026981200000000002 2023-06-10T21:28:42.063 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.556910+0000", 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113881 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557024+0000", 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121586 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.557146+0000", 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033591770000000001 2023-06-10T21:28:42.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.560505+0000", 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018934659999999999 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.562399+0000", 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28803842299999999 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850437+0000", 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2833000000000002e-05 2023-06-10T21:28:42.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850470+0000", 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1829999999999998e-05 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.850502+0000", 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.29387990200000003 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.069 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95682 10.c 10:3eea8292:::benchmark_data_smithi064_312713_object4926:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.465994820000006, 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32186570199999998, 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.070 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95682 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:28:42.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449430+0000", 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449427+0000", 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3190000000000001e-06 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.073 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449437+0000", 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4035e-05 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449451+0000", 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3659999999999997e-06 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.074 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449457+0000", 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8087999999999997e-05 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449515+0000", 2023-06-10T21:28:42.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013555500000000001 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449651+0000", 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013275299999999999 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449783+0000", 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098969969999999994 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459680+0000", 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7057000000000001e-05 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459717+0000", 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31138205299999999 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771099+0000", 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000148877 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771248+0000", 2023-06-10T21:28:42.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6993999999999998e-05 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.771295+0000", 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32186570199999998 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.080 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95683 10.1 10:82260f40:::benchmark_data_smithi064_312713_object4927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.465948827999995, 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604, 2023-06-10T21:28:42.081 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95683 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449475+0000", 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3629999999999998e-06 2023-06-10T21:28:42.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449478+0000", 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4200000000000007e-06 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449487+0000", 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8169999999999998e-06 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449492+0000", 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034418460000000001 2023-06-10T21:28:42.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.452934+0000", 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123182 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.087 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453057+0000", 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012961900000000001 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453186+0000", 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058584609999999997 2023-06-10T21:28:42.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459045+0000", 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6512e-05 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459061+0000", 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32578302999999997 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784844+0000", 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3819000000000001e-05 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784898+0000", 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2079999999999997e-05 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784940+0000", 2023-06-10T21:28:42.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95684 10.c 10:3b0e142b:::benchmark_data_smithi064_312713_object4928:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:42.092 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.462414878999994, 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.326603953, 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.093 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95684 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453010+0000", 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.095 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453008+0000", 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.619e-06 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453016+0000", 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0013000000000001e-05 2023-06-10T21:28:42.096 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453026+0000", 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6809999999999997e-06 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453031+0000", 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00042010499999999999 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453451+0000", 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010725 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453558+0000", 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116557 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453675+0000", 2023-06-10T21:28:42.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0060171319999999997 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459692+0000", 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003798283 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.463490+0000", 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31602760499999999 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779518+0000", 2023-06-10T21:28:42.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3182e-05 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779571+0000", 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2423999999999999e-05 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779614+0000", 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.326603953 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.103 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95687 10.1 10:884db9c7:::benchmark_data_smithi064_312713_object4931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.462382434999995, 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002, 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.104 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95687 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:28:42.105 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.106 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453041+0000", 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5729999999999999e-06 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453045+0000", 2023-06-10T21:28:42.107 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.665e-06 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453051+0000", 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.253e-06 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.108 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453054+0000", 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027574850000000001 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455811+0000", 2023-06-10T21:28:42.109 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2564999999999994e-05 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455884+0000", 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6180999999999994e-05 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.110 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455960+0000", 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031903819999999999 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459151+0000", 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015862529999999999 2023-06-10T21:28:42.111 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.460737+0000", 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32425166599999999 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.112 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784988+0000", 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6248999999999997e-05 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785035+0000", 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8260000000000003e-05 2023-06-10T21:28:42.113 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785073+0000", 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.114 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95691 10.c 10:35e47876:::benchmark_data_smithi064_312713_object4935:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.460928480999996, 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32524502799999999, 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.115 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95691 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.116 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454496+0000", 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:42.117 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454495+0000", 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3430000000000003e-06 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.118 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454500+0000", 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0764999999999999e-05 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454511+0000", 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8950000000000003e-06 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.119 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.454517+0000", 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021666849999999998 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456683+0000", 2023-06-10T21:28:42.120 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6739999999999999e-05 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456780+0000", 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010805 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.121 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.456888+0000", 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029155219999999998 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.122 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459804+0000", 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038126929999999998 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.463616+0000", 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31604337700000001 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.123 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779660+0000", 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5500000000000001e-05 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779705+0000", 2023-06-10T21:28:42.124 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5839e-05 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.779741+0000", 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32524502799999999 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.125 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111471 10.1 10:85cc9677:::benchmark_data_smithi064_312713_object20715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.625839784999997, 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999, 2023-06-10T21:28:42.126 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111471 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.127 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.128 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289583+0000", 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0890000000000002e-06 2023-06-10T21:28:42.129 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289590+0000", 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6314e-05 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.130 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289607+0000", 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.267e-06 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289614+0000", 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3952000000000001e-05 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.131 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289668+0000", 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001314 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.132 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289799+0000", 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9462999999999994e-05 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289889+0000", 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029815369999999998 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.133 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.292870+0000", 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065814400000000005 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.134 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.293528+0000", 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030991312 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324520+0000", 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3043e-05 2023-06-10T21:28:42.135 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324533+0000", 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.00200007 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.136 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326533+0000", 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6407000000000003e-05 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326599+0000", 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.137 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111487 10.1 10:8be62f36:::benchmark_data_smithi064_312713_object20731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.583698558999998, 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997, 2023-06-10T21:28:42.138 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111487 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.139 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:28:42.140 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331724+0000", 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7449999999999995e-06 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.141 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331732+0000", 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5709000000000002e-05 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:28:42.142 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0659999999999998e-06 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331755+0000", 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1113999999999999e-05 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.143 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331806+0000", 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0156999999999999e-05 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331816+0000", 2023-06-10T21:28:42.144 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99503633400000002 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326852+0000", 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9278999999999996e-05 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.145 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326952+0000", 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015410900000000001 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4", 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.327106+0000", 2023-06-10T21:28:42.146 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035978350000000002 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.330703+0000", 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051919100000000001 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.147 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331223+0000", 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1488999999999994e-05 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331284+0000", 2023-06-10T21:28:42.148 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112025 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331396+0000", 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.149 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:28:42.150 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:28:42.150 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:28:42.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:42.179 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-06-10T21:28:42.180 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-06-10T21:28:42.180 DEBUG:teuthology.orchestra.run.smithi064:osd.3> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3.service 2023-06-10T21:28:42.182 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl start ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3 2023-06-10T21:28:42.206 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:42.241 INFO:journalctl@ceph.osd.3.smithi064.stdout:-- Logs begin at Sat 2023-06-10 20:37:47 UTC. -- 2023-06-10T21:28:42.304 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:42.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:42.412 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:42 smithi064 systemd[1]: Starting Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:28:42.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:42.458 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:42.705 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:42.763 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:42.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:42.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:28:42.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:42.913 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:42.990 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:42.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:43.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:43.170 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:42] "GET /metrics HTTP/1.1" 200 223477 "" "Prometheus/2.33.4" 2023-06-10T21:28:43.208 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:43.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:43.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:43.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:43.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:43 smithi064 ceph-mon[180297]: pgmap v866: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 8 objects/s recovering 2023-06-10T21:28:43.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:43.735 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:43.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:43.848 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:43 smithi064 ceph-mon[186706]: pgmap v866: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 8 objects/s recovering 2023-06-10T21:28:43.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:43 smithi152 ceph-mon[156819]: pgmap v866: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.1 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 12461/201750 objects degraded (6.176%); 21070/201750 objects misplaced (10.444%); 0 B/s, 8 objects/s recovering 2023-06-10T21:28:44.540 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:44.545 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:28:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:44.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:44.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:44.700 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:44.786 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:44.799 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2817469818' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.800 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:44.901 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:44.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2817469818' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2817469818' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:28:44.935 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:44.977 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:44.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:44.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:45.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:45.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:45.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-10T21:28:45.409 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:45 smithi064 bash[352534]: --> ceph-volume raw activate successful for osd ID: 3 2023-06-10T21:28:45.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: pgmap v867: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 12190/196314 objects degraded (6.209%); 20551/196314 objects misplaced (10.468%); 0 B/s, 7 objects/s recovering 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12190/196314 objects degraded (6.209%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: failed to encode map e247 with expected crc 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: osdmap e247: 8 total, 7 up, 6 in 2023-06-10T21:28:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[186706]: failed to encode map e247 with expected crc 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: pgmap v867: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 12190/196314 objects degraded (6.209%); 20551/196314 objects misplaced (10.468%); 0 B/s, 7 objects/s recovering 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12190/196314 objects degraded (6.209%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: failed to encode map e247 with expected crc 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:45.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: osdmap e247: 8 total, 7 up, 6 in 2023-06-10T21:28:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:45 smithi064 ceph-mon[180297]: failed to encode map e247 with expected crc 2023-06-10T21:28:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: pgmap v867: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 4.0 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 12190/196314 objects degraded (6.209%); 20551/196314 objects misplaced (10.468%); 0 B/s, 7 objects/s recovering 2023-06-10T21:28:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12190/196314 objects degraded (6.209%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:28:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: failed to encode map e247 with expected crc 2023-06-10T21:28:45.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:28:45.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: osdmap e247: 8 total, 7 up, 6 in 2023-06-10T21:28:45.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:45 smithi152 ceph-mon[156819]: failed to encode map e247 with expected crc 2023-06-10T21:28:46.932 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:46.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:47.041 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:47.096 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:46 smithi064 ceph-mon[180297]: pgmap v869: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 11726/189297 objects degraded (6.194%); 19781/189297 objects misplaced (10.450%); 0 B/s, 6 objects/s recovering 2023-06-10T21:28:47.097 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:46 smithi064 ceph-mon[186706]: pgmap v869: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 11726/189297 objects degraded (6.194%); 19781/189297 objects misplaced (10.450%); 0 B/s, 6 objects/s recovering 2023-06-10T21:28:47.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:46 smithi152 ceph-mon[156819]: pgmap v869: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.9 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 683 op/s; 11726/189297 objects degraded (6.194%); 19781/189297 objects misplaced (10.450%); 0 B/s, 6 objects/s recovering 2023-06-10T21:28:47.349 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:47.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:47.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:47.516 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:47.611 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:47.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:47.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:47.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:47.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:28:47.862 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:47.931 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:47.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:48.001 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:48.030 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:28:48.069 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:28:48.069 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.069 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.069 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-06-10T21:28:48.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:28:48.878 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:48.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:48.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:48.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:28:48.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:48.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[186706]: pgmap v870: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 11444/183480 objects degraded (6.237%); 19228/183480 objects misplaced (10.480%); 0 B/s, 0 objects/s recovering 2023-06-10T21:28:48.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2573690024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:48.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:48.986 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:48.987 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[180297]: pgmap v870: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 11444/183480 objects degraded (6.237%); 19228/183480 objects misplaced (10.480%); 0 B/s, 0 objects/s recovering 2023-06-10T21:28:48.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2573690024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:48.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:48.988 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:49.051 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:49.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:48 smithi152 ceph-mon[156819]: pgmap v870: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 11444/183480 objects degraded (6.237%); 19228/183480 objects misplaced (10.480%); 0 B/s, 0 objects/s recovering 2023-06-10T21:28:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2573690024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:28:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:28:49.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:28:49.661 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:49.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:49.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:28:49.704 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:28:49.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:49.824 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:49.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:49 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10927/176106 objects degraded (6.205%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:49.911 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:49.917 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:49 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10927/176106 objects degraded (6.205%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:49.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:49.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:50.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:50.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:49 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10927/176106 objects degraded (6.205%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:50.076 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:50.139 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:50.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:50.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:28:50.256 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:28:50.303 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:50.371 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:28:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:50.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:28:50.385 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:28:50.386 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:28:50.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:50 smithi064 ceph-osd[354109]: -- 172.21.15.64:0/2816320785 <== mon.0 v2:172.21.15.64:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564009c7c4e0 con 0x564009c4f000 2023-06-10T21:28:50.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:50 smithi064 bash[353821]: 9f2a2ef8b19107f131c42a2b55c2206ff224cee7b187310f70fd69c01fa7368f 2023-06-10T21:28:50.408 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:50 smithi064 systemd[1]: Started Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:28:50.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:50.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:50.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:50 smithi064 ceph-mon[180297]: pgmap v871: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:50.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:50 smithi064 ceph-mon[186706]: pgmap v871: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:50.796 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:50 smithi064 conmon[354039]: 2023-06-10T21:28:50.411+0000 7f90bf03a640 -1 Falling back to public interface 2023-06-10T21:28:51.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:50 smithi152 ceph-mon[156819]: pgmap v871: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:51.287 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:51.513 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:51.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:51.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:51.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:51.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:28:51.655 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:51.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:28:51.723 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:51.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:51.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:51.989 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:52.142 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:52.165 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:52.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:52.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:52.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:52.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:52.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:52.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:52.653 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:52.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:52.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:52.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:52.945 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:53.064 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:52 smithi064 ceph-mon[180297]: pgmap v872: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:53.066 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:28:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:28:52] "GET /metrics HTTP/1.1" 200 223831 "" "Prometheus/2.33.4" 2023-06-10T21:28:53.066 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:52 smithi064 ceph-mon[186706]: pgmap v872: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:53.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:53.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:53.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:52 smithi152 ceph-mon[156819]: pgmap v872: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 853 op/s; 10927/176106 objects degraded (6.205%); 18433/176106 objects misplaced (10.467%) 2023-06-10T21:28:53.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:53.335 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:53.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:53.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:53.508 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:53.519 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:28:53.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:53.675 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:53.839 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:53.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:53.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:28:53.988 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:54.050 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:28:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:54.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:28:54.157 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:54.183 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:54.235 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:54.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:54.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:54.442 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:54.866 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:28:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:54.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:28:54.891 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:28:54.978 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:28:55.029 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:55.057 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:54 smithi064 conmon[354039]: 2023-06-10T21:28:54.900+0000 7f90bf03a640 -1 osd.3 218 log_to_monitors true 2023-06-10T21:28:55.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:54 smithi064 ceph-mon[180297]: pgmap v873: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 10642/170160 objects degraded (6.254%); 17857/170160 objects misplaced (10.494%) 2023-06-10T21:28:55.058 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:55.058 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:54 smithi064 ceph-mon[186706]: pgmap v873: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 10642/170160 objects degraded (6.254%); 17857/170160 objects misplaced (10.494%) 2023-06-10T21:28:55.059 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:55.103 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:28:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:55.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:28:55.112 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:54 smithi152 ceph-mon[156819]: pgmap v873: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.5 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 870 op/s; 10642/170160 objects degraded (6.254%); 17857/170160 objects misplaced (10.494%) 2023-06-10T21:28:55.112 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:28:55.213 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:55.269 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:55.982 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:55.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:56.097 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:56.151 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:56.152 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:56.152 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: osdmap e248: 8 total, 7 up, 6 in 2023-06-10T21:28:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:28:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10642/170160 objects degraded (6.254%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:56.159 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:28:55 smithi064 conmon[354039]: 2023-06-10T21:28:55.901+0000 7f90b04a6700 -1 osd.3 218 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:28:56.160 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:56.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:56.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:56.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: osdmap e248: 8 total, 7 up, 6 in 2023-06-10T21:28:56.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10642/170160 objects degraded (6.254%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1472265685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: osdmap e248: 8 total, 7 up, 6 in 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: failed to encode map e248 with expected crc 2023-06-10T21:28:56.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-10T21:28:56.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10642/170160 objects degraded (6.254%), 9 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2023-06-10T21:28:56.259 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:56.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:56.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:56.445 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:56.521 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:56.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:56.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:56.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:56.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:57.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: pgmap v875: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 10065/162768 objects degraded (6.184%); 17005/162768 objects misplaced (10.447%) 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: osdmap e249: 8 total, 7 up, 6 in 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[186706]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: pgmap v875: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 10065/162768 objects degraded (6.184%); 17005/162768 objects misplaced (10.447%) 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: osdmap e249: 8 total, 7 up, 6 in 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:56 smithi064 ceph-mon[180297]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: pgmap v875: 89 pgs: 5 active+clean+remapped, 37 active+clean, 9 active+undersized+degraded, 38 active+undersized; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 10065/162768 objects degraded (6.184%); 17005/162768 objects misplaced (10.447%) 2023-06-10T21:28:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-10T21:28:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: osdmap e249: 8 total, 7 up, 6 in 2023-06-10T21:28:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: from='osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi064", "root=default"]}]: dispatch 2023-06-10T21:28:57.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:56 smithi152 ceph-mon[156819]: failed to encode map e249 with expected crc 2023-06-10T21:28:57.731 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:57.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:57.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:28:57.867 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:57.890 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:28:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:28:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:28:57.891 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:28:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: failed to encode map e249 with expected crc 2023-06-10T21:28:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:28:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426] boot 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: osdmap e250: 8 total, 8 up, 6 in 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[186706]: osdmap e251: 8 total, 8 up, 6 in 2023-06-10T21:28:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: failed to encode map e249 with expected crc 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426] boot 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: osdmap e250: 8 total, 8 up, 6 in 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:28:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:57 smithi064 ceph-mon[180297]: osdmap e251: 8 total, 8 up, 6 in 2023-06-10T21:28:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: failed to encode map e249 with expected crc 2023-06-10T21:28:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:28:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: failed to encode map e250 with expected crc 2023-06-10T21:28:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: osd.3 [v2:172.21.15.64:6826/235474426,v1:172.21.15.64:6827/235474426] boot 2023-06-10T21:28:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: osdmap e250: 8 total, 8 up, 6 in 2023-06-10T21:28:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-10T21:28:58.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:57 smithi152 ceph-mon[156819]: osdmap e251: 8 total, 8 up, 6 in 2023-06-10T21:28:58.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:28:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:58.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:28:58.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:28:58.693 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:59.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:28:59.076 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: pgmap v878: 89 pgs: 2 remapped+peering, 9 peering, 3 active+clean+remapped, 37 active+clean, 7 active+undersized+degraded, 31 active+undersized; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6645/157905 objects degraded (4.208%); 10089/157905 objects misplaced (6.389%) 2023-06-10T21:28:59.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[186706]: osdmap e252: 8 total, 8 up, 6 in 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: pgmap v878: 89 pgs: 2 remapped+peering, 9 peering, 3 active+clean+remapped, 37 active+clean, 7 active+undersized+degraded, 31 active+undersized; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6645/157905 objects degraded (4.208%); 10089/157905 objects misplaced (6.389%) 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:58 smithi064 ceph-mon[180297]: osdmap e252: 8 total, 8 up, 6 in 2023-06-10T21:28:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:59.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:28:59.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:28:59.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: pgmap v878: 89 pgs: 2 remapped+peering, 9 peering, 3 active+clean+remapped, 37 active+clean, 7 active+undersized+degraded, 31 active+undersized; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 6645/157905 objects degraded (4.208%); 10089/157905 objects misplaced (6.389%) 2023-06-10T21:28:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: failed to encode map e251 with expected crc 2023-06-10T21:28:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:58 smithi152 ceph-mon[156819]: osdmap e252: 8 total, 8 up, 6 in 2023-06-10T21:28:59.207 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:59.249 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:28:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:59.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:28:59.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 1 1 2023-06-10T21:28:59.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:28:59.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:28:59.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:28:59.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:28:59.762 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:28:59.765 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:28:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:28:59.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:28:59.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:28:59.867 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:28:59.883 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:28:59.927 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:28:59.987 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:28:59.990 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:00.035 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:29:00.085 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:29:00.088 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.089 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:29:00.247 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:29:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:29:00.247 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:29:00.248 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:29:00.248 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:29:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[180297]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[180297]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[180297]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[180297]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3396851459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[186706]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.270 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[186706]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.271 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[186706]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.271 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[186706]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.271 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:28:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3396851459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:59 smithi152 ceph-mon[156819]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:59 smithi152 ceph-mon[156819]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:59 smithi152 ceph-mon[156819]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:59 smithi152 ceph-mon[156819]: failed to encode map e252 with expected crc 2023-06-10T21:29:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:28:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3396851459' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:01.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:01.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:01.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:00 smithi064 ceph-mon[186706]: pgmap v881: 89 pgs: 7 remapped+peering, 9 peering, 47 active+clean, 3 active+undersized+degraded, 23 active+undersized; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 3/154035 objects degraded (0.002%) 2023-06-10T21:29:01.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-10T21:29:01.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:00 smithi064 ceph-mon[180297]: pgmap v881: 89 pgs: 7 remapped+peering, 9 peering, 47 active+clean, 3 active+undersized+degraded, 23 active+undersized; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 3/154035 objects degraded (0.002%) 2023-06-10T21:29:01.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-10T21:29:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:00 smithi152 ceph-mon[156819]: pgmap v881: 89 pgs: 7 remapped+peering, 9 peering, 47 active+clean, 3 active+undersized+degraded, 23 active+undersized; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 3/154035 objects degraded (0.002%) 2023-06-10T21:29:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-10T21:29:01.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:01.524 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:01.574 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:01.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:01.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:01.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:01.716 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:01.765 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:01.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:01.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:01.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:01.959 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:01.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:01.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:02.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:02.092 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:02.155 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:02.159 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:02 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized) 2023-06-10T21:29:02.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:02.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:02.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:02.288 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:02.338 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:02.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:02.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:02 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized) 2023-06-10T21:29:02.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:02 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/154035 objects degraded (0.002%), 3 pgs degraded, 18 pgs undersized) 2023-06-10T21:29:02.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:02.476 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:02.886 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:02] "GET /metrics HTTP/1.1" 200 223831 "" "Prometheus/2.33.4" 2023-06-10T21:29:03.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:03.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:03.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:29:03.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:03.142 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:03.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:03 smithi064 ceph-mon[180297]: pgmap v882: 89 pgs: 7 remapped+peering, 9 peering, 73 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 502 op/s 2023-06-10T21:29:03.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:03 smithi064 ceph-mon[186706]: pgmap v882: 89 pgs: 7 remapped+peering, 9 peering, 73 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 502 op/s 2023-06-10T21:29:03.164 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:03.189 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:03.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:03.235 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:03.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:03.320 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:03 smithi152 ceph-mon[156819]: pgmap v882: 89 pgs: 7 remapped+peering, 9 peering, 73 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 502 op/s 2023-06-10T21:29:03.323 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:03.373 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:03.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:03.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:03.513 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:03.562 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:03.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:03.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:03.698 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:03.746 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:03.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:03.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:03.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:04.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:04.169 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:04.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:04.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:04.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:04.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:04.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:29:05.339 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:05.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:05 smithi064 ceph-mon[186706]: pgmap v883: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 81 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 15626/152100 objects degraded (10.274%); 0 B/s, 0 objects/s recovering 2023-06-10T21:29:05.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:05 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 15626/152100 objects degraded (10.274%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:05.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:05.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:05 smithi064 ceph-mon[180297]: pgmap v883: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 81 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 15626/152100 objects degraded (10.274%); 0 B/s, 0 objects/s recovering 2023-06-10T21:29:05.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:05 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 15626/152100 objects degraded (10.274%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:05.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:05 smithi152 ceph-mon[156819]: pgmap v883: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 81 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 15626/152100 objects degraded (10.274%); 0 B/s, 0 objects/s recovering 2023-06-10T21:29:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:05 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 15626/152100 objects degraded (10.274%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:05.449 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:05.485 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:05.537 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:05.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:05.643 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:05.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:06.301 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:06.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:06.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:06.375 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:06.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:06.448 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:06.499 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:06.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:06.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:06.712 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:06.849 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:06.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:06.959 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:06.987 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:07.038 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:07.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:07.108 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:07.145 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:07.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:07.217 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:07.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:07.325 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:07.351 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:07 smithi152 ceph-mon[156819]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 37205/148317 objects degraded (25.085%); 991 KiB/s, 16 objects/s recovering 2023-06-10T21:29:07.354 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:07.405 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:07.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:07 smithi064 ceph-mon[180297]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 37205/148317 objects degraded (25.085%); 991 KiB/s, 16 objects/s recovering 2023-06-10T21:29:07.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:07 smithi064 ceph-mon[186706]: pgmap v884: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 399 op/s; 37205/148317 objects degraded (25.085%); 991 KiB/s, 16 objects/s recovering 2023-06-10T21:29:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:07.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:07.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:07.639 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:07.889 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:07.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:07.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:08.008 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:08.035 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:08.096 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:08.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:08.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:08.229 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:08.263 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:29:08.284 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:08.288 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/529785867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.289 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:08 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.289 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:29:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:08.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:08.295 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-06-10T21:29:08.391 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:08.424 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:08.472 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:08.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:08.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:08.578 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:08.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/529785867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.651 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.652 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:29:08.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/529785867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:08.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:29:08.670 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:09.460 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:09.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:09.541 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 37205/144852 objects degraded (25.685%); 811 KiB/s, 13 objects/s recovering 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: osdmap e253: 8 total, 8 up, 6 in 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: failed to encode map e253 with expected crc 2023-06-10T21:29:09.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[180297]: failed to encode map e253 with expected crc 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 37205/144852 objects degraded (25.685%); 811 KiB/s, 13 objects/s recovering 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: osdmap e253: 8 total, 8 up, 6 in 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: failed to encode map e253 with expected crc 2023-06-10T21:29:09.544 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:09 smithi064 ceph-mon[186706]: failed to encode map e253 with expected crc 2023-06-10T21:29:09.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:09.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:09.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: pgmap v885: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 37205/144852 objects degraded (25.685%); 811 KiB/s, 13 objects/s recovering 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1519969915' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: osdmap e253: 8 total, 8 up, 6 in 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: failed to encode map e253 with expected crc 2023-06-10T21:29:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:09 smithi152 ceph-mon[156819]: failed to encode map e253 with expected crc 2023-06-10T21:29:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 37062/140784 objects degraded (26.325%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:10.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 37062/140784 objects degraded (26.325%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:10.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 37062/140784 objects degraded (26.325%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:11.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:11 smithi064 ceph-mon[180297]: pgmap v887: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:11 smithi064 ceph-mon[186706]: pgmap v887: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:11 smithi152 ceph-mon[156819]: pgmap v887: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:11.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:29:11.901 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:11.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:12.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:12.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:12.278 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:12.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:12.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:12.465 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:12.659 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:12.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:12.685 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:29:12.686 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:29:12.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:29:12.686 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:29:12.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.686 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.25280.0:49 2.2 2:4e99cc3e:::rbd_mirror_snapshot_schedule:head [omap-get-vals] snapc 0=[] RETRY=2 ondisk+retry+read+known_if_redirected+supports_pool_eio e244)", 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.196518740000002, 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0624303289999999, 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.25280", 2023-06-10T21:29:12.687 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2947718283", 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 49 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:29:12.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479548+0000", 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479556+0000", 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3620000000000006e-06 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479566+0000", 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5880000000000004e-06 2023-06-10T21:29:12.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479573+0000", 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.046e-06 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487169+0000", 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075960059999999998 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487178+0000", 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7660000000000004e-06 2023-06-10T21:29:12.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541881+0000", 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.054703094 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541980+0000", 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8992000000000004e-05 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.695 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111488 10.4 10:2ebf3782:::benchmark_data_smithi064_312713_object20732:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e244)", 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.193930618000003, 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058806785, 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.696 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111488 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:29:12.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482137+0000", 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482142+0000", 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:29:12.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482152+0000", 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482162+0000", 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0344e-05 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482209+0000", 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7237000000000002e-05 2023-06-10T21:29:12.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482218+0000", 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2040000000000003e-06 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525337+0000", 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.043118188 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525473+0000", 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136673 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525610+0000", 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136231 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532648+0000", 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070388239999999999 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540822+0000", 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008173441 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540860+0000", 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7815999999999998e-05 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540944+0000", 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4839999999999994e-05 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111524 10.4 10:203e818a:::benchmark_data_smithi064_312713_object20768:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:29:12.707 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.193893248999998, 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0589736080000001, 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111524 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:29:12.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482178+0000", 2023-06-10T21:29:12.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2090000000000001e-06 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482184+0000", 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6080000000000001e-06 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482192+0000", 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:29:12.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5090000000000003e-05 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6470000000000004e-06 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532673+0000", 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0504169059999999 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532795+0000", 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122521 2023-06-10T21:29:12.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532918+0000", 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012313 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538256+0000", 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053373800000000001 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541023+0000", 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002766967 2023-06-10T21:29:12.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541067+0000", 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4270000000000001e-05 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541149+0000", 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1736000000000005e-05 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.719 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111526 10.4 10:21f5efce:::benchmark_data_smithi064_312713_object20770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.193869988000003, 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0590798290000001, 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.720 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111526 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:29:12.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482201+0000", 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6919999999999998e-06 2023-06-10T21:29:12.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482207+0000", 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1460000000000001e-06 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482221+0000", 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3876999999999999e-05 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7775999999999997e-05 2023-06-10T21:29:12.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482285+0000", 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1879999999999999e-06 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535227+0000", 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0529423769999999 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535306+0000", 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8954000000000003e-05 2023-06-10T21:29:12.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535404+0000", 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8250000000000003e-05 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538295+0000", 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028906679999999999 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541164+0000", 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028689599999999998 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541202+0000", 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8213999999999997e-05 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541278+0000", 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6106999999999995e-05 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111533 10.4 10:20aecf0c:::benchmark_data_smithi064_312713_object20777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:29:12.731 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.193823821000002, 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0641654469999999, 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.732 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111533 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:29:12.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.315e-06 2023-06-10T21:29:12.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482263+0000", 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.063e-06 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482314+0000", 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1063999999999998e-05 2023-06-10T21:29:12.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482321+0000", 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8589999999999999e-06 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538345+0000", 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.056023894 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538495+0000", 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149498 2023-06-10T21:29:12.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538638+0000", 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014375100000000001 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541316+0000", 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026775509999999998 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546267+0000", 2023-06-10T21:29:12.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004951324 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546317+0000", 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9876999999999997e-05 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546410+0000", 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2786000000000002e-05 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.743 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111555 10.4 10:24e69e32:::benchmark_data_smithi064_312713_object20799:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 39.193790608, 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0654074570000001, 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.744 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111555 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:29:12.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482277+0000", 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482280+0000", 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5069999999999999e-06 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482286+0000", 2023-06-10T21:29:12.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4180000000000002e-06 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482292+0000", 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3399999999999997e-06 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483443+0000", 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151383 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483452+0000", 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1309999999999995e-06 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541485+0000", 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058032919 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541643+0000", 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158212 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:29:12.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541783+0000", 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140111 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.544945+0000", 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031613840000000002 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547575+0000", 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026302230000000001 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547622+0000", 2023-06-10T21:29:12.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7429999999999998e-05 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547685+0000", 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2838999999999999e-05 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.755 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111621 10.a 10:56721995:::benchmark_data_smithi064_312713_object20865:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.118791838999996, 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96906832399999998, 2023-06-10T21:29:12.756 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111621 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:29:12.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557279+0000", 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.503e-06 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557289+0000", 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0631999999999999e-05 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557294+0000", 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7770000000000002e-06 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557348+0000", 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3779999999999997e-05 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557364+0000", 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6325e-05 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.516904+0000", 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95953967500000004 2023-06-10T21:29:12.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517083+0000", 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017900899999999999 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517231+0000", 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526113+0000", 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088829010000000003 2023-06-10T21:29:12.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526152+0000", 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8665999999999999e-05 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526194+0000", 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2048000000000001e-05 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526241+0000", 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7145999999999999e-05 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526345+0000", 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103491 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111622 10.a 10:5c194bad:::benchmark_data_smithi064_312713_object20866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.115928904, 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97161865400000003, 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111622 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560138+0000", 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560142+0000", 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4659999999999999e-06 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560149+0000", 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0509999999999999e-06 2023-06-10T21:29:12.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560155+0000", 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8510000000000001e-06 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560208+0000", 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2911000000000002e-05 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560219+0000", 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0859999999999999e-05 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520194+0000", 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95997470299999998 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520350+0000", 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015635700000000001 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520521+0000", 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017028799999999999 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526160+0000", 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056392309999999998 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531566+0000", 2023-06-10T21:29:12.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054064539999999998 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531604+0000", 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7854e-05 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531651+0000", 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7215000000000001e-05 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531758+0000", 2023-06-10T21:29:12.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010678100000000001 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111629 10.a 10:56a08505:::benchmark_data_smithi064_312713_object20873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.781 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.108549586000002, 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97554071600000003, 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.782 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111629 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567517+0000", 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567525+0000", 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.994e-06 2023-06-10T21:29:12.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567537+0000", 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1922999999999999e-05 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567545+0000", 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5900000000000002e-06 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567602+0000", 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7065999999999999e-05 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567613+0000", 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523367+0000", 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95575399000000005 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523517+0000", 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014977799999999999 2023-06-10T21:29:12.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523672+0000", 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015482899999999999 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538459+0000", 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0147876 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540585+0000", 2023-06-10T21:29:12.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021253919999999998 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542925+0000", 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023401250000000002 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542957+0000", 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2283999999999998e-05 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543059+0000", 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102527 2023-06-10T21:29:12.793 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111634 10.a 10:5fd06ea2:::benchmark_data_smithi064_312713_object20878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:29:12.794 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.099350483999999, 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.969888847, 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111634 2023-06-10T21:29:12.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576717+0000", 2023-06-10T21:29:12.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576720+0000", 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6340000000000001e-06 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576727+0000", 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.216e-06 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576733+0000", 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4319999999999998e-06 2023-06-10T21:29:12.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576789+0000", 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6227999999999998e-05 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576800+0000", 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0572e-05 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526379+0000", 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94957964100000003 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526493+0000", 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113999 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526618+0000", 2023-06-10T21:29:12.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495299999999999 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540594+0000", 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013976018999999999 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546331+0000", 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005736568 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546401+0000", 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0585000000000003e-05 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546466+0000", 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4929000000000001e-05 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546607+0000", 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014047500000000001 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111642 10.a 10:5fdb9826:::benchmark_data_smithi064_312713_object20886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.086599858, 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96564841499999998, 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111642 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589467+0000", 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589472+0000", 2023-06-10T21:29:12.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0490000000000003e-06 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589483+0000", 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0729e-05 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589490+0000", 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589573+0000", 2023-06-10T21:29:12.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3007e-05 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589585+0000", 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1836e-05 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528718+0000", 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93913300899999996 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528872+0000", 2023-06-10T21:29:12.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154108 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.529036+0000", 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164017 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540601+0000", 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011564677000000001 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546679+0000", 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060784560000000003 2023-06-10T21:29:12.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555011+0000", 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083312960000000002 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555038+0000", 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7032000000000001e-05 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555117+0000", 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9177000000000004e-05 2023-06-10T21:29:12.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111646 10.a 10:58b447f4:::benchmark_data_smithi064_312713_object20890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:29:12.819 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.083317446999999, 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96249102799999997, 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111646 2023-06-10T21:29:12.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592750+0000", 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592755+0000", 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6600000000000003e-06 2023-06-10T21:29:12.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592764+0000", 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4059999999999994e-06 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592769+0000", 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2590000000000002e-06 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592816+0000", 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7280000000000001e-05 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592823+0000", 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1199999999999996e-06 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531787+0000", 2023-06-10T21:29:12.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93896378800000002 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531899+0000", 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111659 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.532018+0000", 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119275 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540608+0000", 2023-06-10T21:29:12.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085904399999999995 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546763+0000", 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061551050000000001 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555159+0000", 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008395563 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555184+0000", 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5528e-05 2023-06-10T21:29:12.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555242+0000", 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7454000000000003e-05 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111648 10.a 10:574e8e24:::benchmark_data_smithi064_312713_object20892:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.078771828000001, 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.958220356, 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.833 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111648 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597295+0000", 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597300+0000", 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0309999999999998e-06 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597309+0000", 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8089999999999999e-06 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597316+0000", 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6630000000000004e-06 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597417+0000", 2023-06-10T21:29:12.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100995 2023-06-10T21:29:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597432+0000", 2023-06-10T21:29:12.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5517000000000001e-05 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534105+0000", 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93667285600000005 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534213+0000", 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010740599999999999 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534345+0000", 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132714 2023-06-10T21:29:12.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540617+0000", 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062720099999999997 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.552679+0000", 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012062154 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555339+0000", 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026591420000000002 2023-06-10T21:29:12.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555401+0000", 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2757999999999998e-05 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555517+0000", 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115498 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.845 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111672 10.d 10:bbd07a43:::benchmark_data_smithi064_312713_object20916:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.631097+0000", 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.04497147, 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88206613599999995, 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.846 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111672 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631097+0000", 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631097+0000", 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631095+0000", 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631103+0000", 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1119999999999996e-06 2023-06-10T21:29:12.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631121+0000", 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7516e-05 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631130+0000", 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3840000000000003e-06 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631191+0000", 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1041000000000001e-05 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.631206+0000", 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4256e-05 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.504292+0000", 2023-06-10T21:29:12.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.873086747 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.504477+0000", 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018453299999999999 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.504650+0000", 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017352599999999999 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.511041+0000", 2023-06-10T21:29:12.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063902129999999996 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.512981+0000", 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001940867 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.513043+0000", 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2058999999999997e-05 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.513083+0000", 2023-06-10T21:29:12.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9542999999999997e-05 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.513163+0000", 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9985000000000001e-05 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111681 10.a 10:560529bb:::benchmark_data_smithi064_312713_object20925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 38.032670861, 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91238480600000005, 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.858 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111681 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:29:12.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643395+0000", 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643408+0000", 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2588e-05 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643431+0000", 2023-06-10T21:29:12.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3011999999999999e-05 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643444+0000", 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3047000000000001e-05 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643726+0000", 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028253799999999998 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643738+0000", 2023-06-10T21:29:12.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1908e-05 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536216+0000", 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89247793399999997 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536328+0000", 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112285 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536430+0000", 2023-06-10T21:29:12.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101981 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546177+0000", 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097468780000000005 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555141+0000", 2023-06-10T21:29:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089643450000000003 2023-06-10T21:29:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555638+0000", 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049625199999999998 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555676+0000", 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8074000000000002e-05 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555782+0000", 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106461 2023-06-10T21:29:12.870 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111709 10.a 10:599039aa:::benchmark_data_smithi064_312713_object20953:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.685608+0000", 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.990460016, 2023-06-10T21:29:12.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87053228500000002, 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111709 2023-06-10T21:29:12.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685608+0000", 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685608+0000", 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685606+0000", 2023-06-10T21:29:12.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685614+0000", 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6939999999999995e-06 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685632+0000", 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7765e-05 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685641+0000", 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6950000000000006e-06 2023-06-10T21:29:12.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685696+0000", 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5080000000000001e-05 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.685706+0000", 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0291e-05 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538487+0000", 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85278087499999999 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538584+0000", 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6918999999999998e-05 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538698+0000", 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114405 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546344+0000", 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076453959999999996 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555296+0000", 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089523659999999998 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556007+0000", 2023-06-10T21:29:12.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000711479 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556054+0000", 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6576000000000003e-05 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556141+0000", 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6619000000000005e-05 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111720 10.d 10:be5a9979:::benchmark_data_smithi064_312713_object20964:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.696397+0000", 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.979670853999998, 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.82079966599999998, 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.884 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111720 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696397+0000", 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696397+0000", 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696396+0000", 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696403+0000", 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9149999999999999e-06 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696419+0000", 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6588000000000001e-05 2023-06-10T21:29:12.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696427+0000", 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9109999999999993e-06 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696492+0000", 2023-06-10T21:29:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5049999999999996e-05 2023-06-10T21:29:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.696503+0000", 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0712e-05 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.506989+0000", 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81048547199999998 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.507149+0000", 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000160347 2023-06-10T21:29:12.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.507340+0000", 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000190896 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.512922+0000", 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0055819329999999999 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.513216+0000", 2023-06-10T21:29:12.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029406300000000001 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517087+0000", 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038710110000000002 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517131+0000", 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4613000000000001e-05 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517197+0000", 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5652000000000001e-05 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.898 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111730 10.a 10:5eda315a:::benchmark_data_smithi064_312713_object20974:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.719706+0000", 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.956362503999998, 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.83660798300000006, 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.899 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111730 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719706+0000", 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719706+0000", 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719705+0000", 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719711+0000", 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4400000000000002e-06 2023-06-10T21:29:12.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719723+0000", 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.253e-05 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719729+0000", 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9240000000000004e-06 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719777+0000", 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7589000000000001e-05 2023-06-10T21:29:12.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.719790+0000", 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2794e-05 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540650+0000", 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.82086048899999997 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540763+0000", 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112365 2023-06-10T21:29:12.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540857+0000", 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4016000000000002e-05 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546352+0000", 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054950900000000002 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555578+0000", 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092257959999999996 2023-06-10T21:29:12.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556179+0000", 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000601043 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556222+0000", 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3903000000000003e-05 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.556314+0000", 2023-06-10T21:29:12.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1298000000000006e-05 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111743 10.a 10:56db7135:::benchmark_data_smithi064_312713_object20987:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.739177+0000", 2023-06-10T21:29:12.911 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.936891660999997, 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81836536299999996, 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111743 2023-06-10T21:29:12.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739177+0000", 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739177+0000", 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739176+0000", 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739181+0000", 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5650000000000001e-06 2023-06-10T21:29:12.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739196+0000", 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4231e-05 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739203+0000", 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6790000000000005e-06 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739264+0000", 2023-06-10T21:29:12.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0618000000000001e-05 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.739279+0000", 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5119999999999999e-05 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543412+0000", 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.80413305499999999 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543523+0000", 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110739 2023-06-10T21:29:12.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543642+0000", 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011883399999999999 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546691+0000", 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030490529999999999 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555837+0000", 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091465339999999996 2023-06-10T21:29:12.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.557415+0000", 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015773359999999999 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.557455+0000", 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0787000000000001e-05 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.557542+0000", 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6706999999999995e-05 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.923 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111765 10.d 10:b4f92082:::benchmark_data_smithi064_312713_object21009:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.772547+0000", 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 37.903521290999997, 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74700785400000003, 2023-06-10T21:29:12.924 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111765 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:12.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772547+0000", 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:12.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772547+0000", 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772546+0000", 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772552+0000", 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.976e-06 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772565+0000", 2023-06-10T21:29:12.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2978e-05 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772575+0000", 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9960000000000005e-06 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772647+0000", 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1800999999999999e-05 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.772663+0000", 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6603e-05 2023-06-10T21:29:12.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.509153+0000", 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73648993299999999 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:12.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.509247+0000", 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3500999999999998e-05 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.509334+0000", 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7118000000000002e-05 2023-06-10T21:29:12.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.513185+0000", 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038512920000000001 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.515491+0000", 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023063150000000002 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.519394+0000", 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003902651 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.519446+0000", 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1820000000000002e-05 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:29:12.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.519555+0000", 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109089 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:12.936 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:29:13.027 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:12] "GET /metrics HTTP/1.1" 200 223588 "" "Prometheus/2.33.4" 2023-06-10T21:29:13.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:13.184 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:29:13.185 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:29:13.185 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:29:13.205 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:29:13.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:29:13.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:13.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:29:13.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:13.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:13.341 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:29:13.376 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:29:13.411 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:13.428 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.428 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.428 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.428 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.428 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.429 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.429 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.429 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:29:13.441 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:13.493 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:13.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:13.598 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:13.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:13 smithi152 ceph-mon[156819]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:13.622 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/608730206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:13.624 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:13 smithi064 ceph-mon[186706]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/608730206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:13 smithi064 ceph-mon[180297]: pgmap v888: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 37062/140784 objects degraded (26.325%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:13.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/608730206' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:13.665 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:13.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:13.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:13.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:29:13.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:13.799 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:13.852 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:13.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:13.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:13.990 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.041 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:14.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:14.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:14.177 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.225 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:14.334 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:14.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:29:14.364 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.416 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:14.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:14.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:14.552 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.598 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:14.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:14.735 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.786 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:14.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:14.920 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:14.971 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:14.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:15.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:15.107 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:15.157 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:15.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:15.186 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:15.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:15.297 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:15.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:15.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:15.394 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:15 smithi152 ceph-mon[156819]: pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 37062/137526 objects degraded (26.949%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:15.454 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:15.485 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:15.529 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:15.531 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:15.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:15.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:15.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:15 smithi064 ceph-mon[180297]: pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 37062/137526 objects degraded (26.949%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:15.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:15 smithi064 ceph-mon[186706]: pgmap v889: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 37062/137526 objects degraded (26.949%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:15.638 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:15.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:16.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:16.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:16.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:16.264 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:16.313 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:16.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:16.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:16.505 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:16 smithi064 ceph-mon[180297]: pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36919/133437 objects degraded (27.668%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 37062/137526 objects degraded (26.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:16 smithi064 ceph-mon[186706]: pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36919/133437 objects degraded (27.668%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:16.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 37062/137526 objects degraded (26.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:17.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:16 smithi152 ceph-mon[156819]: pgmap v890: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36919/133437 objects degraded (27.668%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:17.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 37062/137526 objects degraded (26.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:17.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:29:17.450 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:17.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:17.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:17.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:17.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:18.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:18.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:18.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:18.690 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:18.998 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:19.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:19.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[180297]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36919/130176 objects degraded (28.361%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:19.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:19.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[186706]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36919/130176 objects degraded (28.361%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:19.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:19.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:19 smithi152 ceph-mon[156819]: pgmap v891: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36919/130176 objects degraded (28.361%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:19.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:19.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:19.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:19.740 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:19.787 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:19.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:19.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:19.922 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:19.971 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:19.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:20.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:20.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:20.376 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:20.447 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:20.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:20.555 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:20.722 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:20.818 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:29:20.819 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:29:20.856 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:20.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:20.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:20.963 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:21.039 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:21.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:21.151 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:21.183 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:21.198 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:29:21.235 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:29:21.246 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:21.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 36790/126090 objects degraded (29.178%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:29:21.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1176008164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.249 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36790/126090 objects degraded (29.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:21.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3183137855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.250 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:21.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:21.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:21.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 36790/126090 objects degraded (29.178%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:29:21.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1176008164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36790/126090 objects degraded (29.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3183137855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: pgmap v892: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 36790/126090 objects degraded (29.178%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:29:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1176008164' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:29:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36790/126090 objects degraded (29.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3183137855' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:29:21.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:22.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:29:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:29:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: osdmap e254: 8 total, 8 up, 6 in 2023-06-10T21:29:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:22 smithi152 ceph-mon[156819]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.455 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:22.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:22.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:22.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:29:22.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:29:22.463 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: osdmap e254: 8 total, 8 up, 6 in 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[186706]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:29:22.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: osdmap e254: 8 total, 8 up, 6 in 2023-06-10T21:29:22.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:22 smithi064 ceph-mon[180297]: failed to encode map e254 with expected crc 2023-06-10T21:29:22.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:22.598 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:22.653 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:22.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:22.746 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:29:22.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:22.795 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:22.846 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:22.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:22.955 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:22.983 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:23.037 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:23.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:23.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:23.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:22] "GET /metrics HTTP/1.1" 200 223692 "" "Prometheus/2.33.4" 2023-06-10T21:29:23.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:23.221 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:23.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:23.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:23.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:23.411 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:23.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:23 smithi152 ceph-mon[156819]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 36790/126090 objects degraded (29.178%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:23 smithi064 ceph-mon[186706]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 36790/126090 objects degraded (29.178%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:23.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:23 smithi064 ceph-mon[180297]: pgmap v894: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 36790/126090 objects degraded (29.178%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:24.106 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:24.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:24.229 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:29:24.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:24.290 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:29:24.294 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:24.359 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:24.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:24.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:24.494 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:24.597 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:24.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:24.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:24.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:29:24.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:24.744 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:24.806 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:24.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:24.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 4 1 2023-06-10T21:29:24.913 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:25.024 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:25.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:25.347 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:25.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:25.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:25.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:25 smithi064 ceph-mon[180297]: pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36790/122880 objects degraded (29.940%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:25.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:25 smithi064 ceph-mon[186706]: pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36790/122880 objects degraded (29.940%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:25.550 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:25 smithi152 ceph-mon[156819]: pgmap v895: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.5 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 36790/122880 objects degraded (29.940%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:25.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:25.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:25.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:29:25.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:25.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:29:25.846 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:25.878 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:29:25.906 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:25.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:25.930 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:29:25.982 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:29:26.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:26.047 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:26.054 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:29:26.104 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:29:26.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:26.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:26.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:29:26.170 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:29:26.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:26.271 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36790/122880 objects degraded (29.940%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3858284688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:26.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36790/122880 objects degraded (29.940%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:26.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3858284688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:26.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36790/122880 objects degraded (29.940%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:26.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3858284688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:26.870 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:29:27.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[186706]: pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 36652/118899 objects degraded (30.826%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:27.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[180297]: pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 36652/118899 objects degraded (30.826%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:27.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:27 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:27.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:27.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:27 smithi152 ceph-mon[156819]: pgmap v896: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 36652/118899 objects degraded (30.826%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:27 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:27 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:27.774 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:27.810 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:27.862 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:27.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:27.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:28.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:28.399 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:28.399 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:28.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:28.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:28.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:28.644 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:29.065 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:29.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:29.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:29.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:29.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:29.340 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:29:29.341 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:29:29.341 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:29:29.341 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:29:29.341 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:29:29.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:29 smithi064 ceph-mon[180297]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36652/115602 objects degraded (31.705%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:29.535 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:29 smithi064 ceph-mon[186706]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36652/115602 objects degraded (31.705%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:29.544 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:29.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:29.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:29 smithi152 ceph-mon[156819]: pgmap v897: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36652/115602 objects degraded (31.705%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:29.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:30.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:29:30.540 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:30.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:30.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:30.691 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:30.746 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:30.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:30.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:30.990 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:31 smithi064 ceph-mon[180297]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36515/111519 objects degraded (32.743%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36515/111519 objects degraded (32.743%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:31.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:31 smithi064 ceph-mon[186706]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36515/111519 objects degraded (32.743%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36515/111519 objects degraded (32.743%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:31 smithi152 ceph-mon[156819]: pgmap v898: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 36515/111519 objects degraded (32.743%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:29:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36515/111519 objects degraded (32.743%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:31.778 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:31.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:31.874 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:31.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:31.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:29:32.031 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:32.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:32.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:32.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:32.399 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:32.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:32.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:32.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:32.584 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:32.630 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:32.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:32.666 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:32.740 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:32.767 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:32.821 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:32.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:32.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:32.926 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:33.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:32] "GET /metrics HTTP/1.1" 200 223692 "" "Prometheus/2.33.4" 2023-06-10T21:29:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:33.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:33.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:33 smithi064 ceph-mon[180297]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 36515/111519 objects degraded (32.743%); 1.6 MiB/s, 26 objects/s recovering 2023-06-10T21:29:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:33 smithi064 ceph-mon[186706]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 36515/111519 objects degraded (32.743%); 1.6 MiB/s, 26 objects/s recovering 2023-06-10T21:29:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:33 smithi152 ceph-mon[156819]: pgmap v899: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 462 op/s; 36515/111519 objects degraded (32.743%); 1.6 MiB/s, 26 objects/s recovering 2023-06-10T21:29:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:33.759 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:34.117 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:34.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:34.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:34.323 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:34.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:29:34.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:29:34.401 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-06-10T21:29:34.521 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:34.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:34.627 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:29:34.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:34.994 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:34.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:35.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:35.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:35.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[186706]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 36515/108297 objects degraded (33.717%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:29:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:29:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[186706]: osdmap e255: 8 total, 8 up, 6 in 2023-06-10T21:29:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[186706]: failed to encode map e255 with expected crc 2023-06-10T21:29:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[180297]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 36515/108297 objects degraded (33.717%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:29:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:29:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[180297]: osdmap e255: 8 total, 8 up, 6 in 2023-06-10T21:29:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:35 smithi064 ceph-mon[180297]: failed to encode map e255 with expected crc 2023-06-10T21:29:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:35 smithi152 ceph-mon[156819]: pgmap v900: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 494 op/s; 36515/108297 objects degraded (33.717%); 1.4 MiB/s, 22 objects/s recovering 2023-06-10T21:29:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3762645988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:29:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:35 smithi152 ceph-mon[156819]: osdmap e255: 8 total, 8 up, 6 in 2023-06-10T21:29:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:35 smithi152 ceph-mon[156819]: failed to encode map e255 with expected crc 2023-06-10T21:29:36.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:36 smithi064 ceph-mon[180297]: failed to encode map e255 with expected crc 2023-06-10T21:29:36.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36515/108297 objects degraded (33.717%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:36.575 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:36.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:29:36.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:36 smithi064 ceph-mon[186706]: failed to encode map e255 with expected crc 2023-06-10T21:29:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36515/108297 objects degraded (33.717%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:36 smithi152 ceph-mon[156819]: failed to encode map e255 with expected crc 2023-06-10T21:29:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36515/108297 objects degraded (33.717%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:37.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:37.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:37.139 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:37.166 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:37.211 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:37.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:37.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:37.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:37.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:37 smithi152 ceph-mon[156819]: pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 36383/104010 objects degraded (34.980%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:37.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:37 smithi064 ceph-mon[186706]: pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 36383/104010 objects degraded (34.980%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:37.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:37 smithi064 ceph-mon[180297]: pgmap v902: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 36383/104010 objects degraded (34.980%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:37.941 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:37.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:38.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:38.085 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:38.138 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:38.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:38.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:38.274 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:38.322 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:38.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:38.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:38.452 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:29:38.455 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:38.502 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:38.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:38.612 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:38.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3416799102' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:29:38.720 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:29:38.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:38.746 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3416799102' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.746 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.746 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3416799102' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:29:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:38.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:38.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:29:38.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:38.881 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:38.895 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:29:38.930 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:38.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:38.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:38.952 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:29:38.998 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:29:39.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:39.038 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:29:39.063 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:39.081 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:29:39.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.093 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:29:39.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:39.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:39.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:39.246 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:39.293 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:39.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:39.400 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:39.432 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:39.487 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:39.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:39.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:39.687 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36383/100713 objects degraded (36.125%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:29:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: osdmap e256: 8 total, 8 up, 6 in 2023-06-10T21:29:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3805051472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:39 smithi152 ceph-mon[156819]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36383/100713 objects degraded (36.125%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:39.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: osdmap e256: 8 total, 8 up, 6 in 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3805051472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:39.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[180297]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: pgmap v903: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 36383/100713 objects degraded (36.125%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:29:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:29:39.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: osdmap e256: 8 total, 8 up, 6 in 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: failed to encode map e256 with expected crc 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3805051472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:39.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:39 smithi064 ceph-mon[186706]: failed to encode map e256 with expected crc 2023-06-10T21:29:41.246 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:41.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:41.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:41.382 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:41.433 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:41.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:41.457 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:41.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:41.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:41 smithi064 ceph-mon[180297]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:41.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36238/96696 objects degraded (37.476%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:41.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:41 smithi064 ceph-mon[186706]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:41.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36238/96696 objects degraded (37.476%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:41.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:41 smithi152 ceph-mon[156819]: pgmap v905: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36238/96696 objects degraded (37.476%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:42.182 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:29:42.182 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:29:42.816 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:42.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:42.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:43.027 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:43.079 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:29:43.129 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:42] "GET /metrics HTTP/1.1" 200 223710 "" "Prometheus/2.33.4" 2023-06-10T21:29:43.419 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:29:43.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:43 smithi064 ceph-mon[180297]: pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:43 smithi064 ceph-mon[186706]: pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:43 smithi152 ceph-mon[156819]: pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 482 op/s; 36238/96696 objects degraded (37.476%); 2.2 MiB/s, 34 objects/s recovering 2023-06-10T21:29:44.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:44.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:44.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:44.406 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:44.453 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:44.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:44.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:29:44.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:44.589 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:44.644 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:44.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:44.753 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:29:44.780 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:44.829 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:29:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:44.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:29:44.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:44.964 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:45.010 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:45.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:45.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:45.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:45.311 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:45.466 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:45 smithi064 ceph-mon[180297]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 36238/93510 objects degraded (38.753%); 993 KiB/s, 15 objects/s recovering 2023-06-10T21:29:45.474 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:45.722 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:45.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:45 smithi064 ceph-mon[186706]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 36238/93510 objects degraded (38.753%); 993 KiB/s, 15 objects/s recovering 2023-06-10T21:29:45.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:45.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:45.914 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:45.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:45 smithi152 ceph-mon[156819]: pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 36238/93510 objects degraded (38.753%); 993 KiB/s, 15 objects/s recovering 2023-06-10T21:29:46.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:46.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:46.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:46.313 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:46.454 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:46.741 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 36238/93510 objects degraded (38.753%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:46.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.742 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.743 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:29:46.743 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 36238/93510 objects degraded (38.753%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:46.743 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.744 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:29:46.803 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:29:46.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 36238/93510 objects degraded (38.753%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:29:47.038 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:47.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:47.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:47.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:47.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 36097/89247 objects degraded (40.446%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:47.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: failed to encode map e257 with expected crc 2023-06-10T21:29:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:29:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: osdmap e257: 8 total, 8 up, 6 in 2023-06-10T21:29:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:29:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:29:47.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 36097/89247 objects degraded (40.446%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: failed to encode map e257 with expected crc 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: osdmap e257: 8 total, 8 up, 6 in 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:29:47.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 36097/89247 objects degraded (40.446%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: failed to encode map e257 with expected crc 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3714307334' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: osdmap e257: 8 total, 8 up, 6 in 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:29:47.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:29:47.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:49.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[186706]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 36097/89247 objects degraded (40.446%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:49.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:49.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:49.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[180297]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 36097/89247 objects degraded (40.446%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:49.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:49.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:49 smithi152 ceph-mon[156819]: pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 412 op/s; 36097/89247 objects degraded (40.446%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:29:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:29:49.563 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:49.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:49.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:49.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:49.700 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:49.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 0 1 2023-06-10T21:29:49.754 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:49.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:49.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:49.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:49.967 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:50.244 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:50.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:50.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:50.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:50.426 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:50.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:50.533 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:29:50.565 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:50.616 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:29:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:50.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:29:50.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:29:50.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:50.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:29:50.801 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:29:50.869 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:50.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:29:50.909 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:29:50.948 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:29:50.989 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:29:51.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:29:51.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.070 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.071 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:29:51.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[180297]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:51.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/850119396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:51.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35945/82263 objects degraded (43.695%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[186706]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/850119396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:51.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35945/82263 objects degraded (43.695%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:51 smithi152 ceph-mon[156819]: pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/850119396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:29:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35945/82263 objects degraded (43.695%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:52.174 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:29:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2722102861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2722102861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:52.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2722102861' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:52.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:29:53.000 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:53.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:53.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:29:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:29:52] "GET /metrics HTTP/1.1" 200 223710 "" "Prometheus/2.33.4" 2023-06-10T21:29:53.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:53 smithi152 ceph-mon[156819]: pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:53 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:53 smithi152 ceph-mon[156819]: failed to encode map e258 with expected crc 2023-06-10T21:29:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:29:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:53 smithi152 ceph-mon[156819]: osdmap e258: 8 total, 8 up, 6 in 2023-06-10T21:29:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[186706]: pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[186706]: failed to encode map e258 with expected crc 2023-06-10T21:29:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:29:53.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[186706]: osdmap e258: 8 total, 8 up, 6 in 2023-06-10T21:29:53.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[180297]: pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35945/82263 objects degraded (43.695%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:29:53.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:29:53.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[180297]: failed to encode map e258 with expected crc 2023-06-10T21:29:53.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:29:53.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:53 smithi064 ceph-mon[180297]: osdmap e258: 8 total, 8 up, 6 in 2023-06-10T21:29:53.872 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:53.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:53.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:53.989 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:29:54.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:54.249 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:29:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:54.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:29:54.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:54.393 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:54.450 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:54.453 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:54.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:54.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:54.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:54.582 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:29:54.582 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:29:54.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.582 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.35769.0:156914 10.6 e258/244 10:6216a178:::benchmark_data_smithi064_312713_object66158:head v 258'9852, mlcod=258'9849)", 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:29:54.551916+0000", 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.023713702, 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023754560000000001, 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551916+0000", 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551916+0000", 2023-06-10T21:29:54.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551915+0000", 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551920+0000", 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3819999999999997e-06 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551926+0000", 2023-06-10T21:29:54.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7490000000000004e-06 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.551930+0000", 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.578e-06 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.555018+0000", 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030879919999999999 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:29:54.555039+0000", 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0803999999999999e-05 2023-06-10T21:29:54.588 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2023-06-10T21:29:54.589 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:29:54.590 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:54.629 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:54.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:54.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:54.842 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:54.982 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:54.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:54.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:55.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:29:55.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:55.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:55 smithi064 ceph-mon[186706]: pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 416 op/s; 35945/79236 objects degraded (45.364%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:29:55.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:55.217 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:55 smithi064 ceph-mon[180297]: pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 416 op/s; 35945/79236 objects degraded (45.364%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:29:55.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:55.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:55 smithi152 ceph-mon[156819]: pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 416 op/s; 35945/79236 objects degraded (45.364%); 1.2 MiB/s, 18 objects/s recovering 2023-06-10T21:29:55.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:55.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:56.034 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:56.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:29:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:56.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:56.056 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:29:56.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:56.180 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:56.232 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:56.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:56.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:56.293 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35945/79236 objects degraded (45.364%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:56.294 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35945/79236 objects degraded (45.364%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:56.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35945/79236 objects degraded (45.364%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:29:56.476 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:56.549 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:56.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:56.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:56.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:56.683 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:56.738 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:56.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:56.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:56.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:56.871 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:56.928 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:56.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:57.038 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:29:57.169 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:57.256 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:57.269 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:57.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:29:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:57.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:29:57.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:57 smithi064 ceph-mon[180297]: pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 35794/75324 objects degraded (47.520%); 2.1 MiB/s, 33 objects/s recovering 2023-06-10T21:29:57.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:57 smithi064 ceph-mon[186706]: pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 35794/75324 objects degraded (47.520%); 2.1 MiB/s, 33 objects/s recovering 2023-06-10T21:29:57.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:29:57.498 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:29:57.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:57.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:57 smithi152 ceph-mon[156819]: pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 35794/75324 objects degraded (47.520%); 2.1 MiB/s, 33 objects/s recovering 2023-06-10T21:29:58.077 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:29:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:58.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:29:58.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:58.213 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:58.261 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:58.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:58.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:58.471 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:59.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:29:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:59.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:29:59.233 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-06-10T21:29:59.278 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:29:59.304 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:59.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:29:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:59.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:29:59.453 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:29:59.485 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:59 smithi152 ceph-mon[156819]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35794/75324 objects degraded (47.520%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:59.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1307641676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.486 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:29:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.488 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:29:59.545 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:29:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:29:59.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:29:59.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[180297]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35794/75324 objects degraded (47.520%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:59.581 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1307641676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[186706]: pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.5 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35794/75324 objects degraded (47.520%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:29:59.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1307641676' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:29:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:29:59.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:29:59.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:00.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:30:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: osdmap e259: 8 total, 8 up, 6 in 2023-06-10T21:30:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-10T21:30:00.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: [WRN] PG_DEGRADED: Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.3 is active+recovering+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.5 is active+recovery_wait+undersized+degraded+remapped, acting [2,0] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[186706]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: osdmap e259: 8 total, 8 up, 6 in 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: [WRN] PG_DEGRADED: Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.3 is active+recovering+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.5 is active+recovery_wait+undersized+degraded+remapped, acting [2,0] 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2023-06-10T21:30:00.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:00 smithi064 ceph-mon[180297]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:30:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: osdmap e259: 8 total, 8 up, 6 in 2023-06-10T21:30:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: failed to encode map e259 with expected crc 2023-06-10T21:30:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: [WRN] PG_DEGRADED: Degraded data redundancy: 35794/75324 objects degraded (47.520%), 8 pgs degraded 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.3 is active+recovering+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.4 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.5 is active+recovery_wait+undersized+degraded+remapped, acting [2,0] 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.a is active+recovery_wait+undersized+degraded+remapped, acting [0,2] 2023-06-10T21:30:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.d is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2023-06-10T21:30:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:00.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:00 smithi152 ceph-mon[156819]: pg 10.f is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-06-10T21:30:01.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:01 smithi064 ceph-mon[180297]: pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 35656/68520 objects degraded (52.037%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:30:01.416 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35656/68520 objects degraded (52.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:01.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:01 smithi064 ceph-mon[186706]: pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 35656/68520 objects degraded (52.037%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:30:01.416 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35656/68520 objects degraded (52.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:01 smithi152 ceph-mon[156819]: pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 35656/68520 objects degraded (52.037%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:30:01.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35656/68520 objects degraded (52.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:02.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:30:02.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:02.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:02.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:02.299 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:02.489 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:02.890 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:02] "GET /metrics HTTP/1.1" 200 223710 "" "Prometheus/2.33.4" 2023-06-10T21:30:02.986 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:02.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:02.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:03.002 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:30:03.003 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:30:03.003 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:30:03.003 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:30:03.003 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:30:03.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:03.238 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:03.411 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:30:03.476 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:30:03.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:03.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:03.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[180297]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35656/68520 objects degraded (52.037%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:03.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:03.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:03.497 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[186706]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35656/68520 objects degraded (52.037%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:03.497 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:03.497 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:03.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:30:03.580 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:30:03.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:03.607 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:30:03.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:30:03.692 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:30:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:03 smithi152 ceph-mon[156819]: pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 35656/68520 objects degraded (52.037%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:03.724 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:03.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:30:03.779 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.779 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.779 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.779 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.779 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.780 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.780 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:30:03.780 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:30:04.362 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:30:04.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/417389965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:04.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3701980925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.405 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/417389965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:04.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3701980925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:04 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.412 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:04.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:04.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:04.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/417389965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:04.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3701980925' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:04 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:04.562 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:04.632 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:04.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:04.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:04.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:04.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:05.333 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:05.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:05.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:05.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:05.473 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:30:05.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 460 op/s; 35656/65400 objects degraded (54.520%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:30:05.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:05.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: failed to encode map e260 with expected crc 2023-06-10T21:30:05.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:05.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: osdmap e260: 8 total, 8 up, 6 in 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[186706]: failed to encode map e260 with expected crc 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 460 op/s; 35656/65400 objects degraded (54.520%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: failed to encode map e260 with expected crc 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:05.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: osdmap e260: 8 total, 8 up, 6 in 2023-06-10T21:30:05.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:05 smithi064 ceph-mon[180297]: failed to encode map e260 with expected crc 2023-06-10T21:30:05.559 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:05.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 460 op/s; 35656/65400 objects degraded (54.520%); 1.8 MiB/s, 28 objects/s recovering 2023-06-10T21:30:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: failed to encode map e260 with expected crc 2023-06-10T21:30:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: osdmap e260: 8 total, 8 up, 6 in 2023-06-10T21:30:05.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:05 smithi152 ceph-mon[156819]: failed to encode map e260 with expected crc 2023-06-10T21:30:06.234 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:06.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:06.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:06.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:06.468 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:06.573 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:06.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35656/65400 objects degraded (54.520%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:06.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35656/65400 objects degraded (54.520%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:06.681 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:06.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:06.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:06.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35656/65400 objects degraded (54.520%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:06.789 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:06.825 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:06.896 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:06.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:07.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:07.032 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:07.084 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:07.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:07.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:07.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:07.312 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:07.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:07 smithi064 ceph-mon[186706]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 35513/61764 objects degraded (57.498%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:30:07.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:07 smithi064 ceph-mon[180297]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 35513/61764 objects degraded (57.498%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:30:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:07 smithi152 ceph-mon[156819]: pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 35513/61764 objects degraded (57.498%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:30:07.736 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:07.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:07.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:07.880 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:07.939 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:07.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:08.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:30:08.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:08.073 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:08.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:08.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:08.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:08.327 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:08.377 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:30:09.139 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:09.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:09.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:09.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:09 smithi064 ceph-mon[180297]: pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 35513/61764 objects degraded (57.498%); 1.1 MiB/s, 16 objects/s recovering 2023-06-10T21:30:09.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:09 smithi064 ceph-mon[186706]: pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 35513/61764 objects degraded (57.498%); 1.1 MiB/s, 16 objects/s recovering 2023-06-10T21:30:09.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:09.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:09 smithi152 ceph-mon[156819]: pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 314 op/s; 35513/61764 objects degraded (57.498%); 1.1 MiB/s, 16 objects/s recovering 2023-06-10T21:30:09.780 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:09.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:09.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:09.963 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:09.987 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:10.153 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:10.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:10.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:10.379 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:30:10.380 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:30:10.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:10.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:11.315 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:11.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:11.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:11.449 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:30:11.534 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[180297]: pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 35355/54795 objects degraded (64.522%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3598045271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35355/54795 objects degraded (64.522%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[186706]: pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 35355/54795 objects degraded (64.522%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3598045271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35355/54795 objects degraded (64.522%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:11 smithi152 ceph-mon[156819]: pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 456 op/s; 35355/54795 objects degraded (64.522%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:30:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3598045271' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:30:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35355/54795 objects degraded (64.522%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[180297]: failed to encode map e261 with expected crc 2023-06-10T21:30:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:30:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[180297]: osdmap e261: 8 total, 8 up, 6 in 2023-06-10T21:30:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[180297]: failed to encode map e261 with expected crc 2023-06-10T21:30:12.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[186706]: failed to encode map e261 with expected crc 2023-06-10T21:30:12.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:30:12.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[186706]: osdmap e261: 8 total, 8 up, 6 in 2023-06-10T21:30:12.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:12 smithi064 ceph-mon[186706]: failed to encode map e261 with expected crc 2023-06-10T21:30:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:12 smithi152 ceph-mon[156819]: failed to encode map e261 with expected crc 2023-06-10T21:30:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:30:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:12 smithi152 ceph-mon[156819]: osdmap e261: 8 total, 8 up, 6 in 2023-06-10T21:30:12.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:12 smithi152 ceph-mon[156819]: failed to encode map e261 with expected crc 2023-06-10T21:30:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:12] "GET /metrics HTTP/1.1" 200 223738 "" "Prometheus/2.33.4" 2023-06-10T21:30:13.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:30:13.687 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:13.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:13 smithi152 ceph-mon[156819]: pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 35355/54795 objects degraded (64.522%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:30:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:13.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:13.696 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:13 smithi064 ceph-mon[180297]: pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 35355/54795 objects degraded (64.522%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:30:13.696 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:13 smithi064 ceph-mon[186706]: pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 441 op/s; 35355/54795 objects degraded (64.522%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:30:13.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:13.828 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:13.900 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:13.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:13.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:13.999 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:30:14.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:30:14.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:14.130 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:14.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:14.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:14.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:14.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:15.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:15.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:15.209 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:15.312 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:15.499 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:15.605 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:15.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:15.722 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:15.725 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:15.750 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:15 smithi152 ceph-mon[156819]: pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 35355/51816 objects degraded (68.232%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:30:15.752 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:15.799 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:15.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:15.831 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:30:15.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:15 smithi064 ceph-mon[180297]: pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 35355/51816 objects degraded (68.232%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:30:15.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:15 smithi064 ceph-mon[186706]: pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.1 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 395 op/s; 35355/51816 objects degraded (68.232%); 1.0 MiB/s, 16 objects/s recovering 2023-06-10T21:30:15.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:15.918 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:30:15.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:30:16.026 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:16.051 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:30:16.095 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:30:16.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:30:16.187 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:30:16.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:16.216 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:30:16.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:16.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:16.229 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.229 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.229 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.229 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.230 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.230 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.230 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.230 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:30:16.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:16.356 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:16.414 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:16.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:16.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:16.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:16.609 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:16.613 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/881879745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:16.614 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35355/51816 objects degraded (68.232%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:16.614 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:30:16.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:16.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:16.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:16.753 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:16.799 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:30:16.806 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:16.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:16.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/881879745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:16.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35355/51816 objects degraded (68.232%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:16.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:30:16.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/881879745' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:16.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35355/51816 objects degraded (68.232%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:16.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:30:16.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:16.943 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:16.996 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:17.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:17.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:17.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:17.131 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:17.176 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:17.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:17.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:17.309 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:17.350 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:17.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:17.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:17.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:17.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:17.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 35200/47508 objects degraded (74.093%); 2.0 MiB/s, 31 objects/s recovering 2023-06-10T21:30:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: failed to encode map e262 with expected crc 2023-06-10T21:30:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:30:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: osdmap e262: 8 total, 8 up, 6 in 2023-06-10T21:30:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:17 smithi152 ceph-mon[156819]: failed to encode map e262 with expected crc 2023-06-10T21:30:17.799 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 35200/47508 objects degraded (74.093%); 2.0 MiB/s, 31 objects/s recovering 2023-06-10T21:30:17.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:17.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: failed to encode map e262 with expected crc 2023-06-10T21:30:17.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:30:17.800 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: osdmap e262: 8 total, 8 up, 6 in 2023-06-10T21:30:17.801 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[186706]: failed to encode map e262 with expected crc 2023-06-10T21:30:17.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 35200/47508 objects degraded (74.093%); 2.0 MiB/s, 31 objects/s recovering 2023-06-10T21:30:17.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:17.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: failed to encode map e262 with expected crc 2023-06-10T21:30:17.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/267388682' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:30:17.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: osdmap e262: 8 total, 8 up, 6 in 2023-06-10T21:30:17.803 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:17 smithi064 ceph-mon[180297]: failed to encode map e262 with expected crc 2023-06-10T21:30:18.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:18.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:18.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:18.231 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:18.272 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:18.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:18.379 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:18.408 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:18.472 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:18.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:18.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:18.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:18.823 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:18.828 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:19.538 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:19.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:19.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:19.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:19.687 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:19.742 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:19.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:19.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:19.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:19 smithi064 ceph-mon[186706]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 35200/47508 objects degraded (74.093%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:30:19.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:19 smithi064 ceph-mon[180297]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 35200/47508 objects degraded (74.093%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:30:19.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:19.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:19.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:19 smithi152 ceph-mon[156819]: pgmap v930: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 989 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 303 op/s; 35200/47508 objects degraded (74.093%); 1.2 MiB/s, 19 objects/s recovering 2023-06-10T21:30:20.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:20.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:20.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:20.602 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:20.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:20.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:21.039 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:21.067 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:21.120 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:21.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:21.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:21.304 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:21.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:21.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:21.501 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:30:21.520 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:30:21.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:21 smithi064 ceph-mon[180297]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 35081/40404 objects degraded (86.826%); 2.1 MiB/s, 37 objects/s recovering 2023-06-10T21:30:21.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/40404 objects degraded (86.826%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:21.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:21 smithi064 ceph-mon[186706]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 35081/40404 objects degraded (86.826%); 2.1 MiB/s, 37 objects/s recovering 2023-06-10T21:30:21.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/40404 objects degraded (86.826%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:21.589 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:21.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:30:21.698 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:21.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:30:21.903 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:21.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:21.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:21.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:21 smithi152 ceph-mon[156819]: pgmap v931: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 575 op/s; 35081/40404 objects degraded (86.826%); 2.1 MiB/s, 37 objects/s recovering 2023-06-10T21:30:21.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/40404 objects degraded (86.826%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:21.959 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:30:21.960 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:30:21.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.960 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:169699 10.f 10:fb40e19d:::benchmark_data_smithi064_312713_object78943:head [delete] snapc 0=[] ondisk+write+known_if_redirected e262)", 2023-06-10T21:30:21.960 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:30:21.928570+0000", 2023-06-10T21:30:21.960 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011579905, 2023-06-10T21:30:21.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011612634, 2023-06-10T21:30:21.961 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:21.961 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:30:21.961 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:21.962 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:21.962 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:21.962 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169699 2023-06-10T21:30:21.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.963 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:21.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:21.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928570+0000", 2023-06-10T21:30:21.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:21.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:21.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928570+0000", 2023-06-10T21:30:21.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:21.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:21.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928569+0000", 2023-06-10T21:30:21.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:21.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928579+0000", 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8980000000000008e-06 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:21.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928593+0000", 2023-06-10T21:30:21.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4369000000000001e-05 2023-06-10T21:30:21.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:21.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928604+0000", 2023-06-10T21:30:21.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1029000000000001e-05 2023-06-10T21:30:21.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:21.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928646+0000", 2023-06-10T21:30:21.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1603000000000001e-05 2023-06-10T21:30:21.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:21.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928752+0000", 2023-06-10T21:30:21.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106383 2023-06-10T21:30:21.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,4", 2023-06-10T21:30:21.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928899+0000", 2023-06-10T21:30:21.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014736799999999999 2023-06-10T21:30:21.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:21.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.935579+0000", 2023-06-10T21:30:21.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066800710000000001 2023-06-10T21:30:21.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:21.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.938066+0000", 2023-06-10T21:30:21.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024868609999999999 2023-06-10T21:30:21.976 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:21.977 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:21.977 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:21.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.978 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:169700 10.5 10:affa321d:::benchmark_data_smithi064_312713_object78944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e262)", 2023-06-10T21:30:21.978 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:30:21.928620+0000", 2023-06-10T21:30:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011529947, 2023-06-10T21:30:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011660382, 2023-06-10T21:30:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:30:21.979 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:21.980 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:21.980 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:21.980 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 169700 2023-06-10T21:30:21.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.980 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928620+0000", 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:21.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928620+0000", 2023-06-10T21:30:21.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:21.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928620+0000", 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:21.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928623+0000", 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4979999999999998e-06 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928631+0000", 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9449999999999993e-06 2023-06-10T21:30:21.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.928637+0000", 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.156e-06 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:21.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.929031+0000", 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000393705 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.929190+0000", 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159304 2023-06-10T21:30:21.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:30:21.929350+0000", 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015959899999999999 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:21.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:21.988 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:30:21.988 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 2 2023-06-10T21:30:21.988 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:30:22.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:22.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:22.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:22 smithi064 ceph-mon[186706]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:22.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:22 smithi064 ceph-mon[180297]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:22.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:22 smithi152 ceph-mon[156819]: from='client.55960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:22] "GET /metrics HTTP/1.1" 200 223746 "" "Prometheus/2.33.4" 2023-06-10T21:30:23.362 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:23.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:23.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:23.512 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:23.560 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:23.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:23.604 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":931,"stamp":"2023-06-10T21:30:19.711706+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":881596326,"num_objects":13468,"num_object_clones":0,"num_object_copies":40404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13468,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":168325,"num_write_kb":5810639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20308,"num_bytes_recovered":1323376156,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126857,"ondisk_log_size":126857,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5135004,"kb_used_data":3047468,"kb_used_omap":30,"kb_used_meta":2087329,"kb_avail":557335908,"statfs":{"total":575970213888,"available":570711969792,"internally_reserved":0,"allocated":3120607232,"data_stored":3116090143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2137424934},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314376192,"num_objects":-4797,"num_object_clones":0,"num_object_copies":-14391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4797,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":313,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.329181"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":253,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.407635+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:30:11.407635+0000","last_peered":"2023-06-10T21:30:11.407635+0000","last_clean":"2023-06-10T21:30:11.407635+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:30:11.407635+0000","last_undegraded":"2023-06-10T21:30:11.407635+0000","last_fullsized":"2023-06-10T21:30:11.407635+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":193,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449998+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:30:05.449998+0000","last_peered":"2023-06-10T21:30:05.449998+0000","last_clean":"2023-06-10T21:30:05.449998+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:30:05.449998+0000","last_undegraded":"2023-06-10T21:30:05.449998+0000","last_fullsized":"2023-06-10T21:30:05.449998+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":168,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449376+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:30:05.449376+0000","last_peered":"2023-06-10T21:30:05.449376+0000","last_clean":"2023-06-10T21:30:05.449376+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:30:05.449376+0000","last_undegraded":"2023-06-10T21:30:05.449376+0000","last_fullsized":"2023-06-10T21:30:05.449376+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412422+0000","last_change":"2023-06-10T21:28:57.902734+0000","last_active":"2023-06-10T21:30:11.412422+0000","last_peered":"2023-06-10T21:30:11.412422+0000","last_clean":"2023-06-10T21:30:11.412422+0000","last_became_active":"2023-06-10T21:28:57.902279+0000","last_became_peered":"2023-06-10T21:28:57.902279+0000","last_unstale":"2023-06-10T21:30:11.412422+0000","last_undegraded":"2023-06-10T21:30:11.412422+0000","last_fullsized":"2023-06-10T21:30:11.412422+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:46.251698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":233,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449413+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:30:05.449413+0000","last_peered":"2023-06-10T21:30:05.449413+0000","last_clean":"2023-06-10T21:30:05.449413+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:30:05.449413+0000","last_undegraded":"2023-06-10T21:30:05.449413+0000","last_fullsized":"2023-06-10T21:30:05.449413+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":226,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410932+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:30:11.410932+0000","last_peered":"2023-06-10T21:30:11.410932+0000","last_clean":"2023-06-10T21:30:11.410932+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:30:11.410932+0000","last_undegraded":"2023-06-10T21:30:11.410932+0000","last_fullsized":"2023-06-10T21:30:11.410932+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410999+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:30:11.410999+0000","last_peered":"2023-06-10T21:30:11.410999+0000","last_clean":"2023-06-10T21:30:11.410999+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:30:11.410999+0000","last_undegraded":"2023-06-10T21:30:11.410999+0000","last_fullsized":"2023-06-10T21:30:11.410999+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":207,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449018+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:30:05.449018+0000","last_peered":"2023-06-10T21:30:05.449018+0000","last_clean":"2023-06-10T21:30:05.449018+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:30:05.449018+0000","last_undegraded":"2023-06-10T21:30:05.449018+0000","last_fullsized":"2023-06-10T21:30:05.449018+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394947+0000","last_change":"2023-06-10T21:28:57.899055+0000","last_active":"2023-06-10T21:30:11.394947+0000","last_peered":"2023-06-10T21:30:11.394947+0000","last_clean":"2023-06-10T21:30:11.394947+0000","last_became_active":"2023-06-10T21:28:57.898692+0000","last_became_peered":"2023-06-10T21:28:57.898692+0000","last_unstale":"2023-06-10T21:30:11.394947+0000","last_undegraded":"2023-06-10T21:30:11.394947+0000","last_fullsized":"2023-06-10T21:30:11.394947+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":186,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449642+0000","last_change":"2023-06-10T21:28:57.895548+0000","last_active":"2023-06-10T21:30:05.449642+0000","last_peered":"2023-06-10T21:30:05.449642+0000","last_clean":"2023-06-10T21:30:05.449642+0000","last_became_active":"2023-06-10T21:28:57.895217+0000","last_became_peered":"2023-06-10T21:28:57.895217+0000","last_unstale":"2023-06-10T21:30:05.449642+0000","last_undegraded":"2023-06-10T21:30:05.449642+0000","last_fullsized":"2023-06-10T21:30:05.449642+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:46:40.560609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":247,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394941+0000","last_change":"2023-06-10T21:28:57.899846+0000","last_active":"2023-06-10T21:30:11.394941+0000","last_peered":"2023-06-10T21:30:11.394941+0000","last_clean":"2023-06-10T21:30:11.394941+0000","last_became_active":"2023-06-10T21:28:57.898805+0000","last_became_peered":"2023-06-10T21:28:57.898805+0000","last_unstale":"2023-06-10T21:30:11.394941+0000","last_undegraded":"2023-06-10T21:30:11.394941+0000","last_fullsized":"2023-06-10T21:30:11.394941+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.407549+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:30:11.407549+0000","last_peered":"2023-06-10T21:30:11.407549+0000","last_clean":"2023-06-10T21:30:11.407549+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:30:11.407549+0000","last_undegraded":"2023-06-10T21:30:11.407549+0000","last_fullsized":"2023-06-10T21:30:11.407549+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":132,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394661+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:30:11.394661+0000","last_peered":"2023-06-10T21:30:11.394661+0000","last_clean":"2023-06-10T21:30:11.394661+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:30:11.394661+0000","last_undegraded":"2023-06-10T21:30:11.394661+0000","last_fullsized":"2023-06-10T21:30:11.394661+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405429+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:30:11.405429+0000","last_peered":"2023-06-10T21:30:11.405429+0000","last_clean":"2023-06-10T21:30:11.405429+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:30:11.405429+0000","last_undegraded":"2023-06-10T21:30:11.405429+0000","last_fullsized":"2023-06-10T21:30:11.405429+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":261,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394303+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:30:11.394303+0000","last_peered":"2023-06-10T21:30:11.394303+0000","last_clean":"2023-06-10T21:30:11.394303+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:30:11.394303+0000","last_undegraded":"2023-06-10T21:30:11.394303+0000","last_fullsized":"2023-06-10T21:30:11.394303+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":193,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449841+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:30:05.449841+0000","last_peered":"2023-06-10T21:30:05.449841+0000","last_clean":"2023-06-10T21:30:05.449841+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:30:05.449841+0000","last_undegraded":"2023-06-10T21:30:05.449841+0000","last_fullsized":"2023-06-10T21:30:05.449841+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.414440+0000","last_change":"2023-06-10T21:28:57.904420+0000","last_active":"2023-06-10T21:30:11.414440+0000","last_peered":"2023-06-10T21:30:11.414440+0000","last_clean":"2023-06-10T21:30:11.414440+0000","last_became_active":"2023-06-10T21:28:57.904027+0000","last_became_peered":"2023-06-10T21:28:57.904027+0000","last_unstale":"2023-06-10T21:30:11.414440+0000","last_undegraded":"2023-06-10T21:30:11.414440+0000","last_fullsized":"2023-06-10T21:30:11.414440+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:15:36.667597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":190,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.395072+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:30:11.395072+0000","last_peered":"2023-06-10T21:30:11.395072+0000","last_clean":"2023-06-10T21:30:11.395072+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:30:11.395072+0000","last_undegraded":"2023-06-10T21:30:11.395072+0000","last_fullsized":"2023-06-10T21:30:11.395072+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":238,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.396813+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:30:11.396813+0000","last_peered":"2023-06-10T21:30:11.396813+0000","last_clean":"2023-06-10T21:30:11.396813+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:30:11.396813+0000","last_undegraded":"2023-06-10T21:30:11.396813+0000","last_fullsized":"2023-06-10T21:30:11.396813+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412265+0000","last_change":"2023-06-10T21:28:57.902565+0000","last_active":"2023-06-10T21:30:11.412265+0000","last_peered":"2023-06-10T21:30:11.412265+0000","last_clean":"2023-06-10T21:30:11.412265+0000","last_became_active":"2023-06-10T21:28:57.901902+0000","last_became_peered":"2023-06-10T21:28:57.901902+0000","last_unstale":"2023-06-10T21:30:11.412265+0000","last_undegraded":"2023-06-10T21:30:11.412265+0000","last_fullsized":"2023-06-10T21:30:11.412265+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:54.534642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":235,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.409075+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:30:11.409075+0000","last_peered":"2023-06-10T21:30:11.409075+0000","last_clean":"2023-06-10T21:30:11.409075+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:30:11.409075+0000","last_undegraded":"2023-06-10T21:30:11.409075+0000","last_fullsized":"2023-06-10T21:30:11.409075+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":193,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449860+0000","last_change":"2023-06-10T21:28:57.889771+0000","last_active":"2023-06-10T21:30:05.449860+0000","last_peered":"2023-06-10T21:30:05.449860+0000","last_clean":"2023-06-10T21:30:05.449860+0000","last_became_active":"2023-06-10T21:28:57.887036+0000","last_became_peered":"2023-06-10T21:28:57.887036+0000","last_unstale":"2023-06-10T21:30:05.449860+0000","last_undegraded":"2023-06-10T21:30:05.449860+0000","last_fullsized":"2023-06-10T21:30:05.449860+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:05.952139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"262'10523","reported_seq":12270,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.704965+0000","last_change":"2023-06-10T21:28:34.624898+0000","last_active":"2023-06-10T21:30:17.704965+0000","last_peered":"2023-06-10T21:30:17.704965+0000","last_clean":"2023-06-10T21:30:17.704965+0000","last_became_active":"2023-06-10T21:28:34.501130+0000","last_became_peered":"2023-06-10T21:28:34.501130+0000","last_unstale":"2023-06-10T21:30:17.704965+0000","last_undegraded":"2023-06-10T21:30:17.704965+0000","last_fullsized":"2023-06-10T21:30:17.704965+0000","mapping_epoch":244,"log_start":"223'4400","ondisk_log_start":"223'4400","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6123,"log_dups_size":0,"ondisk_log_size":6123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:11:04.611578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54722560,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":363456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":832,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":271,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.395234+0000","last_change":"2023-06-10T21:28:34.522514+0000","last_active":"2023-06-10T21:30:11.395234+0000","last_peered":"2023-06-10T21:30:11.395234+0000","last_clean":"2023-06-10T21:30:11.395234+0000","last_became_active":"2023-06-10T21:28:34.522038+0000","last_became_peered":"2023-06-10T21:28:34.522038+0000","last_unstale":"2023-06-10T21:30:11.395234+0000","last_undegraded":"2023-06-10T21:30:11.395234+0000","last_fullsized":"2023-06-10T21:30:11.395234+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394535+0000","last_change":"2023-06-10T21:28:57.899217+0000","last_active":"2023-06-10T21:30:11.394535+0000","last_peered":"2023-06-10T21:30:11.394535+0000","last_clean":"2023-06-10T21:30:11.394535+0000","last_became_active":"2023-06-10T21:28:57.898541+0000","last_became_peered":"2023-06-10T21:28:57.898541+0000","last_unstale":"2023-06-10T21:30:11.394535+0000","last_undegraded":"2023-06-10T21:30:11.394535+0000","last_fullsized":"2023-06-10T21:30:11.394535+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"262'10567","reported_seq":19084,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:19.111878+0000","last_change":"2023-06-10T21:28:59.502716+0000","last_active":"2023-06-10T21:30:19.111878+0000","last_peered":"2023-06-10T21:30:19.111878+0000","last_clean":"2023-06-10T21:28:56.868806+0000","last_became_active":"2023-06-10T21:28:59.444403+0000","last_became_peered":"2023-06-10T21:28:59.444403+0000","last_unstale":"2023-06-10T21:30:19.111878+0000","last_undegraded":"2023-06-10T21:28:59.066225+0000","last_fullsized":"2023-06-10T21:28:58.880029+0000","mapping_epoch":251,"log_start":"200'500","ondisk_log_start":"200'500","created":194,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10067,"log_dups_size":0,"ondisk_log_size":10067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:26.444508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10567,"num_write_kb":365440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4198,"num_bytes_recovered":275054592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1461},{"shards":"2,4,6","objects":853}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":231,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.395044+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:30:11.395044+0000","last_peered":"2023-06-10T21:30:11.395044+0000","last_clean":"2023-06-10T21:30:11.395044+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:30:11.395044+0000","last_undegraded":"2023-06-10T21:30:11.395044+0000","last_fullsized":"2023-06-10T21:30:11.395044+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449235+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:30:05.449235+0000","last_peered":"2023-06-10T21:30:05.449235+0000","last_clean":"2023-06-10T21:30:05.449235+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:30:05.449235+0000","last_undegraded":"2023-06-10T21:30:05.449235+0000","last_fullsized":"2023-06-10T21:30:05.449235+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"262'10565","reported_seq":10650,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:17.700405+0000","last_change":"2023-06-10T21:28:59.404926+0000","last_active":"2023-06-10T21:30:17.700405+0000","last_peered":"2023-06-10T21:30:17.700405+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:28:59.373005+0000","last_became_peered":"2023-06-10T21:28:59.373005+0000","last_unstale":"2023-06-10T21:30:17.700405+0000","last_undegraded":"2023-06-10T21:28:58.922286+0000","last_fullsized":"2023-06-10T21:28:58.878960+0000","mapping_epoch":251,"log_start":"222'4400","ondisk_log_start":"222'4400","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6165,"log_dups_size":0,"ondisk_log_size":6165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:21.872755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56164352,"num_objects":857,"num_object_clones":0,"num_object_copies":2571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10565,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":-1362}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":256,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394844+0000","last_change":"2023-06-10T21:28:57.900006+0000","last_active":"2023-06-10T21:30:11.394844+0000","last_peered":"2023-06-10T21:30:11.394844+0000","last_clean":"2023-06-10T21:30:11.394844+0000","last_became_active":"2023-06-10T21:28:57.898709+0000","last_became_peered":"2023-06-10T21:28:57.898709+0000","last_unstale":"2023-06-10T21:30:11.394844+0000","last_undegraded":"2023-06-10T21:30:11.394844+0000","last_fullsized":"2023-06-10T21:30:11.394844+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":138,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394742+0000","last_change":"2023-06-10T21:28:34.551591+0000","last_active":"2023-06-10T21:30:11.394742+0000","last_peered":"2023-06-10T21:30:11.394742+0000","last_clean":"2023-06-10T21:30:11.394742+0000","last_became_active":"2023-06-10T21:28:34.551254+0000","last_became_peered":"2023-06-10T21:28:34.551254+0000","last_unstale":"2023-06-10T21:30:11.394742+0000","last_undegraded":"2023-06-10T21:30:11.394742+0000","last_fullsized":"2023-06-10T21:30:11.394742+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"262'10404","reported_seq":10508,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:19.072848+0000","last_change":"2023-06-10T21:28:59.445842+0000","last_active":"2023-06-10T21:30:19.072848+0000","last_peered":"2023-06-10T21:30:19.072848+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:28:59.419763+0000","last_became_peered":"2023-06-10T21:28:59.419763+0000","last_unstale":"2023-06-10T21:30:19.072848+0000","last_undegraded":"2023-06-10T21:28:58.932194+0000","last_fullsized":"2023-06-10T21:28:58.879691+0000","mapping_epoch":251,"log_start":"200'400","ondisk_log_start":"200'400","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10004,"log_dups_size":0,"ondisk_log_size":10004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:46.862125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56229888,"num_objects":858,"num_object_clones":0,"num_object_copies":2574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4698,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10404,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-1430}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449987+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:30:05.449987+0000","last_peered":"2023-06-10T21:30:05.449987+0000","last_clean":"2023-06-10T21:30:05.449987+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:30:05.449987+0000","last_undegraded":"2023-06-10T21:30:05.449987+0000","last_fullsized":"2023-06-10T21:30:05.449987+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411430+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:30:11.411430+0000","last_peered":"2023-06-10T21:30:11.411430+0000","last_clean":"2023-06-10T21:30:11.411430+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:30:11.411430+0000","last_undegraded":"2023-06-10T21:30:11.411430+0000","last_fullsized":"2023-06-10T21:30:11.411430+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"262'10636","reported_seq":10727,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.782908+0000","last_change":"2023-06-10T21:28:34.940046+0000","last_active":"2023-06-10T21:30:17.782908+0000","last_peered":"2023-06-10T21:30:17.782908+0000","last_clean":"2023-06-10T21:30:17.782908+0000","last_became_active":"2023-06-10T21:28:34.501595+0000","last_became_peered":"2023-06-10T21:28:34.501595+0000","last_unstale":"2023-06-10T21:30:17.782908+0000","last_undegraded":"2023-06-10T21:30:17.782908+0000","last_fullsized":"2023-06-10T21:30:17.782908+0000","mapping_epoch":244,"log_start":"223'4502","ondisk_log_start":"223'4502","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6134,"log_dups_size":0,"ondisk_log_size":6134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:59:45.255711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10636,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":210,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450603+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:30:05.450603+0000","last_peered":"2023-06-10T21:30:05.450603+0000","last_clean":"2023-06-10T21:30:05.450603+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:30:05.450603+0000","last_undegraded":"2023-06-10T21:30:05.450603+0000","last_fullsized":"2023-06-10T21:30:05.450603+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450067+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:30:05.450067+0000","last_peered":"2023-06-10T21:30:05.450067+0000","last_clean":"2023-06-10T21:30:05.450067+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:30:05.450067+0000","last_undegraded":"2023-06-10T21:30:05.450067+0000","last_fullsized":"2023-06-10T21:30:05.450067+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"262'10231","reported_seq":16573,"reported_epoch":262,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-10T21:30:19.138538+0000","last_change":"2023-06-10T21:30:18.062068+0000","last_active":"2023-06-10T21:30:19.138538+0000","last_peered":"2023-06-10T21:30:19.138538+0000","last_clean":"2023-06-10T21:26:26.761025+0000","last_became_active":"2023-06-10T21:28:59.376175+0000","last_became_peered":"2023-06-10T21:28:59.376175+0000","last_unstale":"2023-06-10T21:30:19.138538+0000","last_undegraded":"2023-06-10T21:30:19.138538+0000","last_fullsized":"2023-06-10T21:28:58.879642+0000","mapping_epoch":251,"log_start":"200'200","ondisk_log_start":"200'200","created":194,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10031,"log_dups_size":0,"ondisk_log_size":10031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:28.355418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54067200,"num_objects":825,"num_object_clones":0,"num_object_copies":2475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10231,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3115,"num_bytes_recovered":201523200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":825}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":33,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450619+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:30:05.450619+0000","last_peered":"2023-06-10T21:30:05.450619+0000","last_clean":"2023-06-10T21:30:05.450619+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:30:05.450619+0000","last_undegraded":"2023-06-10T21:30:05.450619+0000","last_fullsized":"2023-06-10T21:30:05.450619+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":251,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405130+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:30:11.405130+0000","last_peered":"2023-06-10T21:30:11.405130+0000","last_clean":"2023-06-10T21:30:11.405130+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:30:11.405130+0000","last_undegraded":"2023-06-10T21:30:11.405130+0000","last_fullsized":"2023-06-10T21:30:11.405130+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"262'10641","reported_seq":16065,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:19.492539+0000","last_change":"2023-06-10T21:27:45.367487+0000","last_active":"2023-06-10T21:30:19.492539+0000","last_peered":"2023-06-10T21:30:19.492539+0000","last_clean":"2023-06-10T21:30:19.492539+0000","last_became_active":"2023-06-10T21:27:45.353438+0000","last_became_peered":"2023-06-10T21:27:45.353438+0000","last_unstale":"2023-06-10T21:30:19.492539+0000","last_undegraded":"2023-06-10T21:30:19.492539+0000","last_fullsized":"2023-06-10T21:30:19.492539+0000","mapping_epoch":234,"log_start":"212'2900","ondisk_log_start":"212'2900","created":194,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7741,"log_dups_size":0,"ondisk_log_size":7741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10641,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3133,"num_bytes_recovered":205324288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":237,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405033+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:30:11.405033+0000","last_peered":"2023-06-10T21:30:11.405033+0000","last_clean":"2023-06-10T21:30:11.405033+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:30:11.405033+0000","last_undegraded":"2023-06-10T21:30:11.405033+0000","last_fullsized":"2023-06-10T21:30:11.405033+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":33,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450136+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:30:05.450136+0000","last_peered":"2023-06-10T21:30:05.450136+0000","last_clean":"2023-06-10T21:30:05.450136+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:30:05.450136+0000","last_undegraded":"2023-06-10T21:30:05.450136+0000","last_fullsized":"2023-06-10T21:30:05.450136+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.1","version":"262'10540","reported_seq":10650,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:19.393983+0000","last_change":"2023-06-10T21:28:34.572666+0000","last_active":"2023-06-10T21:30:19.393983+0000","last_peered":"2023-06-10T21:30:19.393983+0000","last_clean":"2023-06-10T21:30:19.393983+0000","last_became_active":"2023-06-10T21:28:34.522012+0000","last_became_peered":"2023-06-10T21:28:34.522012+0000","last_unstale":"2023-06-10T21:30:19.393983+0000","last_undegraded":"2023-06-10T21:30:19.393983+0000","last_fullsized":"2023-06-10T21:30:19.393983+0000","mapping_epoch":244,"log_start":"212'2800","ondisk_log_start":"212'2800","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7740,"log_dups_size":0,"ondisk_log_size":7740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10540,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":195,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449291+0000","last_change":"2023-06-10T21:28:57.894408+0000","last_active":"2023-06-10T21:30:05.449291+0000","last_peered":"2023-06-10T21:30:05.449291+0000","last_clean":"2023-06-10T21:30:05.449291+0000","last_became_active":"2023-06-10T21:28:57.893595+0000","last_became_peered":"2023-06-10T21:28:57.893595+0000","last_unstale":"2023-06-10T21:30:05.449291+0000","last_undegraded":"2023-06-10T21:30:05.449291+0000","last_fullsized":"2023-06-10T21:30:05.449291+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:03.720304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450221+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:30:05.450221+0000","last_peered":"2023-06-10T21:30:05.450221+0000","last_clean":"2023-06-10T21:30:05.450221+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:30:05.450221+0000","last_undegraded":"2023-06-10T21:30:05.450221+0000","last_fullsized":"2023-06-10T21:30:05.450221+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"262'10627","reported_seq":10735,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:19.597949+0000","last_change":"2023-06-10T21:28:34.926507+0000","last_active":"2023-06-10T21:30:19.597949+0000","last_peered":"2023-06-10T21:30:19.597949+0000","last_clean":"2023-06-10T21:30:19.597949+0000","last_became_active":"2023-06-10T21:28:34.541984+0000","last_became_peered":"2023-06-10T21:28:34.541984+0000","last_unstale":"2023-06-10T21:30:19.597949+0000","last_undegraded":"2023-06-10T21:30:19.597949+0000","last_fullsized":"2023-06-10T21:30:19.597949+0000","mapping_epoch":244,"log_start":"212'2900","ondisk_log_start":"212'2900","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7727,"log_dups_size":0,"ondisk_log_size":7727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52101120,"num_objects":795,"num_object_clones":0,"num_object_copies":2385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10627,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411675+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:30:11.411675+0000","last_peered":"2023-06-10T21:30:11.411675+0000","last_clean":"2023-06-10T21:30:11.411675+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:30:11.411675+0000","last_undegraded":"2023-06-10T21:30:11.411675+0000","last_fullsized":"2023-06-10T21:30:11.411675+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450174+0000","last_change":"2023-06-10T21:28:57.946656+0000","last_active":"2023-06-10T21:30:05.450174+0000","last_peered":"2023-06-10T21:30:05.450174+0000","last_clean":"2023-06-10T21:30:05.450174+0000","last_became_active":"2023-06-10T21:28:57.946186+0000","last_became_peered":"2023-06-10T21:28:57.946186+0000","last_unstale":"2023-06-10T21:30:05.450174+0000","last_undegraded":"2023-06-10T21:30:05.450174+0000","last_fullsized":"2023-06-10T21:30:05.450174+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:31.259349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":103,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411784+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:30:11.411784+0000","last_peered":"2023-06-10T21:30:11.411784+0000","last_clean":"2023-06-10T21:30:11.411784+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:30:11.411784+0000","last_undegraded":"2023-06-10T21:30:11.411784+0000","last_fullsized":"2023-06-10T21:30:11.411784+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"262'10494","reported_seq":12819,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:17.688659+0000","last_change":"2023-06-10T21:28:59.454017+0000","last_active":"2023-06-10T21:30:17.688659+0000","last_peered":"2023-06-10T21:30:17.688659+0000","last_clean":"2023-06-10T21:28:56.879563+0000","last_became_active":"2023-06-10T21:28:59.417345+0000","last_became_peered":"2023-06-10T21:28:59.417345+0000","last_unstale":"2023-06-10T21:30:17.688659+0000","last_undegraded":"2023-06-10T21:28:58.916372+0000","last_fullsized":"2023-06-10T21:28:58.878970+0000","mapping_epoch":251,"log_start":"222'4300","ondisk_log_start":"222'4300","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:23.819643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55574528,"num_objects":848,"num_object_clones":0,"num_object_copies":2544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5415,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10494,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1110,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2","objects":-2072},{"shards":"0,2,6","objects":848}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":129,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411858+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:30:11.411858+0000","last_peered":"2023-06-10T21:30:11.411858+0000","last_clean":"2023-06-10T21:30:11.411858+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:30:11.411858+0000","last_undegraded":"2023-06-10T21:30:11.411858+0000","last_fullsized":"2023-06-10T21:30:11.411858+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":203,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411172+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:30:11.411172+0000","last_peered":"2023-06-10T21:30:11.411172+0000","last_clean":"2023-06-10T21:30:11.411172+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:30:11.411172+0000","last_undegraded":"2023-06-10T21:30:11.411172+0000","last_fullsized":"2023-06-10T21:30:11.411172+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405480+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:30:11.405480+0000","last_peered":"2023-06-10T21:30:11.405480+0000","last_clean":"2023-06-10T21:30:11.405480+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:30:11.405480+0000","last_undegraded":"2023-06-10T21:30:11.405480+0000","last_fullsized":"2023-06-10T21:30:11.405480+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"262'10362","reported_seq":12148,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:19.130797+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:30:19.130797+0000","last_peered":"2023-06-10T21:30:19.130797+0000","last_clean":"2023-06-10T21:30:19.130797+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:30:19.130797+0000","last_undegraded":"2023-06-10T21:30:19.130797+0000","last_fullsized":"2023-06-10T21:30:19.130797+0000","mapping_epoch":210,"log_start":"200'300","ondisk_log_start":"200'300","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56885248,"num_objects":868,"num_object_clones":0,"num_object_copies":2604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10362,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":104,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405052+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:30:11.405052+0000","last_peered":"2023-06-10T21:30:11.405052+0000","last_clean":"2023-06-10T21:30:11.405052+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:30:11.405052+0000","last_undegraded":"2023-06-10T21:30:11.405052+0000","last_fullsized":"2023-06-10T21:30:11.405052+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.404774+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:30:11.404774+0000","last_peered":"2023-06-10T21:30:11.404774+0000","last_clean":"2023-06-10T21:30:11.404774+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:30:11.404774+0000","last_undegraded":"2023-06-10T21:30:11.404774+0000","last_fullsized":"2023-06-10T21:30:11.404774+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":134,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410868+0000","last_change":"2023-06-10T21:28:34.518545+0000","last_active":"2023-06-10T21:30:11.410868+0000","last_peered":"2023-06-10T21:30:11.410868+0000","last_clean":"2023-06-10T21:30:11.410868+0000","last_became_active":"2023-06-10T21:28:34.517742+0000","last_became_peered":"2023-06-10T21:28:34.517742+0000","last_unstale":"2023-06-10T21:30:11.410868+0000","last_undegraded":"2023-06-10T21:30:11.410868+0000","last_fullsized":"2023-06-10T21:30:11.410868+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:13.761246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"262'10478","reported_seq":16877,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.751586+0000","last_change":"2023-06-10T21:28:34.988147+0000","last_active":"2023-06-10T21:30:17.751586+0000","last_peered":"2023-06-10T21:30:17.751586+0000","last_clean":"2023-06-10T21:30:17.751586+0000","last_became_active":"2023-06-10T21:28:34.506722+0000","last_became_peered":"2023-06-10T21:28:34.506722+0000","last_unstale":"2023-06-10T21:30:17.751586+0000","last_undegraded":"2023-06-10T21:30:17.751586+0000","last_fullsized":"2023-06-10T21:30:17.751586+0000","mapping_epoch":244,"log_start":"222'4300","ondisk_log_start":"222'4300","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6178,"log_dups_size":0,"ondisk_log_size":6178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:56:33.672188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55443456,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10478,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3157,"num_bytes_recovered":206438400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":266,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405333+0000","last_change":"2023-06-10T21:28:57.907800+0000","last_active":"2023-06-10T21:30:11.405333+0000","last_peered":"2023-06-10T21:30:11.405333+0000","last_clean":"2023-06-10T21:30:11.405333+0000","last_became_active":"2023-06-10T21:28:57.907509+0000","last_became_peered":"2023-06-10T21:28:57.907509+0000","last_unstale":"2023-06-10T21:30:11.405333+0000","last_undegraded":"2023-06-10T21:30:11.405333+0000","last_fullsized":"2023-06-10T21:30:11.405333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:56.001988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":549,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394459+0000","last_change":"2023-06-10T21:28:34.503678+0000","last_active":"2023-06-10T21:30:11.394459+0000","last_peered":"2023-06-10T21:30:11.394459+0000","last_clean":"2023-06-10T21:30:11.394459+0000","last_became_active":"2023-06-10T21:28:34.503254+0000","last_became_peered":"2023-06-10T21:28:34.503254+0000","last_unstale":"2023-06-10T21:30:11.394459+0000","last_undegraded":"2023-06-10T21:30:11.394459+0000","last_fullsized":"2023-06-10T21:30:11.394459+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":196,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450347+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:30:05.450347+0000","last_peered":"2023-06-10T21:30:05.450347+0000","last_clean":"2023-06-10T21:30:05.450347+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:30:05.450347+0000","last_undegraded":"2023-06-10T21:30:05.450347+0000","last_fullsized":"2023-06-10T21:30:05.450347+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":76,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394998+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:30:11.394998+0000","last_peered":"2023-06-10T21:30:11.394998+0000","last_clean":"2023-06-10T21:30:11.394998+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:30:11.394998+0000","last_undegraded":"2023-06-10T21:30:11.394998+0000","last_fullsized":"2023-06-10T21:30:11.394998+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"262'10661","reported_seq":10756,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.746615+0000","last_change":"2023-06-10T21:28:34.951633+0000","last_active":"2023-06-10T21:30:17.746615+0000","last_peered":"2023-06-10T21:30:17.746615+0000","last_clean":"2023-06-10T21:30:17.746615+0000","last_became_active":"2023-06-10T21:28:34.509441+0000","last_became_peered":"2023-06-10T21:28:34.509441+0000","last_unstale":"2023-06-10T21:30:17.746615+0000","last_undegraded":"2023-06-10T21:30:17.746615+0000","last_fullsized":"2023-06-10T21:30:17.746615+0000","mapping_epoch":244,"log_start":"223'4500","ondisk_log_start":"223'4500","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:34:59.163129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55246848,"num_objects":843,"num_object_clones":0,"num_object_copies":2529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10661,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":281,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394377+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:30:11.394377+0000","last_peered":"2023-06-10T21:30:11.394377+0000","last_clean":"2023-06-10T21:30:11.394377+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:30:11.394377+0000","last_undegraded":"2023-06-10T21:30:11.394377+0000","last_fullsized":"2023-06-10T21:30:11.394377+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410031+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:30:11.410031+0000","last_peered":"2023-06-10T21:30:11.410031+0000","last_clean":"2023-06-10T21:30:11.410031+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:30:11.410031+0000","last_undegraded":"2023-06-10T21:30:11.410031+0000","last_fullsized":"2023-06-10T21:30:11.410031+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412193+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:30:11.412193+0000","last_peered":"2023-06-10T21:30:11.412193+0000","last_clean":"2023-06-10T21:30:11.412193+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:30:11.412193+0000","last_undegraded":"2023-06-10T21:30:11.412193+0000","last_fullsized":"2023-06-10T21:30:11.412193+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"262'10609","reported_seq":11216,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:17.784284+0000","last_change":"2023-06-10T21:28:59.491171+0000","last_active":"2023-06-10T21:30:17.784284+0000","last_peered":"2023-06-10T21:30:17.784284+0000","last_clean":"2023-06-10T21:28:56.884482+0000","last_became_active":"2023-06-10T21:28:59.418669+0000","last_became_peered":"2023-06-10T21:28:59.418669+0000","last_unstale":"2023-06-10T21:30:17.784284+0000","last_undegraded":"2023-06-10T21:28:58.914273+0000","last_fullsized":"2023-06-10T21:28:58.879027+0000","mapping_epoch":251,"log_start":"222'4400","ondisk_log_start":"222'4400","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:47:23.775373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54919192,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10609,"num_write_kb":366273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":255,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-2173},{"shards":"0,4,6","objects":839}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":266,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394895+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:30:11.394895+0000","last_peered":"2023-06-10T21:30:11.394895+0000","last_clean":"2023-06-10T21:30:11.394895+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:30:11.394895+0000","last_undegraded":"2023-06-10T21:30:11.394895+0000","last_fullsized":"2023-06-10T21:30:11.394895+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405550+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:30:11.405550+0000","last_peered":"2023-06-10T21:30:11.405550+0000","last_clean":"2023-06-10T21:30:11.405550+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:30:11.405550+0000","last_undegraded":"2023-06-10T21:30:11.405550+0000","last_fullsized":"2023-06-10T21:30:11.405550+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":79,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411231+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:30:11.411231+0000","last_peered":"2023-06-10T21:30:11.411231+0000","last_clean":"2023-06-10T21:30:11.411231+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:30:11.411231+0000","last_undegraded":"2023-06-10T21:30:11.411231+0000","last_fullsized":"2023-06-10T21:30:11.411231+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"262'10446","reported_seq":11103,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:19.134617+0000","last_change":"2023-06-10T21:28:59.476469+0000","last_active":"2023-06-10T21:30:19.134617+0000","last_peered":"2023-06-10T21:30:19.134617+0000","last_clean":"2023-06-10T21:28:56.852537+0000","last_became_active":"2023-06-10T21:28:59.439769+0000","last_became_peered":"2023-06-10T21:28:59.439769+0000","last_unstale":"2023-06-10T21:30:19.134617+0000","last_undegraded":"2023-06-10T21:28:58.939041+0000","last_fullsized":"2023-06-10T21:28:58.883972+0000","mapping_epoch":251,"log_start":"200'400","ondisk_log_start":"200'400","created":194,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:10:03.064078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55705600,"num_objects":850,"num_object_clones":0,"num_object_copies":2550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10446,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":267,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2092},{"shards":"2,4,6","objects":850}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":210,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449957+0000","last_change":"2023-06-10T21:28:57.899002+0000","last_active":"2023-06-10T21:30:05.449957+0000","last_peered":"2023-06-10T21:30:05.449957+0000","last_clean":"2023-06-10T21:30:05.449957+0000","last_became_active":"2023-06-10T21:28:57.898639+0000","last_became_peered":"2023-06-10T21:28:57.898639+0000","last_unstale":"2023-06-10T21:30:05.449957+0000","last_undegraded":"2023-06-10T21:30:05.449957+0000","last_fullsized":"2023-06-10T21:30:05.449957+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:12:01.141117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":33,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449916+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:30:05.449916+0000","last_peered":"2023-06-10T21:30:05.449916+0000","last_clean":"2023-06-10T21:30:05.449916+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:30:05.449916+0000","last_undegraded":"2023-06-10T21:30:05.449916+0000","last_fullsized":"2023-06-10T21:30:05.449916+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":199,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449232+0000","last_change":"2023-06-10T21:28:57.895650+0000","last_active":"2023-06-10T21:30:05.449232+0000","last_peered":"2023-06-10T21:30:05.449232+0000","last_clean":"2023-06-10T21:30:05.449232+0000","last_became_active":"2023-06-10T21:28:57.895320+0000","last_became_peered":"2023-06-10T21:28:57.895320+0000","last_unstale":"2023-06-10T21:30:05.449232+0000","last_undegraded":"2023-06-10T21:30:05.449232+0000","last_fullsized":"2023-06-10T21:30:05.449232+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:35:20.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"262'10280","reported_seq":17069,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:19.124637+0000","last_change":"2023-06-10T21:28:59.500885+0000","last_active":"2023-06-10T21:30:19.124637+0000","last_peered":"2023-06-10T21:30:19.124637+0000","last_clean":"2023-06-10T21:28:56.851941+0000","last_became_active":"2023-06-10T21:28:59.444754+0000","last_became_peered":"2023-06-10T21:28:59.444754+0000","last_unstale":"2023-06-10T21:30:19.124637+0000","last_undegraded":"2023-06-10T21:28:59.081148+0000","last_fullsized":"2023-06-10T21:28:58.879700+0000","mapping_epoch":251,"log_start":"200'200","ondisk_log_start":"200'200","created":194,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:59.029499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53346304,"num_objects":814,"num_object_clones":0,"num_object_copies":2442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4601,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10280,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3339,"num_bytes_recovered":218628096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1397},{"shards":"2,4,6","objects":814}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":252,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.395166+0000","last_change":"2023-06-10T21:28:57.899350+0000","last_active":"2023-06-10T21:30:11.395166+0000","last_peered":"2023-06-10T21:30:11.395166+0000","last_clean":"2023-06-10T21:30:11.395166+0000","last_became_active":"2023-06-10T21:28:57.898711+0000","last_became_peered":"2023-06-10T21:28:57.898711+0000","last_unstale":"2023-06-10T21:30:11.395166+0000","last_undegraded":"2023-06-10T21:30:11.395166+0000","last_fullsized":"2023-06-10T21:30:11.395166+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":87,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412346+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:30:11.412346+0000","last_peered":"2023-06-10T21:30:11.412346+0000","last_clean":"2023-06-10T21:30:11.412346+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:30:11.412346+0000","last_undegraded":"2023-06-10T21:30:11.412346+0000","last_fullsized":"2023-06-10T21:30:11.412346+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":197,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.409878+0000","last_change":"2023-06-10T21:28:34.521765+0000","last_active":"2023-06-10T21:30:11.409878+0000","last_peered":"2023-06-10T21:30:11.409878+0000","last_clean":"2023-06-10T21:30:11.409878+0000","last_became_active":"2023-06-10T21:28:34.521409+0000","last_became_peered":"2023-06-10T21:28:34.521409+0000","last_unstale":"2023-06-10T21:30:11.409878+0000","last_undegraded":"2023-06-10T21:30:11.409878+0000","last_fullsized":"2023-06-10T21:30:11.409878+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:05:44.771841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411105+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:30:11.411105+0000","last_peered":"2023-06-10T21:30:11.411105+0000","last_clean":"2023-06-10T21:30:11.411105+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:30:11.411105+0000","last_undegraded":"2023-06-10T21:30:11.411105+0000","last_fullsized":"2023-06-10T21:30:11.411105+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405009+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:30:11.405009+0000","last_peered":"2023-06-10T21:30:11.405009+0000","last_clean":"2023-06-10T21:30:11.405009+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:30:11.405009+0000","last_undegraded":"2023-06-10T21:30:11.405009+0000","last_fullsized":"2023-06-10T21:30:11.405009+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":62,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.394403+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:30:11.394403+0000","last_peered":"2023-06-10T21:30:11.394403+0000","last_clean":"2023-06-10T21:30:11.394403+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:30:11.394403+0000","last_undegraded":"2023-06-10T21:30:11.394403+0000","last_fullsized":"2023-06-10T21:30:11.394403+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450517+0000","last_change":"2023-06-10T21:28:57.898684+0000","last_active":"2023-06-10T21:30:05.450517+0000","last_peered":"2023-06-10T21:30:05.450517+0000","last_clean":"2023-06-10T21:30:05.450517+0000","last_became_active":"2023-06-10T21:28:57.898387+0000","last_became_peered":"2023-06-10T21:28:57.898387+0000","last_unstale":"2023-06-10T21:30:05.450517+0000","last_undegraded":"2023-06-10T21:30:05.450517+0000","last_fullsized":"2023-06-10T21:30:05.450517+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:55:36.556314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":37,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449700+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:30:05.449700+0000","last_peered":"2023-06-10T21:30:05.449700+0000","last_clean":"2023-06-10T21:30:05.449700+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:30:05.449700+0000","last_undegraded":"2023-06-10T21:30:05.449700+0000","last_fullsized":"2023-06-10T21:30:05.449700+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":197,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.450543+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:30:05.450543+0000","last_peered":"2023-06-10T21:30:05.450543+0000","last_clean":"2023-06-10T21:30:05.450543+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:30:05.450543+0000","last_undegraded":"2023-06-10T21:30:05.450543+0000","last_fullsized":"2023-06-10T21:30:05.450543+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":242,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449749+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:30:05.449749+0000","last_peered":"2023-06-10T21:30:05.449749+0000","last_clean":"2023-06-10T21:30:05.449749+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:30:05.449749+0000","last_undegraded":"2023-06-10T21:30:05.449749+0000","last_fullsized":"2023-06-10T21:30:05.449749+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":246,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449529+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:30:05.449529+0000","last_peered":"2023-06-10T21:30:05.449529+0000","last_clean":"2023-06-10T21:30:05.449529+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:30:05.449529+0000","last_undegraded":"2023-06-10T21:30:05.449529+0000","last_fullsized":"2023-06-10T21:30:05.449529+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":148,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410797+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:30:11.410797+0000","last_peered":"2023-06-10T21:30:11.410797+0000","last_clean":"2023-06-10T21:30:11.410797+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:30:11.410797+0000","last_undegraded":"2023-06-10T21:30:11.410797+0000","last_fullsized":"2023-06-10T21:30:11.410797+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":33,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-06-10T21:30:05.449330+0000","last_change":"2023-06-10T21:28:57.894606+0000","last_active":"2023-06-10T21:30:05.449330+0000","last_peered":"2023-06-10T21:30:05.449330+0000","last_clean":"2023-06-10T21:30:05.449330+0000","last_became_active":"2023-06-10T21:28:57.894004+0000","last_became_peered":"2023-06-10T21:28:57.894004+0000","last_unstale":"2023-06-10T21:30:05.449330+0000","last_undegraded":"2023-06-10T21:30:05.449330+0000","last_fullsized":"2023-06-10T21:30:05.449330+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:28:45.258464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":881000472,"num_objects":13444,"num_object_clones":0,"num_object_copies":40332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13444,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168064,"num_write_kb":5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20275,"num_bytes_recovered":1321598976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4107546624,"data_stored":4107534408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126662,"ondisk_log_size":126662,"up":48,"acting":40,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":6076,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430599,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":780380,"kb_used_data":264276,"kb_used_omap":0,"kb_used_meta":516096,"kb_avail":92964772,"statfs":{"total":95995035648,"available":95195926528,"internally_reserved":0,"allocated":270618624,"data_stored":270073019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":528482304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7080000000000002,"5min":9.7080000000000002,"15min":9.7080000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":89.245000000000005,"5min":89.245000000000005,"15min":89.245000000000005},"last":2.9540000000000002},{"interface":"front","average":{"1min":10.050000000000001,"5min":10.050000000000001,"15min":10.050000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":90.027000000000001,"5min":90.027000000000001,"15min":90.027000000000001},"last":2.9329999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.2069999999999999,"15min":2.8370000000000002},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0629999999999997,"5min":93.093999999999994,"15min":130.19399999999999},"last":2.641},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.796,"15min":3.1589999999999998},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.673,"5min":93.119,"15min":212.20599999999999},"last":2.6899999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6040000000000001,"15min":2.3929999999999998},"min":{"1min":0.377,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8129999999999997,"5min":160.99299999999999,"15min":160.99299999999999},"last":4.1269999999999998},{"interface":"front","average":{"1min":1.998,"5min":4.327,"15min":3.0030000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0529999999999999,"5min":160.53700000000001,"15min":212.13399999999999},"last":3.2309999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:29:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.5,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.2490000000000006,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":3.0800000000000001},{"interface":"front","average":{"1min":1.833,"5min":1.833,"15min":1.833},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.242,"5min":4.242,"15min":4.242},"last":2.6600000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.74299999999999999,"15min":0.97299999999999998},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":3.9199999999999999,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.379},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.95299999999999996,"15min":1.363},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.0470000000000006,"5min":14.657999999999999,"15min":14.657999999999999},"last":0.27200000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.1000000000000001,"15min":2.1560000000000001},"min":{"1min":0.33400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":2.1160000000000001,"5min":12.048,"15min":211.99799999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.92400000000000004,"15min":1.9670000000000001},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.7,"5min":14.988,"15min":212.23699999999999},"last":0.34999999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.0589999999999999,"15min":1.2989999999999999},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1680000000000001,"5min":13.073,"15min":13.073},"last":0.315},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.71699999999999997,"15min":1.111},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.724,"5min":3.8239999999999998,"15min":10.874000000000001},"last":0.39600000000000002}]}]},{"osd":6,"up_from":0,"seq":180388626827,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566269,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":686144,"kb_used_data":237560,"kb_used_omap":4,"kb_used_meta":448571,"kb_avail":93059008,"statfs":{"total":95995035648,"available":95292424192,"internally_reserved":0,"allocated":243261440,"data_stored":242442003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":459337614},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:34 2023","interfaces":[{"interface":"back","average":{"1min":13.73,"5min":13.73,"15min":13.73},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":107.825,"5min":107.825,"15min":107.825},"last":19.23},{"interface":"front","average":{"1min":13.321,"5min":13.321,"15min":13.321},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":110.928,"5min":110.928,"15min":110.928},"last":19.154}]},{"osd":2,"last update":"Sat Jun 10 21:29:27 2023","interfaces":[{"interface":"back","average":{"1min":9.0310000000000006,"5min":3.8580000000000001,"15min":2.2149999999999999},"min":{"1min":0.23799999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":107.857,"5min":107.857,"15min":107.857},"last":4.4740000000000002},{"interface":"front","average":{"1min":8.8510000000000009,"5min":3.7320000000000002,"15min":1.8720000000000001},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":107.958,"5min":107.958,"15min":107.958},"last":19.277999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.976,"5min":1.976,"15min":1.976},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.445,"5min":11.445,"15min":11.445},"last":0.65400000000000003},{"interface":"front","average":{"1min":3.1819999999999999,"5min":3.1819999999999999,"15min":3.1819999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":10.856,"5min":10.856,"15min":10.856},"last":19.190999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3839999999999999,"15min":1.71},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":16.059999999999999,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.641,"5min":1.03,"15min":1.5920000000000001},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.8949999999999996,"5min":6.8949999999999996,"15min":14.460000000000001},"last":0.76000000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.4490000000000001,"15min":4.2670000000000003},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":5.0700000000000003,"5min":6.9550000000000001,"15min":383.83100000000002},"last":0.73399999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.389,"15min":4.1070000000000002},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.29699999999999999},"max":{"1min":16.146999999999998,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.68200000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.387,"5min":1.905,"15min":2.1589999999999998},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.33800000000000002},"max":{"1min":6.6859999999999999,"5min":13.465,"15min":14.858000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":2,"5min":1.7270000000000001,"15min":1.956},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":11.476000000000001,"5min":13.48,"15min":13.48},"last":0.81100000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.6839999999999999,"15min":2.7810000000000001},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.32600000000000001},"max":{"1min":16.113,"5min":16.113,"15min":162.399},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.3280000000000001,"15min":2.367},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9459999999999997,"5min":6.9459999999999997,"15min":162.27199999999999},"last":0.79400000000000004}]}]},{"osd":0,"up_from":244,"seq":1047972020246,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724636,"kb_used_data":562964,"kb_used_omap":10,"kb_used_meta":161653,"kb_avail":93020516,"statfs":{"total":95995035648,"available":95253008384,"internally_reserved":0,"allocated":576475136,"data_stored":575568737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":165532724},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4100000000000001,"5min":4.4100000000000001,"15min":4.4100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":19.806999999999999,"5min":19.806999999999999,"15min":19.806999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":6.0620000000000003,"5min":6.0620000000000003,"15min":6.0620000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":29.841000000000001,"5min":29.841000000000001,"15min":29.841000000000001},"last":8.9949999999999992}]},{"osd":2,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.952,"5min":5.952,"15min":5.952},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":14.051,"5min":14.051,"15min":14.051},"last":7.4039999999999999},{"interface":"front","average":{"1min":5.1059999999999999,"5min":5.1059999999999999,"15min":5.1059999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":19.882999999999999,"5min":19.882999999999999,"15min":19.882999999999999},"last":5.952}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":6.2869999999999999,"5min":6.2869999999999999,"15min":6.2869999999999999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":17.088000000000001,"5min":17.088000000000001,"15min":17.088000000000001},"last":7.4290000000000003},{"interface":"front","average":{"1min":7.2839999999999998,"5min":7.2839999999999998,"15min":7.2839999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":32.822000000000003,"5min":32.822000000000003,"15min":32.822000000000003},"last":1.115}]},{"osd":4,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.0970000000000004,"5min":5.0970000000000004,"15min":5.0970000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":29.709,"5min":29.709,"15min":29.709},"last":8.923},{"interface":"front","average":{"1min":4.2750000000000004,"5min":4.2750000000000004,"15min":4.2750000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":13.872,"5min":13.872,"15min":13.872},"last":7.3120000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3860000000000001,"5min":4.3860000000000001,"15min":4.3860000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":7.3470000000000004},{"interface":"front","average":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":19.846,"5min":19.846,"15min":19.846},"last":0.57999999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.3170000000000002,"5min":5.3170000000000002,"15min":5.3170000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":29.806000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":8.8789999999999996},{"interface":"front","average":{"1min":4.423,"5min":4.423,"15min":4.423},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":13.82,"5min":13.82,"15min":13.82},"last":7.3810000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0880000000000001,"5min":4.0880000000000001,"15min":4.0880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":19.866,"5min":19.866,"15min":19.866},"last":0.61199999999999999},{"interface":"front","average":{"1min":5.3150000000000004,"5min":5.3150000000000004,"15min":5.3150000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":29.77,"5min":29.77,"15min":29.77},"last":8.9629999999999992}]}]},{"osd":2,"up_from":120,"seq":515396075658,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":743360,"kb_used_data":442104,"kb_used_omap":7,"kb_used_meta":301112,"kb_avail":93001792,"statfs":{"total":95995035648,"available":95233835008,"internally_reserved":0,"allocated":452714496,"data_stored":451903035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":308338769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:35 2023","interfaces":[{"interface":"back","average":{"1min":4.7039999999999997,"5min":4.7039999999999997,"15min":4.7039999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":14.304,"5min":14.304,"15min":14.304},"last":1.7470000000000001},{"interface":"front","average":{"1min":5.3920000000000003,"5min":5.3920000000000003,"15min":5.3920000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":20.792999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":5.2240000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:29:18 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.841,"15min":1.399},"min":{"1min":0.218,"5min":0.19,"15min":0.19},"max":{"1min":11.6,"5min":11.6,"15min":29.486000000000001},"last":5.5010000000000003},{"interface":"front","average":{"1min":2.5019999999999998,"5min":1.496,"15min":1.2789999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":14.334,"5min":14.334,"15min":28.013999999999999},"last":1.7789999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:29:57 2023","interfaces":[{"interface":"back","average":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":5.2409999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":14.272,"5min":14.272,"15min":14.272},"last":4.1920000000000002},{"interface":"front","average":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":16.411000000000001,"5min":16.411000000000001,"15min":16.411000000000001},"last":5.2809999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:29:18 2023","interfaces":[{"interface":"back","average":{"1min":8.7910000000000004,"5min":5.3019999999999996,"15min":2.4729999999999999},"min":{"1min":0.38400000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":110.337,"5min":224.00999999999999,"15min":224.00999999999999},"last":5.1459999999999999},{"interface":"front","average":{"1min":8.375,"5min":5.2960000000000003,"15min":2.516},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":110.2,"5min":223.28399999999999,"15min":223.28399999999999},"last":5.5330000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:29:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3180000000000001,"5min":1.3460000000000001,"15min":1.218},"min":{"1min":0.34200000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":14.180999999999999,"5min":14.180999999999999,"15min":14.180999999999999},"last":1.891},{"interface":"front","average":{"1min":2.4660000000000002,"5min":1.736,"15min":1.383},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":16.125,"5min":16.125,"15min":16.125},"last":5.1029999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:29:18 2023","interfaces":[{"interface":"back","average":{"1min":2.5899999999999999,"5min":1.974,"15min":1.4399999999999999},"min":{"1min":0.38,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":16.183,"5min":16.183,"15min":16.183},"last":5.1970000000000001},{"interface":"front","average":{"1min":2.4510000000000001,"5min":2.0459999999999998,"15min":1.5409999999999999},"min":{"1min":0.35999999999999999,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":10.763999999999999,"5min":11.029,"15min":17.244},"last":5.5629999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:29:18 2023","interfaces":[{"interface":"back","average":{"1min":8.484,"5min":2.806,"15min":1.6160000000000001},"min":{"1min":0.34799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":113.913,"5min":113.913,"15min":113.913},"last":1.861},{"interface":"front","average":{"1min":9.1189999999999998,"5min":3.2269999999999999,"15min":1.881},"min":{"1min":0.42899999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":113.937,"5min":113.937,"15min":113.937},"last":5.1710000000000003}]}]},{"osd":3,"up_from":250,"seq":1073741824017,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":940684,"kb_used_data":836804,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":92804468,"statfs":{"total":95995035648,"available":95031775232,"internally_reserved":0,"allocated":856887296,"data_stored":855996146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":106356755},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":5.5490000000000004,"15min":5.5490000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":56.828000000000003,"5min":56.828000000000003,"15min":56.828000000000003},"last":3.3010000000000002},{"interface":"front","average":{"1min":5.3330000000000002,"5min":5.3330000000000002,"15min":5.3330000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":57.600000000000001,"5min":57.600000000000001,"15min":57.600000000000001},"last":2.222}]},{"osd":1,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":16.925999999999998,"5min":16.925999999999998,"15min":16.925999999999998},"last":4.173},{"interface":"front","average":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":3.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.2579999999999991,"5min":8.2579999999999991,"15min":8.2579999999999991},"last":3.1480000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":12.922000000000001,"5min":12.922000000000001,"15min":12.922000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":87.555999999999997,"5min":87.555999999999997,"15min":87.555999999999997},"last":4.0599999999999996},{"interface":"front","average":{"1min":14.343,"5min":14.343,"15min":14.343},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":80.566999999999993,"5min":80.566999999999993,"15min":80.566999999999993},"last":9.4220000000000006}]},{"osd":4,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.7690000000000001,"5min":7.7690000000000001,"15min":7.7690000000000001},"last":1.5940000000000001},{"interface":"front","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7149999999999999,"5min":8.7149999999999999,"15min":8.7149999999999999},"last":4.2160000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.393,"15min":1.393},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.2480000000000002,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":3.2029999999999998},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":7.8319999999999999,"5min":7.8319999999999999,"15min":7.8319999999999999},"last":1.6339999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":8.6370000000000005,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":4.2469999999999999},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3449999999999998,"5min":6.3449999999999998,"15min":6.3449999999999998},"last":3.2490000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.778,"5min":2.778,"15min":2.778},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.6809999999999992,"5min":8.6809999999999992,"15min":8.6809999999999992},"last":4.2789999999999999},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.3029999999999999,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":3.2770000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019283,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259800,"kb_used_data":703760,"kb_used_omap":0,"kb_used_meta":556032,"kb_avail":92485352,"statfs":{"total":95995035648,"available":94705000448,"internally_reserved":0,"allocated":720650240,"data_stored":720107203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":569376768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:33 2023","interfaces":[{"interface":"back","average":{"1min":10.678000000000001,"5min":10.678000000000001,"15min":10.678000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":144.22399999999999,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.433},{"interface":"front","average":{"1min":9.9399999999999995,"5min":9.9399999999999995,"15min":9.9399999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":144.303,"5min":144.303,"15min":144.303},"last":38.420000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.496,"15min":2.9900000000000002},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6,"5min":15.063000000000001,"15min":182.035},"last":6},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.718,"15min":2.8940000000000001},"min":{"1min":0.34799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.4760000000000009,"5min":15.052,"15min":181.73099999999999},"last":1.119}]},{"osd":2,"last update":"Sat Jun 10 21:29:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.036,"15min":1.2370000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.3639999999999999,"5min":22.614000000000001,"15min":22.614000000000001},"last":5.4699999999999998},{"interface":"front","average":{"1min":1.7370000000000001,"5min":1.617,"15min":1.22},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.6769999999999996,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.3919999999999999,"15min":2.3919999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.2379999999999995,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.044,"5min":2.044,"15min":2.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9640000000000004,"5min":5.9640000000000004,"15min":5.9640000000000004},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:29:23 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":1.1879999999999999,"15min":1.163},"min":{"1min":0.26000000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.742,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.2909999999999999,"15min":1.2549999999999999},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":5.4000000000000004,"5min":11.904999999999999,"15min":11.904999999999999},"last":0.65400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.149,"15min":1.2070000000000001},"min":{"1min":0.317,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.077,"5min":7.5090000000000003,"15min":8.2599999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.3340000000000001,"15min":1.28},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":6.8300000000000001,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.206,"15min":1.448},"min":{"1min":0.32900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.7689999999999999,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.0980000000000001,"15min":1.3180000000000001},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.052,"5min":8.9900000000000002,"15min":9.5060000000000002},"last":0.36799999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823054,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":561778688,"data_stored":561774616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":239403008,"data_stored":239403008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":436928512,"data_stored":436928512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":854261760,"data_stored":854261760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":694161408,"data_stored":694157336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3473408,"data_stored":3473408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1051856896,"data_stored":1051852824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":265682944,"data_stored":265682944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:30:23.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:30:23.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:23 smithi064 ceph-mon[180297]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 35081/40404 objects degraded (86.826%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:23.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:23 smithi064 ceph-mon[186706]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 35081/40404 objects degraded (86.826%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:23.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:23.698 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:23.724 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:30:23.744 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:23.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:23.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:23.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:23 smithi152 ceph-mon[156819]: pgmap v932: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 841 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 35081/40404 objects degraded (86.826%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:23.953 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:23.978 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:24.025 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:24.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:24.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:24.246 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:24.357 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:24.589 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:24.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:24.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:24.739 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:24.793 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:24.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:24.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:25.031 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:25 smithi064 ceph-mon[180297]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 777 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 35081/37359 objects degraded (93.902%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:25 smithi064 ceph-mon[186706]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 777 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 35081/37359 objects degraded (93.902%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:25 smithi152 ceph-mon[156819]: pgmap v933: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 777 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 35081/37359 objects degraded (93.902%); 1.7 MiB/s, 31 objects/s recovering 2023-06-10T21:30:26.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:26.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:26.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:26.387 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:26.439 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:26.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:26.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:26.580 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:26.629 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:26.634 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/37359 objects degraded (93.902%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:26.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:26.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:26.763 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:26.824 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:26.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:26.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:26.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/37359 objects degraded (93.902%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:26.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/37359 objects degraded (93.902%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:26.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:27.052 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:27.225 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:27.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:27.260 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:27.272 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:27.334 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:27.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:27.663 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:27.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:27.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:27.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:27 smithi064 ceph-mon[180297]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 35081/33204 objects degraded (105.653%); 761 KiB/s, 31 objects/s recovering 2023-06-10T21:30:27.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:27 smithi064 ceph-mon[186706]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 35081/33204 objects degraded (105.653%); 761 KiB/s, 31 objects/s recovering 2023-06-10T21:30:27.877 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:27.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:27 smithi152 ceph-mon[156819]: pgmap v934: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 35081/33204 objects degraded (105.653%); 761 KiB/s, 31 objects/s recovering 2023-06-10T21:30:28.099 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:28.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:28.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:28.240 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:28.291 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:28.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:28.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:28.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:28.434 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:30:28.480 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:30:28.526 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:30:28.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:28.570 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:30:28.623 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:30:28.657 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:30:28.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:30:28.741 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:30:28.750 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.750 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.751 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.752 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.752 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.752 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:30:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1918192957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:28.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1918192957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:28.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1918192957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:29.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:29.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:29.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:29.146 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:29.202 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:29.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:29.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:29.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:29.431 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:29.627 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:30:29.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[180297]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 35081/33204 objects degraded (105.653%); 697 KiB/s, 28 objects/s recovering 2023-06-10T21:30:29.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4011568485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:29.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:29.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[186706]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 35081/33204 objects degraded (105.653%); 697 KiB/s, 28 objects/s recovering 2023-06-10T21:30:29.668 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4011568485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:29.669 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:29 smithi152 ceph-mon[156819]: pgmap v935: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 691 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 35081/33204 objects degraded (105.653%); 697 KiB/s, 28 objects/s recovering 2023-06-10T21:30:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4011568485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:29.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:30.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:30:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[186706]: failed to encode map e263 with expected crc 2023-06-10T21:30:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[186706]: osdmap e263: 8 total, 8 up, 6 in 2023-06-10T21:30:30.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[186706]: failed to encode map e263 with expected crc 2023-06-10T21:30:30.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:30:30.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[180297]: failed to encode map e263 with expected crc 2023-06-10T21:30:30.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:30.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[180297]: osdmap e263: 8 total, 8 up, 6 in 2023-06-10T21:30:30.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:30 smithi064 ceph-mon[180297]: failed to encode map e263 with expected crc 2023-06-10T21:30:30.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:30 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:30:30.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:30 smithi152 ceph-mon[156819]: failed to encode map e263 with expected crc 2023-06-10T21:30:30.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:30.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:30 smithi152 ceph-mon[156819]: osdmap e263: 8 total, 8 up, 6 in 2023-06-10T21:30:30.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:30 smithi152 ceph-mon[156819]: failed to encode map e263 with expected crc 2023-06-10T21:30:31.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:31 smithi064 ceph-mon[180297]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:31.619 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/25701 objects degraded (136.497%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:31.666 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:31.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:31.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:31.863 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:31.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:31 smithi064 ceph-mon[186706]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:31.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/25701 objects degraded (136.497%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:31 smithi152 ceph-mon[156819]: pgmap v937: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:31.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/25701 objects degraded (136.497%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:32.222 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:32.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:32.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:32.458 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:32.600 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:32.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:32.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:32.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:32 smithi064 ceph-mon[180297]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:32.788 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:32 smithi064 ceph-mon[186706]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:32.797 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:32 smithi152 ceph-mon[156819]: pgmap v938: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 534 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 489 op/s; 35081/25701 objects degraded (136.497%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:32.974 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:32.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:33.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:30:33.116 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:33.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:32] "GET /metrics HTTP/1.1" 200 223746 "" "Prometheus/2.33.4" 2023-06-10T21:30:33.165 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:30:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:33.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:30:33.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:33.316 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:33.317 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:30:33.317 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:30:33.368 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:33.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:33.475 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:33.564 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:33.583 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:30:33.628 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:30:33.707 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:33.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:33.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:33.903 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:33.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:30:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[186706]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:34 smithi064 ceph-mon[180297]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:34.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:30:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:34 smithi152 ceph-mon[156819]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:34.773 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:34.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:34.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:34.906 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:34.965 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:34.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:34.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:35.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:35.172 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:35 smithi152 ceph-mon[156819]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 474 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 35081/22779 objects degraded (154.006%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:35.191 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:35.557 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:35 smithi064 ceph-mon[180297]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 474 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 35081/22779 objects degraded (154.006%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:35.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:35 smithi064 ceph-mon[186706]: pgmap v939: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 474 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 485 op/s; 35081/22779 objects degraded (154.006%); 0 B/s, 34 objects/s recovering 2023-06-10T21:30:35.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:30:35.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:35.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:35.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:35.676 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":938,"stamp":"2023-06-10T21:30:31.720630+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":560404390,"num_objects":8567,"num_object_clones":0,"num_object_copies":25701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8567,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":173226,"num_write_kb":5810639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20656,"num_bytes_recovered":1323376156,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126958,"ondisk_log_size":126958,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4278852,"kb_used_data":2174868,"kb_used_omap":30,"kb_used_meta":2103777,"kb_avail":558192060,"statfs":{"total":575970213888,"available":571588669440,"internally_reserved":0,"allocated":2227064832,"data_stored":2222546418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2154267686},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,3,5,8,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-321191936,"num_objects":-4901,"num_object_clones":0,"num_object_copies":-14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4901,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":348,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008142"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":253,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.407635+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:30:11.407635+0000","last_peered":"2023-06-10T21:30:11.407635+0000","last_clean":"2023-06-10T21:30:11.407635+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:30:11.407635+0000","last_undegraded":"2023-06-10T21:30:11.407635+0000","last_fullsized":"2023-06-10T21:30:11.407635+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":195,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216122+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:30:17.216122+0000","last_peered":"2023-06-10T21:30:17.216122+0000","last_clean":"2023-06-10T21:30:17.216122+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:30:17.216122+0000","last_undegraded":"2023-06-10T21:30:17.216122+0000","last_fullsized":"2023-06-10T21:30:17.216122+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":170,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197539+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:30:17.197539+0000","last_peered":"2023-06-10T21:30:17.197539+0000","last_clean":"2023-06-10T21:30:17.197539+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:30:17.197539+0000","last_undegraded":"2023-06-10T21:30:17.197539+0000","last_fullsized":"2023-06-10T21:30:17.197539+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412422+0000","last_change":"2023-06-10T21:28:57.902734+0000","last_active":"2023-06-10T21:30:11.412422+0000","last_peered":"2023-06-10T21:30:11.412422+0000","last_clean":"2023-06-10T21:30:11.412422+0000","last_became_active":"2023-06-10T21:28:57.902279+0000","last_became_peered":"2023-06-10T21:28:57.902279+0000","last_unstale":"2023-06-10T21:30:11.412422+0000","last_undegraded":"2023-06-10T21:30:11.412422+0000","last_fullsized":"2023-06-10T21:30:11.412422+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:46.251698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197664+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:30:17.197664+0000","last_peered":"2023-06-10T21:30:17.197664+0000","last_clean":"2023-06-10T21:30:17.197664+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:30:17.197664+0000","last_undegraded":"2023-06-10T21:30:17.197664+0000","last_fullsized":"2023-06-10T21:30:17.197664+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":226,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410932+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:30:11.410932+0000","last_peered":"2023-06-10T21:30:11.410932+0000","last_clean":"2023-06-10T21:30:11.410932+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:30:11.410932+0000","last_undegraded":"2023-06-10T21:30:11.410932+0000","last_fullsized":"2023-06-10T21:30:11.410932+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410999+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:30:11.410999+0000","last_peered":"2023-06-10T21:30:11.410999+0000","last_clean":"2023-06-10T21:30:11.410999+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:30:11.410999+0000","last_undegraded":"2023-06-10T21:30:11.410999+0000","last_fullsized":"2023-06-10T21:30:11.410999+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197422+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:30:17.197422+0000","last_peered":"2023-06-10T21:30:17.197422+0000","last_clean":"2023-06-10T21:30:17.197422+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:30:17.197422+0000","last_undegraded":"2023-06-10T21:30:17.197422+0000","last_fullsized":"2023-06-10T21:30:17.197422+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.185945+0000","last_change":"2023-06-10T21:28:57.899055+0000","last_active":"2023-06-10T21:30:17.185945+0000","last_peered":"2023-06-10T21:30:17.185945+0000","last_clean":"2023-06-10T21:30:17.185945+0000","last_became_active":"2023-06-10T21:28:57.898692+0000","last_became_peered":"2023-06-10T21:28:57.898692+0000","last_unstale":"2023-06-10T21:30:17.185945+0000","last_undegraded":"2023-06-10T21:30:17.185945+0000","last_fullsized":"2023-06-10T21:30:17.185945+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197725+0000","last_change":"2023-06-10T21:28:57.895548+0000","last_active":"2023-06-10T21:30:17.197725+0000","last_peered":"2023-06-10T21:30:17.197725+0000","last_clean":"2023-06-10T21:30:17.197725+0000","last_became_active":"2023-06-10T21:28:57.895217+0000","last_became_peered":"2023-06-10T21:28:57.895217+0000","last_unstale":"2023-06-10T21:30:17.197725+0000","last_undegraded":"2023-06-10T21:30:17.197725+0000","last_fullsized":"2023-06-10T21:30:17.197725+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:46:40.560609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":248,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186181+0000","last_change":"2023-06-10T21:28:57.899846+0000","last_active":"2023-06-10T21:30:17.186181+0000","last_peered":"2023-06-10T21:30:17.186181+0000","last_clean":"2023-06-10T21:30:17.186181+0000","last_became_active":"2023-06-10T21:28:57.898805+0000","last_became_peered":"2023-06-10T21:28:57.898805+0000","last_unstale":"2023-06-10T21:30:17.186181+0000","last_undegraded":"2023-06-10T21:30:17.186181+0000","last_fullsized":"2023-06-10T21:30:17.186181+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.407549+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:30:11.407549+0000","last_peered":"2023-06-10T21:30:11.407549+0000","last_clean":"2023-06-10T21:30:11.407549+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:30:11.407549+0000","last_undegraded":"2023-06-10T21:30:11.407549+0000","last_fullsized":"2023-06-10T21:30:11.407549+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":133,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186022+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:30:17.186022+0000","last_peered":"2023-06-10T21:30:17.186022+0000","last_clean":"2023-06-10T21:30:17.186022+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:30:17.186022+0000","last_undegraded":"2023-06-10T21:30:17.186022+0000","last_fullsized":"2023-06-10T21:30:17.186022+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405429+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:30:11.405429+0000","last_peered":"2023-06-10T21:30:11.405429+0000","last_clean":"2023-06-10T21:30:11.405429+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:30:11.405429+0000","last_undegraded":"2023-06-10T21:30:11.405429+0000","last_fullsized":"2023-06-10T21:30:11.405429+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":262,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.185855+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:30:17.185855+0000","last_peered":"2023-06-10T21:30:17.185855+0000","last_clean":"2023-06-10T21:30:17.185855+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:30:17.185855+0000","last_undegraded":"2023-06-10T21:30:17.185855+0000","last_fullsized":"2023-06-10T21:30:17.185855+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215751+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:30:17.215751+0000","last_peered":"2023-06-10T21:30:17.215751+0000","last_clean":"2023-06-10T21:30:17.215751+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:30:17.215751+0000","last_undegraded":"2023-06-10T21:30:17.215751+0000","last_fullsized":"2023-06-10T21:30:17.215751+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.414440+0000","last_change":"2023-06-10T21:28:57.904420+0000","last_active":"2023-06-10T21:30:11.414440+0000","last_peered":"2023-06-10T21:30:11.414440+0000","last_clean":"2023-06-10T21:30:11.414440+0000","last_became_active":"2023-06-10T21:28:57.904027+0000","last_became_peered":"2023-06-10T21:28:57.904027+0000","last_unstale":"2023-06-10T21:30:11.414440+0000","last_undegraded":"2023-06-10T21:30:11.414440+0000","last_fullsized":"2023-06-10T21:30:11.414440+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:15:36.667597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":191,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186316+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:30:17.186316+0000","last_peered":"2023-06-10T21:30:17.186316+0000","last_clean":"2023-06-10T21:30:17.186316+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:30:17.186316+0000","last_undegraded":"2023-06-10T21:30:17.186316+0000","last_fullsized":"2023-06-10T21:30:17.186316+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.185388+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:30:17.185388+0000","last_peered":"2023-06-10T21:30:17.185388+0000","last_clean":"2023-06-10T21:30:17.185388+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:30:17.185388+0000","last_undegraded":"2023-06-10T21:30:17.185388+0000","last_fullsized":"2023-06-10T21:30:17.185388+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412265+0000","last_change":"2023-06-10T21:28:57.902565+0000","last_active":"2023-06-10T21:30:11.412265+0000","last_peered":"2023-06-10T21:30:11.412265+0000","last_clean":"2023-06-10T21:30:11.412265+0000","last_became_active":"2023-06-10T21:28:57.901902+0000","last_became_peered":"2023-06-10T21:28:57.901902+0000","last_unstale":"2023-06-10T21:30:11.412265+0000","last_undegraded":"2023-06-10T21:30:11.412265+0000","last_fullsized":"2023-06-10T21:30:11.412265+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:54.534642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":235,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.409075+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:30:11.409075+0000","last_peered":"2023-06-10T21:30:11.409075+0000","last_clean":"2023-06-10T21:30:11.409075+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:30:11.409075+0000","last_undegraded":"2023-06-10T21:30:11.409075+0000","last_fullsized":"2023-06-10T21:30:11.409075+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216442+0000","last_change":"2023-06-10T21:28:57.889771+0000","last_active":"2023-06-10T21:30:17.216442+0000","last_peered":"2023-06-10T21:30:17.216442+0000","last_clean":"2023-06-10T21:30:17.216442+0000","last_became_active":"2023-06-10T21:28:57.887036+0000","last_became_peered":"2023-06-10T21:28:57.887036+0000","last_unstale":"2023-06-10T21:30:17.216442+0000","last_undegraded":"2023-06-10T21:30:17.216442+0000","last_fullsized":"2023-06-10T21:30:17.216442+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:05.952139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"262'10816","reported_seq":12563,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:27.789886+0000","last_change":"2023-06-10T21:28:34.624898+0000","last_active":"2023-06-10T21:30:27.789886+0000","last_peered":"2023-06-10T21:30:27.789886+0000","last_clean":"2023-06-10T21:30:27.789886+0000","last_became_active":"2023-06-10T21:28:34.501130+0000","last_became_peered":"2023-06-10T21:28:34.501130+0000","last_unstale":"2023-06-10T21:30:27.789886+0000","last_undegraded":"2023-06-10T21:30:27.789886+0000","last_fullsized":"2023-06-10T21:30:27.789886+0000","mapping_epoch":244,"log_start":"223'4600","ondisk_log_start":"223'4600","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6216,"log_dups_size":0,"ondisk_log_size":6216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:11:04.611578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35520512,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10816,"num_write_kb":363456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":832,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":272,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186496+0000","last_change":"2023-06-10T21:28:34.522514+0000","last_active":"2023-06-10T21:30:17.186496+0000","last_peered":"2023-06-10T21:30:17.186496+0000","last_clean":"2023-06-10T21:30:17.186496+0000","last_became_active":"2023-06-10T21:28:34.522038+0000","last_became_peered":"2023-06-10T21:28:34.522038+0000","last_unstale":"2023-06-10T21:30:17.186496+0000","last_undegraded":"2023-06-10T21:30:17.186496+0000","last_fullsized":"2023-06-10T21:30:17.186496+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.189371+0000","last_change":"2023-06-10T21:28:57.899217+0000","last_active":"2023-06-10T21:30:17.189371+0000","last_peered":"2023-06-10T21:30:17.189371+0000","last_clean":"2023-06-10T21:30:17.189371+0000","last_became_active":"2023-06-10T21:28:57.898541+0000","last_became_peered":"2023-06-10T21:28:57.898541+0000","last_unstale":"2023-06-10T21:30:17.189371+0000","last_undegraded":"2023-06-10T21:30:17.189371+0000","last_fullsized":"2023-06-10T21:30:17.189371+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"262'10895","reported_seq":19412,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:29.164954+0000","last_change":"2023-06-10T21:28:59.502716+0000","last_active":"2023-06-10T21:30:29.164954+0000","last_peered":"2023-06-10T21:30:29.164954+0000","last_clean":"2023-06-10T21:28:56.868806+0000","last_became_active":"2023-06-10T21:28:59.444403+0000","last_became_peered":"2023-06-10T21:28:59.444403+0000","last_unstale":"2023-06-10T21:30:29.164954+0000","last_undegraded":"2023-06-10T21:28:59.066225+0000","last_fullsized":"2023-06-10T21:28:58.880029+0000","mapping_epoch":251,"log_start":"200'800","ondisk_log_start":"200'800","created":194,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10095,"log_dups_size":0,"ondisk_log_size":10095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:26.444508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34406400,"num_objects":525,"num_object_clones":0,"num_object_copies":1575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10895,"num_write_kb":365440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4198,"num_bytes_recovered":275054592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1789},{"shards":"2,4,6","objects":525}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":232,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186036+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:30:17.186036+0000","last_peered":"2023-06-10T21:30:17.186036+0000","last_clean":"2023-06-10T21:30:17.186036+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:30:17.186036+0000","last_undegraded":"2023-06-10T21:30:17.186036+0000","last_fullsized":"2023-06-10T21:30:17.186036+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":134,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197510+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:30:17.197510+0000","last_peered":"2023-06-10T21:30:17.197510+0000","last_clean":"2023-06-10T21:30:17.197510+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:30:17.197510+0000","last_undegraded":"2023-06-10T21:30:17.197510+0000","last_fullsized":"2023-06-10T21:30:17.197510+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"262'10869","reported_seq":10954,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:27.686104+0000","last_change":"2023-06-10T21:28:59.404926+0000","last_active":"2023-06-10T21:30:27.686104+0000","last_peered":"2023-06-10T21:30:27.686104+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:28:59.373005+0000","last_became_peered":"2023-06-10T21:28:59.373005+0000","last_unstale":"2023-06-10T21:30:27.686104+0000","last_undegraded":"2023-06-10T21:28:58.922286+0000","last_fullsized":"2023-06-10T21:28:58.878960+0000","mapping_epoch":251,"log_start":"223'4700","ondisk_log_start":"223'4700","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:21.872755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10869,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":-1666}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":257,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186025+0000","last_change":"2023-06-10T21:28:57.900006+0000","last_active":"2023-06-10T21:30:17.186025+0000","last_peered":"2023-06-10T21:30:17.186025+0000","last_clean":"2023-06-10T21:30:17.186025+0000","last_became_active":"2023-06-10T21:28:57.898709+0000","last_became_peered":"2023-06-10T21:28:57.898709+0000","last_unstale":"2023-06-10T21:30:17.186025+0000","last_undegraded":"2023-06-10T21:30:17.186025+0000","last_fullsized":"2023-06-10T21:30:17.186025+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.185434+0000","last_change":"2023-06-10T21:28:34.551591+0000","last_active":"2023-06-10T21:30:17.185434+0000","last_peered":"2023-06-10T21:30:17.185434+0000","last_clean":"2023-06-10T21:30:17.185434+0000","last_became_active":"2023-06-10T21:28:34.551254+0000","last_became_peered":"2023-06-10T21:28:34.551254+0000","last_unstale":"2023-06-10T21:30:17.185434+0000","last_undegraded":"2023-06-10T21:30:17.185434+0000","last_fullsized":"2023-06-10T21:30:17.185434+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"262'10729","reported_seq":10833,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:29.102304+0000","last_change":"2023-06-10T21:28:59.445842+0000","last_active":"2023-06-10T21:30:29.102304+0000","last_peered":"2023-06-10T21:30:29.102304+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:28:59.419763+0000","last_became_peered":"2023-06-10T21:28:59.419763+0000","last_unstale":"2023-06-10T21:30:29.102304+0000","last_undegraded":"2023-06-10T21:28:58.932194+0000","last_fullsized":"2023-06-10T21:28:58.879691+0000","mapping_epoch":251,"log_start":"200'700","ondisk_log_start":"200'700","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10029,"log_dups_size":0,"ondisk_log_size":10029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:46.862125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4698,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10729,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-1755}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":142,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215859+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:30:17.215859+0000","last_peered":"2023-06-10T21:30:17.215859+0000","last_clean":"2023-06-10T21:30:17.215859+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:30:17.215859+0000","last_undegraded":"2023-06-10T21:30:17.215859+0000","last_fullsized":"2023-06-10T21:30:17.215859+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411430+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:30:11.411430+0000","last_peered":"2023-06-10T21:30:11.411430+0000","last_clean":"2023-06-10T21:30:11.411430+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:30:11.411430+0000","last_undegraded":"2023-06-10T21:30:11.411430+0000","last_fullsized":"2023-06-10T21:30:11.411430+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"262'10943","reported_seq":11035,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:27.776077+0000","last_change":"2023-06-10T21:28:34.940046+0000","last_active":"2023-06-10T21:30:27.776077+0000","last_peered":"2023-06-10T21:30:27.776077+0000","last_clean":"2023-06-10T21:30:27.776077+0000","last_became_active":"2023-06-10T21:28:34.501595+0000","last_became_peered":"2023-06-10T21:28:34.501595+0000","last_unstale":"2023-06-10T21:30:27.776077+0000","last_undegraded":"2023-06-10T21:30:27.776077+0000","last_fullsized":"2023-06-10T21:30:27.776077+0000","mapping_epoch":244,"log_start":"223'4802","ondisk_log_start":"223'4802","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:59:45.255711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37814272,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10943,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":212,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215770+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:30:17.215770+0000","last_peered":"2023-06-10T21:30:17.215770+0000","last_clean":"2023-06-10T21:30:17.215770+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:30:17.215770+0000","last_undegraded":"2023-06-10T21:30:17.215770+0000","last_fullsized":"2023-06-10T21:30:17.215770+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":132,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216156+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:30:17.216156+0000","last_peered":"2023-06-10T21:30:17.216156+0000","last_clean":"2023-06-10T21:30:17.216156+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:30:17.216156+0000","last_undegraded":"2023-06-10T21:30:17.216156+0000","last_fullsized":"2023-06-10T21:30:17.216156+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"262'10518","reported_seq":17556,"reported_epoch":262,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-10T21:30:29.179268+0000","last_change":"2023-06-10T21:30:18.062068+0000","last_active":"2023-06-10T21:30:29.179268+0000","last_peered":"2023-06-10T21:30:29.179268+0000","last_clean":"2023-06-10T21:26:26.761025+0000","last_became_active":"2023-06-10T21:28:59.376175+0000","last_became_peered":"2023-06-10T21:28:59.376175+0000","last_unstale":"2023-06-10T21:30:29.179268+0000","last_undegraded":"2023-06-10T21:30:29.179268+0000","last_fullsized":"2023-06-10T21:28:58.879642+0000","mapping_epoch":251,"log_start":"200'500","ondisk_log_start":"200'500","created":194,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:28.355418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35258368,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10518,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3463,"num_bytes_recovered":201523200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":538}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":35,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215897+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:30:17.215897+0000","last_peered":"2023-06-10T21:30:17.215897+0000","last_clean":"2023-06-10T21:30:17.215897+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:30:17.215897+0000","last_undegraded":"2023-06-10T21:30:17.215897+0000","last_fullsized":"2023-06-10T21:30:17.215897+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":251,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405130+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:30:11.405130+0000","last_peered":"2023-06-10T21:30:11.405130+0000","last_clean":"2023-06-10T21:30:11.405130+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:30:11.405130+0000","last_undegraded":"2023-06-10T21:30:11.405130+0000","last_fullsized":"2023-06-10T21:30:11.405130+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"262'10937","reported_seq":16362,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.613182+0000","last_change":"2023-06-10T21:27:45.367487+0000","last_active":"2023-06-10T21:30:29.613182+0000","last_peered":"2023-06-10T21:30:29.613182+0000","last_clean":"2023-06-10T21:30:29.613182+0000","last_became_active":"2023-06-10T21:27:45.353438+0000","last_became_peered":"2023-06-10T21:27:45.353438+0000","last_unstale":"2023-06-10T21:30:29.613182+0000","last_undegraded":"2023-06-10T21:30:29.613182+0000","last_fullsized":"2023-06-10T21:30:29.613182+0000","mapping_epoch":234,"log_start":"218'3200","ondisk_log_start":"218'3200","created":194,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35061760,"num_objects":535,"num_object_clones":0,"num_object_copies":1605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10937,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3133,"num_bytes_recovered":205324288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":238,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.192493+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:30:17.192493+0000","last_peered":"2023-06-10T21:30:17.192493+0000","last_clean":"2023-06-10T21:30:17.192493+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:30:17.192493+0000","last_undegraded":"2023-06-10T21:30:17.192493+0000","last_fullsized":"2023-06-10T21:30:17.192493+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":35,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216377+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:30:17.216377+0000","last_peered":"2023-06-10T21:30:17.216377+0000","last_clean":"2023-06-10T21:30:17.216377+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:30:17.216377+0000","last_undegraded":"2023-06-10T21:30:17.216377+0000","last_fullsized":"2023-06-10T21:30:17.216377+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.1","version":"262'10859","reported_seq":10971,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.617106+0000","last_change":"2023-06-10T21:28:34.572666+0000","last_active":"2023-06-10T21:30:29.617106+0000","last_peered":"2023-06-10T21:30:29.617106+0000","last_clean":"2023-06-10T21:30:29.617106+0000","last_became_active":"2023-06-10T21:28:34.522012+0000","last_became_peered":"2023-06-10T21:28:34.522012+0000","last_unstale":"2023-06-10T21:30:29.617106+0000","last_undegraded":"2023-06-10T21:30:29.617106+0000","last_fullsized":"2023-06-10T21:30:29.617106+0000","mapping_epoch":244,"log_start":"212'3100","ondisk_log_start":"212'3100","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7759,"log_dups_size":0,"ondisk_log_size":7759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31391744,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10859,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":197,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197573+0000","last_change":"2023-06-10T21:28:57.894408+0000","last_active":"2023-06-10T21:30:17.197573+0000","last_peered":"2023-06-10T21:30:17.197573+0000","last_clean":"2023-06-10T21:30:17.197573+0000","last_became_active":"2023-06-10T21:28:57.893595+0000","last_became_peered":"2023-06-10T21:28:57.893595+0000","last_unstale":"2023-06-10T21:30:17.197573+0000","last_undegraded":"2023-06-10T21:30:17.197573+0000","last_fullsized":"2023-06-10T21:30:17.197573+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:03.720304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":171,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216512+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:30:17.216512+0000","last_peered":"2023-06-10T21:30:17.216512+0000","last_clean":"2023-06-10T21:30:17.216512+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:30:17.216512+0000","last_undegraded":"2023-06-10T21:30:17.216512+0000","last_fullsized":"2023-06-10T21:30:17.216512+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"262'10935","reported_seq":11044,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.632490+0000","last_change":"2023-06-10T21:28:34.926507+0000","last_active":"2023-06-10T21:30:29.632490+0000","last_peered":"2023-06-10T21:30:29.632490+0000","last_clean":"2023-06-10T21:30:29.632490+0000","last_became_active":"2023-06-10T21:28:34.541984+0000","last_became_peered":"2023-06-10T21:28:34.541984+0000","last_unstale":"2023-06-10T21:30:29.632490+0000","last_undegraded":"2023-06-10T21:30:29.632490+0000","last_fullsized":"2023-06-10T21:30:29.632490+0000","mapping_epoch":244,"log_start":"218'3200","ondisk_log_start":"218'3200","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31916032,"num_objects":487,"num_object_clones":0,"num_object_copies":1461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10935,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411675+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:30:11.411675+0000","last_peered":"2023-06-10T21:30:11.411675+0000","last_clean":"2023-06-10T21:30:11.411675+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:30:11.411675+0000","last_undegraded":"2023-06-10T21:30:11.411675+0000","last_fullsized":"2023-06-10T21:30:11.411675+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":246,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216242+0000","last_change":"2023-06-10T21:28:57.946656+0000","last_active":"2023-06-10T21:30:17.216242+0000","last_peered":"2023-06-10T21:30:17.216242+0000","last_clean":"2023-06-10T21:30:17.216242+0000","last_became_active":"2023-06-10T21:28:57.946186+0000","last_became_peered":"2023-06-10T21:28:57.946186+0000","last_unstale":"2023-06-10T21:30:17.216242+0000","last_undegraded":"2023-06-10T21:30:17.216242+0000","last_fullsized":"2023-06-10T21:30:17.216242+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:31.259349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":103,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411784+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:30:11.411784+0000","last_peered":"2023-06-10T21:30:11.411784+0000","last_clean":"2023-06-10T21:30:11.411784+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:30:11.411784+0000","last_undegraded":"2023-06-10T21:30:11.411784+0000","last_fullsized":"2023-06-10T21:30:11.411784+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"262'10774","reported_seq":13098,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:27.792194+0000","last_change":"2023-06-10T21:28:59.454017+0000","last_active":"2023-06-10T21:30:27.792194+0000","last_peered":"2023-06-10T21:30:27.792194+0000","last_clean":"2023-06-10T21:28:56.879563+0000","last_became_active":"2023-06-10T21:28:59.417345+0000","last_became_peered":"2023-06-10T21:28:59.417345+0000","last_unstale":"2023-06-10T21:30:27.792194+0000","last_undegraded":"2023-06-10T21:28:58.916372+0000","last_fullsized":"2023-06-10T21:28:58.878970+0000","mapping_epoch":251,"log_start":"223'4600","ondisk_log_start":"223'4600","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6174,"log_dups_size":0,"ondisk_log_size":6174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:23.819643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5415,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10774,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1110,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2","objects":-2352},{"shards":"0,2,6","objects":568}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":129,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411858+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:30:11.411858+0000","last_peered":"2023-06-10T21:30:11.411858+0000","last_clean":"2023-06-10T21:30:11.411858+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:30:11.411858+0000","last_undegraded":"2023-06-10T21:30:11.411858+0000","last_fullsized":"2023-06-10T21:30:11.411858+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":203,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411172+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:30:11.411172+0000","last_peered":"2023-06-10T21:30:11.411172+0000","last_clean":"2023-06-10T21:30:11.411172+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:30:11.411172+0000","last_undegraded":"2023-06-10T21:30:11.411172+0000","last_fullsized":"2023-06-10T21:30:11.411172+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405480+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:30:11.405480+0000","last_peered":"2023-06-10T21:30:11.405480+0000","last_clean":"2023-06-10T21:30:11.405480+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:30:11.405480+0000","last_undegraded":"2023-06-10T21:30:11.405480+0000","last_fullsized":"2023-06-10T21:30:11.405480+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"262'10720","reported_seq":12506,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.150533+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:30:29.150533+0000","last_peered":"2023-06-10T21:30:29.150533+0000","last_clean":"2023-06-10T21:30:29.150533+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:30:29.150533+0000","last_undegraded":"2023-06-10T21:30:29.150533+0000","last_fullsized":"2023-06-10T21:30:29.150533+0000","mapping_epoch":210,"log_start":"200'700","ondisk_log_start":"200'700","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10720,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":104,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405052+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:30:11.405052+0000","last_peered":"2023-06-10T21:30:11.405052+0000","last_clean":"2023-06-10T21:30:11.405052+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:30:11.405052+0000","last_undegraded":"2023-06-10T21:30:11.405052+0000","last_fullsized":"2023-06-10T21:30:11.405052+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.404774+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:30:11.404774+0000","last_peered":"2023-06-10T21:30:11.404774+0000","last_clean":"2023-06-10T21:30:11.404774+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:30:11.404774+0000","last_undegraded":"2023-06-10T21:30:11.404774+0000","last_fullsized":"2023-06-10T21:30:11.404774+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":134,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410868+0000","last_change":"2023-06-10T21:28:34.518545+0000","last_active":"2023-06-10T21:30:11.410868+0000","last_peered":"2023-06-10T21:30:11.410868+0000","last_clean":"2023-06-10T21:30:11.410868+0000","last_became_active":"2023-06-10T21:28:34.517742+0000","last_became_peered":"2023-06-10T21:28:34.517742+0000","last_unstale":"2023-06-10T21:30:11.410868+0000","last_undegraded":"2023-06-10T21:30:11.410868+0000","last_fullsized":"2023-06-10T21:30:11.410868+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:13.761246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"262'10771","reported_seq":17170,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:27.700683+0000","last_change":"2023-06-10T21:28:34.988147+0000","last_active":"2023-06-10T21:30:27.700683+0000","last_peered":"2023-06-10T21:30:27.700683+0000","last_clean":"2023-06-10T21:30:27.700683+0000","last_became_active":"2023-06-10T21:28:34.506722+0000","last_became_peered":"2023-06-10T21:28:34.506722+0000","last_unstale":"2023-06-10T21:30:27.700683+0000","last_undegraded":"2023-06-10T21:30:27.700683+0000","last_fullsized":"2023-06-10T21:30:27.700683+0000","mapping_epoch":244,"log_start":"223'4600","ondisk_log_start":"223'4600","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:56:33.672188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10771,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3157,"num_bytes_recovered":206438400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":266,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405333+0000","last_change":"2023-06-10T21:28:57.907800+0000","last_active":"2023-06-10T21:30:11.405333+0000","last_peered":"2023-06-10T21:30:11.405333+0000","last_clean":"2023-06-10T21:30:11.405333+0000","last_became_active":"2023-06-10T21:28:57.907509+0000","last_became_peered":"2023-06-10T21:28:57.907509+0000","last_unstale":"2023-06-10T21:30:11.405333+0000","last_undegraded":"2023-06-10T21:30:11.405333+0000","last_fullsized":"2023-06-10T21:30:11.405333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:56.001988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":550,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.189317+0000","last_change":"2023-06-10T21:28:34.503678+0000","last_active":"2023-06-10T21:30:17.189317+0000","last_peered":"2023-06-10T21:30:17.189317+0000","last_clean":"2023-06-10T21:30:17.189317+0000","last_became_active":"2023-06-10T21:28:34.503254+0000","last_became_peered":"2023-06-10T21:28:34.503254+0000","last_unstale":"2023-06-10T21:30:17.189317+0000","last_undegraded":"2023-06-10T21:30:17.189317+0000","last_fullsized":"2023-06-10T21:30:17.189317+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":198,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197696+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:30:17.197696+0000","last_peered":"2023-06-10T21:30:17.197696+0000","last_clean":"2023-06-10T21:30:17.197696+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:30:17.197696+0000","last_undegraded":"2023-06-10T21:30:17.197696+0000","last_fullsized":"2023-06-10T21:30:17.197696+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":77,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186256+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:30:17.186256+0000","last_peered":"2023-06-10T21:30:17.186256+0000","last_clean":"2023-06-10T21:30:17.186256+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:30:17.186256+0000","last_undegraded":"2023-06-10T21:30:17.186256+0000","last_fullsized":"2023-06-10T21:30:17.186256+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"262'10939","reported_seq":11034,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:27.759890+0000","last_change":"2023-06-10T21:28:34.951633+0000","last_active":"2023-06-10T21:30:27.759890+0000","last_peered":"2023-06-10T21:30:27.759890+0000","last_clean":"2023-06-10T21:30:27.759890+0000","last_became_active":"2023-06-10T21:28:34.509441+0000","last_became_peered":"2023-06-10T21:28:34.509441+0000","last_unstale":"2023-06-10T21:30:27.759890+0000","last_undegraded":"2023-06-10T21:30:27.759890+0000","last_fullsized":"2023-06-10T21:30:27.759890+0000","mapping_epoch":244,"log_start":"223'4800","ondisk_log_start":"223'4800","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6139,"log_dups_size":0,"ondisk_log_size":6139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:34:59.163129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37027840,"num_objects":565,"num_object_clones":0,"num_object_copies":1695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10939,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":282,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.189313+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:30:17.189313+0000","last_peered":"2023-06-10T21:30:17.189313+0000","last_clean":"2023-06-10T21:30:17.189313+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:30:17.189313+0000","last_undegraded":"2023-06-10T21:30:17.189313+0000","last_fullsized":"2023-06-10T21:30:17.189313+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":98,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410031+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:30:11.410031+0000","last_peered":"2023-06-10T21:30:11.410031+0000","last_clean":"2023-06-10T21:30:11.410031+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:30:11.410031+0000","last_undegraded":"2023-06-10T21:30:11.410031+0000","last_fullsized":"2023-06-10T21:30:11.410031+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412193+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:30:11.412193+0000","last_peered":"2023-06-10T21:30:11.412193+0000","last_clean":"2023-06-10T21:30:11.412193+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:30:11.412193+0000","last_undegraded":"2023-06-10T21:30:11.412193+0000","last_fullsized":"2023-06-10T21:30:11.412193+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"262'10910","reported_seq":11517,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:27.722121+0000","last_change":"2023-06-10T21:28:59.491171+0000","last_active":"2023-06-10T21:30:27.722121+0000","last_peered":"2023-06-10T21:30:27.722121+0000","last_clean":"2023-06-10T21:28:56.884482+0000","last_became_active":"2023-06-10T21:28:59.418669+0000","last_became_peered":"2023-06-10T21:28:59.418669+0000","last_unstale":"2023-06-10T21:30:27.722121+0000","last_undegraded":"2023-06-10T21:28:58.914273+0000","last_fullsized":"2023-06-10T21:28:58.879027+0000","mapping_epoch":251,"log_start":"223'4700","ondisk_log_start":"223'4700","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6210,"log_dups_size":0,"ondisk_log_size":6210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:47:23.775373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192856,"num_objects":538,"num_object_clones":0,"num_object_copies":1614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":538,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10910,"num_write_kb":366273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":255,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-2474},{"shards":"0,4,6","objects":538}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":267,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186103+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:30:17.186103+0000","last_peered":"2023-06-10T21:30:17.186103+0000","last_clean":"2023-06-10T21:30:17.186103+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:30:17.186103+0000","last_undegraded":"2023-06-10T21:30:17.186103+0000","last_fullsized":"2023-06-10T21:30:17.186103+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405550+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:30:11.405550+0000","last_peered":"2023-06-10T21:30:11.405550+0000","last_clean":"2023-06-10T21:30:11.405550+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:30:11.405550+0000","last_undegraded":"2023-06-10T21:30:11.405550+0000","last_fullsized":"2023-06-10T21:30:11.405550+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":79,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411231+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:30:11.411231+0000","last_peered":"2023-06-10T21:30:11.411231+0000","last_clean":"2023-06-10T21:30:11.411231+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:30:11.411231+0000","last_undegraded":"2023-06-10T21:30:11.411231+0000","last_fullsized":"2023-06-10T21:30:11.411231+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"262'10757","reported_seq":11414,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:29.036028+0000","last_change":"2023-06-10T21:28:59.476469+0000","last_active":"2023-06-10T21:30:29.036028+0000","last_peered":"2023-06-10T21:30:29.036028+0000","last_clean":"2023-06-10T21:28:56.852537+0000","last_became_active":"2023-06-10T21:28:59.439769+0000","last_became_peered":"2023-06-10T21:28:59.439769+0000","last_unstale":"2023-06-10T21:30:29.036028+0000","last_undegraded":"2023-06-10T21:28:58.939041+0000","last_fullsized":"2023-06-10T21:28:58.883972+0000","mapping_epoch":251,"log_start":"200'700","ondisk_log_start":"200'700","created":194,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:10:03.064078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35323904,"num_objects":539,"num_object_clones":0,"num_object_copies":1617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10757,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":267,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2403},{"shards":"2,4,6","objects":539}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":212,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216207+0000","last_change":"2023-06-10T21:28:57.899002+0000","last_active":"2023-06-10T21:30:17.216207+0000","last_peered":"2023-06-10T21:30:17.216207+0000","last_clean":"2023-06-10T21:30:17.216207+0000","last_became_active":"2023-06-10T21:28:57.898639+0000","last_became_peered":"2023-06-10T21:28:57.898639+0000","last_unstale":"2023-06-10T21:30:17.216207+0000","last_undegraded":"2023-06-10T21:30:17.216207+0000","last_fullsized":"2023-06-10T21:30:17.216207+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:12:01.141117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":35,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216216+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:30:17.216216+0000","last_peered":"2023-06-10T21:30:17.216216+0000","last_clean":"2023-06-10T21:30:17.216216+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:30:17.216216+0000","last_undegraded":"2023-06-10T21:30:17.216216+0000","last_fullsized":"2023-06-10T21:30:17.216216+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":201,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197527+0000","last_change":"2023-06-10T21:28:57.895650+0000","last_active":"2023-06-10T21:30:17.197527+0000","last_peered":"2023-06-10T21:30:17.197527+0000","last_clean":"2023-06-10T21:30:17.197527+0000","last_became_active":"2023-06-10T21:28:57.895320+0000","last_became_peered":"2023-06-10T21:28:57.895320+0000","last_unstale":"2023-06-10T21:30:17.197527+0000","last_undegraded":"2023-06-10T21:30:17.197527+0000","last_fullsized":"2023-06-10T21:30:17.197527+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:35:20.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"262'10593","reported_seq":17382,"reported_epoch":262,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:29.165396+0000","last_change":"2023-06-10T21:28:59.500885+0000","last_active":"2023-06-10T21:30:29.165396+0000","last_peered":"2023-06-10T21:30:29.165396+0000","last_clean":"2023-06-10T21:28:56.851941+0000","last_became_active":"2023-06-10T21:28:59.444754+0000","last_became_peered":"2023-06-10T21:28:59.444754+0000","last_unstale":"2023-06-10T21:30:29.165396+0000","last_undegraded":"2023-06-10T21:28:59.081148+0000","last_fullsized":"2023-06-10T21:28:58.879700+0000","mapping_epoch":251,"log_start":"200'500","ondisk_log_start":"200'500","created":194,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:59.029499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32833536,"num_objects":501,"num_object_clones":0,"num_object_copies":1503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4601,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10593,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3339,"num_bytes_recovered":218628096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1710},{"shards":"2,4,6","objects":501}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":253,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.186481+0000","last_change":"2023-06-10T21:28:57.899350+0000","last_active":"2023-06-10T21:30:17.186481+0000","last_peered":"2023-06-10T21:30:17.186481+0000","last_clean":"2023-06-10T21:30:17.186481+0000","last_became_active":"2023-06-10T21:28:57.898711+0000","last_became_peered":"2023-06-10T21:28:57.898711+0000","last_unstale":"2023-06-10T21:30:17.186481+0000","last_undegraded":"2023-06-10T21:30:17.186481+0000","last_fullsized":"2023-06-10T21:30:17.186481+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":87,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.412346+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:30:11.412346+0000","last_peered":"2023-06-10T21:30:11.412346+0000","last_clean":"2023-06-10T21:30:11.412346+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:30:11.412346+0000","last_undegraded":"2023-06-10T21:30:11.412346+0000","last_fullsized":"2023-06-10T21:30:11.412346+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":197,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.409878+0000","last_change":"2023-06-10T21:28:34.521765+0000","last_active":"2023-06-10T21:30:11.409878+0000","last_peered":"2023-06-10T21:30:11.409878+0000","last_clean":"2023-06-10T21:30:11.409878+0000","last_became_active":"2023-06-10T21:28:34.521409+0000","last_became_peered":"2023-06-10T21:28:34.521409+0000","last_unstale":"2023-06-10T21:30:11.409878+0000","last_undegraded":"2023-06-10T21:30:11.409878+0000","last_fullsized":"2023-06-10T21:30:11.409878+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:05:44.771841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":83,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.411105+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:30:11.411105+0000","last_peered":"2023-06-10T21:30:11.411105+0000","last_clean":"2023-06-10T21:30:11.411105+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:30:11.411105+0000","last_undegraded":"2023-06-10T21:30:11.411105+0000","last_fullsized":"2023-06-10T21:30:11.411105+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.405009+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:30:11.405009+0000","last_peered":"2023-06-10T21:30:11.405009+0000","last_clean":"2023-06-10T21:30:11.405009+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:30:11.405009+0000","last_undegraded":"2023-06-10T21:30:11.405009+0000","last_fullsized":"2023-06-10T21:30:11.405009+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.185856+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:30:17.185856+0000","last_peered":"2023-06-10T21:30:17.185856+0000","last_clean":"2023-06-10T21:30:17.185856+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:30:17.185856+0000","last_undegraded":"2023-06-10T21:30:17.185856+0000","last_fullsized":"2023-06-10T21:30:17.185856+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":134,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197703+0000","last_change":"2023-06-10T21:28:57.898684+0000","last_active":"2023-06-10T21:30:17.197703+0000","last_peered":"2023-06-10T21:30:17.197703+0000","last_clean":"2023-06-10T21:30:17.197703+0000","last_became_active":"2023-06-10T21:28:57.898387+0000","last_became_peered":"2023-06-10T21:28:57.898387+0000","last_unstale":"2023-06-10T21:30:17.197703+0000","last_undegraded":"2023-06-10T21:30:17.197703+0000","last_fullsized":"2023-06-10T21:30:17.197703+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:55:36.556314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":39,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215863+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:30:17.215863+0000","last_peered":"2023-06-10T21:30:17.215863+0000","last_clean":"2023-06-10T21:30:17.215863+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:30:17.215863+0000","last_undegraded":"2023-06-10T21:30:17.215863+0000","last_fullsized":"2023-06-10T21:30:17.215863+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":199,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.215882+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:30:17.215882+0000","last_peered":"2023-06-10T21:30:17.215882+0000","last_clean":"2023-06-10T21:30:17.215882+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:30:17.215882+0000","last_undegraded":"2023-06-10T21:30:17.215882+0000","last_fullsized":"2023-06-10T21:30:17.215882+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":244,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.216063+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:30:17.216063+0000","last_peered":"2023-06-10T21:30:17.216063+0000","last_clean":"2023-06-10T21:30:17.216063+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:30:17.216063+0000","last_undegraded":"2023-06-10T21:30:17.216063+0000","last_fullsized":"2023-06-10T21:30:17.216063+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":248,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197632+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:30:17.197632+0000","last_peered":"2023-06-10T21:30:17.197632+0000","last_clean":"2023-06-10T21:30:17.197632+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:30:17.197632+0000","last_undegraded":"2023-06-10T21:30:17.197632+0000","last_fullsized":"2023-06-10T21:30:17.197632+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":148,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-06-10T21:30:11.410797+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:30:11.410797+0000","last_peered":"2023-06-10T21:30:11.410797+0000","last_clean":"2023-06-10T21:30:11.410797+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:30:11.410797+0000","last_undegraded":"2023-06-10T21:30:11.410797+0000","last_fullsized":"2023-06-10T21:30:11.410797+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":35,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-06-10T21:30:17.197758+0000","last_change":"2023-06-10T21:28:57.894606+0000","last_active":"2023-06-10T21:30:17.197758+0000","last_peered":"2023-06-10T21:30:17.197758+0000","last_clean":"2023-06-10T21:30:17.197758+0000","last_became_active":"2023-06-10T21:28:57.894004+0000","last_became_peered":"2023-06-10T21:28:57.894004+0000","last_unstale":"2023-06-10T21:30:17.197758+0000","last_undegraded":"2023-06-10T21:30:17.197758+0000","last_fullsized":"2023-06-10T21:30:17.197758+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:28:45.258464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":559808536,"num_objects":8543,"num_object_clones":0,"num_object_copies":25629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8543,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172965,"num_write_kb":5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20623,"num_bytes_recovered":1321598976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3212259328,"data_stored":3212247112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126763,"ondisk_log_size":126763,"up":48,"acting":40,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":6076,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430601,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685980,"kb_used_data":164628,"kb_used_omap":0,"kb_used_meta":521344,"kb_avail":93059172,"statfs":{"total":95995035648,"available":95292592128,"internally_reserved":0,"allocated":168579072,"data_stored":168033467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533856256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:35 2023","interfaces":[{"interface":"back","average":{"1min":9.7080000000000002,"5min":9.7080000000000002,"15min":9.7080000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":89.245000000000005,"5min":89.245000000000005,"15min":89.245000000000005},"last":0.97499999999999998},{"interface":"front","average":{"1min":10.050000000000001,"5min":10.050000000000001,"15min":10.050000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":90.027000000000001,"5min":90.027000000000001,"15min":90.027000000000001},"last":3.9180000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.2069999999999999,"15min":2.8370000000000002},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0629999999999997,"5min":93.093999999999994,"15min":130.19399999999999},"last":1.9870000000000001},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.796,"15min":3.1589999999999998},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.673,"5min":93.119,"15min":212.20599999999999},"last":1.127}]},{"osd":2,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6040000000000001,"15min":2.3929999999999998},"min":{"1min":0.377,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8129999999999997,"5min":160.99299999999999,"15min":160.99299999999999},"last":2.5499999999999998},{"interface":"front","average":{"1min":1.998,"5min":4.327,"15min":3.0030000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0529999999999999,"5min":160.53700000000001,"15min":212.13399999999999},"last":1.2090000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:29:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.5,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.2490000000000006,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":1.236},{"interface":"front","average":{"1min":1.833,"5min":1.833,"15min":1.833},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.242,"5min":4.242,"15min":4.242},"last":3.3380000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.74299999999999999,"15min":0.97299999999999998},"min":{"1min":0.25900000000000001,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":3.9199999999999999,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.95299999999999996,"15min":1.363},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":9.0470000000000006,"5min":14.657999999999999,"15min":14.657999999999999},"last":0.315}]},{"osd":5,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.1000000000000001,"15min":2.1560000000000001},"min":{"1min":0.33400000000000002,"5min":0.27400000000000002,"15min":0.27100000000000002},"max":{"1min":2.1160000000000001,"5min":12.048,"15min":211.99799999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.92400000000000004,"15min":1.9670000000000001},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.7,"5min":14.988,"15min":212.23699999999999},"last":0.39200000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.0589999999999999,"15min":1.2989999999999999},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1680000000000001,"5min":13.073,"15min":13.073},"last":0.435},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.71699999999999997,"15min":1.111},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.724,"5min":3.8239999999999998,"15min":10.874000000000001},"last":0.371}]}]},{"osd":6,"up_from":0,"seq":180388626829,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566272,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565648,"kb_used_data":125256,"kb_used_omap":4,"kb_used_meta":440379,"kb_avail":93179504,"statfs":{"total":95995035648,"available":95415812096,"internally_reserved":0,"allocated":128262144,"data_stored":127442177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":450949006},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:34 2023","interfaces":[{"interface":"back","average":{"1min":13.73,"5min":13.73,"15min":13.73},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":107.825,"5min":107.825,"15min":107.825},"last":0.58199999999999996},{"interface":"front","average":{"1min":13.321,"5min":13.321,"15min":13.321},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":110.928,"5min":110.928,"15min":110.928},"last":0.68300000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":9.843,"5min":5.5069999999999997,"15min":2.8380000000000001},"min":{"1min":0.39100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":107.39700000000001,"5min":107.857,"15min":107.857},"last":1.9750000000000001},{"interface":"front","average":{"1min":11.939,"5min":5.8049999999999997,"15min":2.637},"min":{"1min":0.61099999999999999,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":106.907,"5min":107.958,"15min":107.958},"last":1.1859999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.976,"5min":1.976,"15min":1.976},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.445,"5min":11.445,"15min":11.445},"last":4.8399999999999999},{"interface":"front","average":{"1min":3.1819999999999999,"5min":3.1819999999999999,"15min":3.1819999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":10.856,"5min":10.856,"15min":10.856},"last":0.65200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3839999999999999,"15min":1.71},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":16.059999999999999,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.641,"5min":1.03,"15min":1.5920000000000001},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.8949999999999996,"5min":6.8949999999999996,"15min":14.460000000000001},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.4490000000000001,"15min":4.2670000000000003},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":5.0700000000000003,"5min":6.9550000000000001,"15min":383.83100000000002},"last":1.9490000000000001},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.389,"15min":4.1070000000000002},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.29699999999999999},"max":{"1min":16.146999999999998,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.387,"5min":1.905,"15min":2.1589999999999998},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.33800000000000002},"max":{"1min":6.6859999999999999,"5min":13.465,"15min":14.858000000000001},"last":1.913},{"interface":"front","average":{"1min":2,"5min":1.7270000000000001,"15min":1.956},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":11.476000000000001,"5min":13.48,"15min":13.48},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.6839999999999999,"15min":2.7810000000000001},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.32600000000000001},"max":{"1min":16.113,"5min":16.113,"15min":162.399},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.3280000000000001,"15min":2.367},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9459999999999997,"5min":6.9459999999999997,"15min":162.27199999999999},"last":0.46899999999999997}]}]},{"osd":0,"up_from":244,"seq":1047972020248,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522788,"kb_used_data":369308,"kb_used_omap":10,"kb_used_meta":153461,"kb_avail":93222364,"statfs":{"total":95995035648,"available":95459700736,"internally_reserved":0,"allocated":378171392,"data_stored":377264728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":157144116},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.4100000000000001,"5min":4.4100000000000001,"15min":4.4100000000000001},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":19.806999999999999,"5min":19.806999999999999,"15min":19.806999999999999},"last":3.7999999999999998},{"interface":"front","average":{"1min":6.0620000000000003,"5min":6.0620000000000003,"15min":6.0620000000000003},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":29.841000000000001,"5min":29.841000000000001,"15min":29.841000000000001},"last":11.295}]},{"osd":2,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.952,"5min":5.952,"15min":5.952},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":14.051,"5min":14.051,"15min":14.051},"last":2.996},{"interface":"front","average":{"1min":5.1059999999999999,"5min":5.1059999999999999,"15min":5.1059999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":19.882999999999999,"5min":19.882999999999999,"15min":19.882999999999999},"last":4.0460000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":6.2869999999999999,"5min":6.2869999999999999,"15min":6.2869999999999999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":17.088000000000001,"5min":17.088000000000001,"15min":17.088000000000001},"last":4.492},{"interface":"front","average":{"1min":7.2839999999999998,"5min":7.2839999999999998,"15min":7.2839999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":32.822000000000003,"5min":32.822000000000003,"15min":32.822000000000003},"last":4.407}]},{"osd":4,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.0970000000000004,"5min":5.0970000000000004,"15min":5.0970000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":29.709,"5min":29.709,"15min":29.709},"last":10.135999999999999},{"interface":"front","average":{"1min":4.2750000000000004,"5min":4.2750000000000004,"15min":4.2750000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":13.872,"5min":13.872,"15min":13.872},"last":2.1930000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.3860000000000001,"5min":4.3860000000000001,"15min":4.3860000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":14.105,"5min":14.105,"15min":14.105},"last":2.2389999999999999},{"interface":"front","average":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":19.846,"5min":19.846,"15min":19.846},"last":4.8890000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":5.3170000000000002,"5min":5.3170000000000002,"15min":5.3170000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":29.806000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":10.18},{"interface":"front","average":{"1min":4.423,"5min":4.423,"15min":4.423},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":13.82,"5min":13.82,"15min":13.82},"last":2.2749999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:29:36 2023","interfaces":[{"interface":"back","average":{"1min":4.0880000000000001,"5min":4.0880000000000001,"15min":4.0880000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":19.866,"5min":19.866,"15min":19.866},"last":5.298},{"interface":"front","average":{"1min":5.3150000000000004,"5min":5.3150000000000004,"15min":5.3150000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":29.77,"5min":29.77,"15min":29.77},"last":10.218999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075660,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":593088,"kb_used_data":281976,"kb_used_omap":7,"kb_used_meta":310968,"kb_avail":93152064,"statfs":{"total":95995035648,"available":95387713536,"internally_reserved":0,"allocated":288743424,"data_stored":287931963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":318431313},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:35 2023","interfaces":[{"interface":"back","average":{"1min":4.7039999999999997,"5min":4.7039999999999997,"15min":4.7039999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":14.304,"5min":14.304,"15min":14.304},"last":2.7770000000000001},{"interface":"front","average":{"1min":5.3920000000000003,"5min":5.3920000000000003,"15min":5.3920000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":20.792999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":1.401}]},{"osd":1,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":5.0439999999999996,"5min":2.3580000000000001,"15min":1.71},"min":{"1min":0.35699999999999998,"5min":0.19,"15min":0.19},"max":{"1min":10.736000000000001,"5min":11.6,"15min":29.486000000000001},"last":1.9099999999999999},{"interface":"front","average":{"1min":3.5430000000000001,"5min":1.776,"15min":1.488},"min":{"1min":0.28899999999999998,"5min":0.214,"15min":0.214},"max":{"1min":11.336,"5min":14.334,"15min":28.013999999999999},"last":2.8370000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:29:57 2023","interfaces":[{"interface":"back","average":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":5.2409999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":14.272,"5min":14.272,"15min":14.272},"last":0.52400000000000002},{"interface":"front","average":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":16.411000000000001,"5min":16.411000000000001,"15min":16.411000000000001},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.044,"5min":5.5250000000000004,"15min":2.6320000000000001},"min":{"1min":0.314,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.0969999999999995,"5min":224.00999999999999,"15min":224.00999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":4.6299999999999999,"5min":5.7709999999999999,"15min":2.7799999999999998},"min":{"1min":0.377,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":17.535,"5min":223.28399999999999,"15min":223.28399999999999},"last":1.5960000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":1.595,"15min":1.339},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.473000000000001,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":3.0609999999999999,"5min":1.788,"15min":1.542},"min":{"1min":0.442,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.1989999999999998,"5min":16.125,"15min":16.125},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0830000000000002,"5min":2.077,"15min":1.599},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.1690000000000005,"5min":16.183,"15min":16.183},"last":0.44900000000000001},{"interface":"front","average":{"1min":4.3399999999999999,"5min":2.306,"15min":1.782},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":10.625999999999999,"5min":11.029,"15min":17.244},"last":1.6519999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":3.02,"15min":1.7330000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.44,"5min":113.913,"15min":113.913},"last":0.55700000000000005},{"interface":"front","average":{"1min":3.0609999999999999,"5min":3.3149999999999999,"15min":2.036},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.141,"5min":113.937,"15min":113.937},"last":0.47199999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824020,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902364,"kb_used_data":784532,"kb_used_omap":7,"kb_used_meta":117816,"kb_avail":92842788,"statfs":{"total":95995035648,"available":95071014912,"internally_reserved":0,"allocated":803360768,"data_stored":802469088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":120643603},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":5.5490000000000004,"15min":5.5490000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":56.828000000000003,"5min":56.828000000000003,"15min":56.828000000000003},"last":0.84799999999999998},{"interface":"front","average":{"1min":5.3330000000000002,"5min":5.3330000000000002,"15min":5.3330000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":57.600000000000001,"5min":57.600000000000001,"15min":57.600000000000001},"last":0.83699999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":16.925999999999998,"5min":16.925999999999998,"15min":16.925999999999998},"last":0.96599999999999997},{"interface":"front","average":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":3.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.2579999999999991,"5min":8.2579999999999991,"15min":8.2579999999999991},"last":0.874}]},{"osd":2,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":12.922000000000001,"5min":12.922000000000001,"15min":12.922000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":87.555999999999997,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.36199999999999999},{"interface":"front","average":{"1min":14.343,"5min":14.343,"15min":14.343},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":80.566999999999993,"5min":80.566999999999993,"15min":80.566999999999993},"last":0.94099999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.7690000000000001,"5min":7.7690000000000001,"15min":7.7690000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7149999999999999,"5min":8.7149999999999999,"15min":8.7149999999999999},"last":0.85999999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.393,"15min":1.393},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.2480000000000002,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":7.8319999999999999,"5min":7.8319999999999999,"15min":7.8319999999999999},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":8.6370000000000005,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3449999999999998,"5min":6.3449999999999998,"15min":6.3449999999999998},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.778,"5min":2.778,"15min":2.778},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.6809999999999992,"5min":8.6809999999999992,"15min":8.6809999999999992},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.3029999999999999,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.64400000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019285,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1008984,"kb_used_data":449168,"kb_used_omap":0,"kb_used_meta":559808,"kb_avail":92736168,"statfs":{"total":95995035648,"available":94961836032,"internally_reserved":0,"allocated":459948032,"data_stored":459404995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":573243392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:29:33 2023","interfaces":[{"interface":"back","average":{"1min":10.678000000000001,"5min":10.678000000000001,"15min":10.678000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":144.22399999999999,"5min":144.22399999999999,"15min":144.22399999999999},"last":2.4830000000000001},{"interface":"front","average":{"1min":9.9399999999999995,"5min":9.9399999999999995,"15min":9.9399999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":144.303,"5min":144.303,"15min":144.303},"last":1.8029999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.496,"15min":2.9900000000000002},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6,"5min":15.063000000000001,"15min":182.035},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.718,"15min":2.8940000000000001},"min":{"1min":0.34799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.4760000000000009,"5min":15.052,"15min":181.73099999999999},"last":7.5570000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.0369999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.319000000000001,"5min":22.614000000000001,"15min":22.614000000000001},"last":6.8979999999999997},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.748,"15min":1.377},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":18.329999999999998,"5min":22.326000000000001,"15min":22.326000000000001},"last":18.329999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.3919999999999999,"15min":2.3919999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.2379999999999995,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":7.8529999999999998},{"interface":"front","average":{"1min":2.044,"5min":2.044,"15min":2.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9640000000000004,"5min":5.9640000000000004,"15min":5.9640000000000004},"last":3.3279999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.76200000000000001,"15min":1.1399999999999999},"min":{"1min":0.30299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.80000000000000004,"5min":6.742,"15min":13.941000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.88900000000000001,"15min":1.2070000000000001},"min":{"1min":0.311,"5min":0.247,"15min":0.247},"max":{"1min":0.85199999999999998,"5min":6.0819999999999999,"15min":11.904999999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.149,"15min":1.2070000000000001},"min":{"1min":0.317,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.077,"5min":7.5090000000000003,"15min":8.2599999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.3340000000000001,"15min":1.28},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":6.8300000000000001,"5min":13.845000000000001,"15min":13.845000000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.206,"15min":1.448},"min":{"1min":0.32900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.7689999999999999,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.0980000000000001,"15min":1.3180000000000001},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.052,"5min":8.9900000000000002,"15min":9.5060000000000002},"last":0.44}]}]},{"osd":5,"up_from":0,"seq":154618823057,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":362745856,"data_stored":362741784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":123731968,"data_stored":123731968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":275251200,"data_stored":275251200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":799277056,"data_stored":799277056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":431951872,"data_stored":431947800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3473408,"data_stored":3473408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1051856896,"data_stored":1051852824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":163971072,"data_stored":163971072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:30:35.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:30:35.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:35.778 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:35.826 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:35.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:35.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:35.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:35.971 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:36.011 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:36.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:36.045 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:36.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:36.144 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:36.193 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:36.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:36.300 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:36.357 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:36.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/22779 objects degraded (154.006%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:36.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/22779 objects degraded (154.006%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:36.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/22779 objects degraded (154.006%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:36.882 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:36.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:36.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:36.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:37.029 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:37.078 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:37.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:37.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:37.303 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:37 smithi152 ceph-mon[156819]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:37.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:37 smithi064 ceph-mon[180297]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:37 smithi064 ceph-mon[186706]: pgmap v940: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:38.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:38.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:38.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:38.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:38.590 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:38.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:38.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:38.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:39.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:39.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:39.099 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:39.134 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:39.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:39.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:39.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:39 smithi064 ceph-mon[186706]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:39.251 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:30:39.253 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:39.279 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:39.324 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:30:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:39.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:30:39.433 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:39.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:39 smithi152 ceph-mon[156819]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:39.463 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:39.511 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:39.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:39.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:39.679 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:39 smithi064 ceph-mon[180297]: pgmap v941: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 392 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 478 op/s; 35081/18858 objects degraded (186.027%); 0 B/s, 35 objects/s recovering 2023-06-10T21:30:39.724 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:40.117 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:30:40.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:30:40.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:40.249 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:40.306 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:40.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:40.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:40.436 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:30:40.477 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:30:40.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:40.574 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:30:40.661 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:30:40.693 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:30:40.744 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:30:40.789 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:30:40.831 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:30:40.840 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.841 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:30:40.842 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:30:41.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[180297]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 31 objects/s recovering 2023-06-10T21:30:41.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2782201965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:41.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/11844 objects degraded (296.192%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:41.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3386891582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[186706]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 31 objects/s recovering 2023-06-10T21:30:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2782201965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/11844 objects degraded (296.192%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3386891582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:41 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:41 smithi152 ceph-mon[156819]: pgmap v942: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 31 objects/s recovering 2023-06-10T21:30:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2782201965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/11844 objects degraded (296.192%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3386891582' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:41 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:30:41.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:41.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:41.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:41.812 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:30:41.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:41.929 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:42.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:42.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:30:42.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[180297]: osdmap e264: 8 total, 8 up, 6 in 2023-06-10T21:30:42.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[180297]: failed to encode map e264 with expected crc 2023-06-10T21:30:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:30:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[186706]: osdmap e264: 8 total, 8 up, 6 in 2023-06-10T21:30:42.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:42 smithi064 ceph-mon[186706]: failed to encode map e264 with expected crc 2023-06-10T21:30:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:42 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:42 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:30:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:42 smithi152 ceph-mon[156819]: osdmap e264: 8 total, 8 up, 6 in 2023-06-10T21:30:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:42 smithi152 ceph-mon[156819]: failed to encode map e264 with expected crc 2023-06-10T21:30:42.738 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:42.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:42.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:42.943 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:43.137 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:43.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:42] "GET /metrics HTTP/1.1" 200 223717 "" "Prometheus/2.33.4" 2023-06-10T21:30:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:43.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:43.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:43.317 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:43.372 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:43.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:43.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[180297]: pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 26 objects/s recovering 2023-06-10T21:30:43.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[180297]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[180297]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[180297]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[186706]: pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 26 objects/s recovering 2023-06-10T21:30:43.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[186706]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[186706]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:43 smithi064 ceph-mon[186706]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:43.513 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:43.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:43.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:43.570 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:43 smithi152 ceph-mon[156819]: pgmap v943: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 246 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 35081/11844 objects degraded (296.192%); 0 B/s, 26 objects/s recovering 2023-06-10T21:30:43.570 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:43 smithi152 ceph-mon[156819]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.571 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:43 smithi152 ceph-mon[156819]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.571 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:43 smithi152 ceph-mon[156819]: failed to encode map e264 with expected crc 2023-06-10T21:30:43.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:43.700 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:43.743 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:43.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:43.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:43.900 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:44.714 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:44.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:44.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:30:44.858 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:44.909 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:30:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:44.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:30:45.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:45.051 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:45.099 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:45.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:45.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:30:45.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:45.331 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:45.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:45 smithi064 ceph-mon[180297]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 181 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 35081/8730 objects degraded (401.844%); 0 B/s, 32 objects/s recovering 2023-06-10T21:30:45.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:45 smithi064 ceph-mon[186706]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 181 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 35081/8730 objects degraded (401.844%); 0 B/s, 32 objects/s recovering 2023-06-10T21:30:45.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:45.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:45.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:45.545 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:30:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:30:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:30:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:30:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13232 10.9 10:9012193c:::benchmark_data_smithi064_312713_object13231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e201)", 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 300.27071077199997, 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7512189839999999, 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13232 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:30:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264129+0000", 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264126+0000", 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:30:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264155+0000", 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8109000000000001e-05 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264178+0000", 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3536999999999998e-05 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264189+0000", 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0995000000000001e-05 2023-06-10T21:30:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264267+0000", 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7470000000000002e-05 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.264279+0000", 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2045000000000001e-05 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614347+0000", 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35006879200000002 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614373+0000", 2023-06-10T21:30:45.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5544000000000001e-05 2023-06-10T21:30:45.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.003258+0000", 2023-06-10T21:30:45.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3888854589999999 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.004667+0000", 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014090960000000001 2023-06-10T21:30:45.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015232+0000", 2023-06-10T21:30:45.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010564163999999999 2023-06-10T21:30:45.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015348+0000", 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115995 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13262 10.9 10:9b8ca29e:::benchmark_data_smithi064_312713_object13261:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 300.25339011800003, 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7392644879999999, 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.562 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13262 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281449+0000", 2023-06-10T21:30:45.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281448+0000", 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281480+0000", 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2228999999999999e-05 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281490+0000", 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8079999999999999e-06 2023-06-10T21:30:45.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281496+0000", 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7790000000000001e-06 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281565+0000", 2023-06-10T21:30:45.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8689999999999995e-05 2023-06-10T21:30:45.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.281579+0000", 2023-06-10T21:30:45.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4499e-05 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614432+0000", 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33285274500000001 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614439+0000", 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.007692+0000", 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.393252581 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.009132+0000", 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014407669999999999 2023-06-10T21:30:45.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.011401+0000", 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002268927 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015111+0000", 2023-06-10T21:30:45.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037098830000000002 2023-06-10T21:30:45.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015417+0000", 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030584399999999999 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020055+0000", 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046377069999999996 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020100+0000", 2023-06-10T21:30:45.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5522999999999998e-05 2023-06-10T21:30:45.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020714+0000", 2023-06-10T21:30:45.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00061340799999999997 2023-06-10T21:30:45.578 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.578 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.578 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.580 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:13275 10.9 10:9047ac4e:::benchmark_data_smithi064_312713_object13274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:30:45.580 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:30:45.580 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 300.245498951, 2023-06-10T21:30:45.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7361003699999999, 2023-06-10T21:30:45.581 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.581 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.582 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.582 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 13275 2023-06-10T21:30:45.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.582 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:30:45.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289340+0000", 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289339+0000", 2023-06-10T21:30:45.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:30:45.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289384+0000", 2023-06-10T21:30:45.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5176000000000003e-05 2023-06-10T21:30:45.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289404+0000", 2023-06-10T21:30:45.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0273999999999999e-05 2023-06-10T21:30:45.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289414+0000", 2023-06-10T21:30:45.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028e-05 2023-06-10T21:30:45.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289471+0000", 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6205999999999997e-05 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.289480+0000", 2023-06-10T21:30:45.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2690000000000005e-06 2023-06-10T21:30:45.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614466+0000", 2023-06-10T21:30:45.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32498670400000002 2023-06-10T21:30:45.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:45.614474+0000", 2023-06-10T21:30:45.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8569999999999995e-06 2023-06-10T21:30:45.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015439+0000", 2023-06-10T21:30:45.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.400964885 2023-06-10T21:30:45.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015551+0000", 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111486 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,7", 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.015728+0000", 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000177412 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020768+0000", 2023-06-10T21:30:45.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050401889999999996 2023-06-10T21:30:45.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.020809+0000", 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1056000000000001e-05 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024793+0000", 2023-06-10T21:30:45.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003983444 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.024844+0000", 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1477000000000002e-05 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:25:47.025441+0000", 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00059653199999999999 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54555 10.7 10:e3cbd15b:::benchmark_data_smithi064_312713_object54554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.73019936999998, 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1739721809999999, 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.603 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54555 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804640+0000", 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804638+0000", 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804665+0000", 2023-06-10T21:30:45.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6137000000000001e-05 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804675+0000", 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0899e-05 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804691+0000", 2023-06-10T21:30:45.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5661000000000001e-05 2023-06-10T21:30:45.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804758+0000", 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6879999999999997e-05 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804769+0000", 2023-06-10T21:30:45.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1028e-05 2023-06-10T21:30:45.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818008+0000", 2023-06-10T21:30:45.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0132391670000001 2023-06-10T21:30:45.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818017+0000", 2023-06-10T21:30:45.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3829999999999992e-06 2023-06-10T21:30:45.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.955590+0000", 2023-06-10T21:30:45.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375735200000001 2023-06-10T21:30:45.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.618 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.956353+0000", 2023-06-10T21:30:45.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000762807 2023-06-10T21:30:45.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978548+0000", 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022194875999999999 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978612+0000", 2023-06-10T21:30:45.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4449999999999994e-05 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.622 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54557 10.f 10:f1e22a0f:::benchmark_data_smithi064_312713_object54556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e219)", 2023-06-10T21:30:45.624 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:30:45.624 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.73011429899998, 2023-06-10T21:30:45.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1819947270000002, 2023-06-10T21:30:45.624 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.625 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.625 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.626 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.626 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54557 2023-06-10T21:30:45.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804725+0000", 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804724+0000", 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804750+0000", 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5415999999999999e-05 2023-06-10T21:30:45.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804759+0000", 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5969999999999998e-06 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804767+0000", 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804793+0000", 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5993e-05 2023-06-10T21:30:45.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.804800+0000", 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1019999999999999e-06 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818039+0000", 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.013239368 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818047+0000", 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5510000000000002e-06 2023-06-10T21:30:45.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.977925+0000", 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.159878645 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978506+0000", 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00058025500000000001 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986600+0000", 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080940130000000006 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986720+0000", 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120254 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54569 10.3 10:c761f176:::benchmark_data_smithi064_312713_object54568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.729298056, 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0199541459999999, 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.637 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54569 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805541+0000", 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805540+0000", 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805563+0000", 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2124e-05 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805572+0000", 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2550000000000003e-06 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805580+0000", 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7929999999999998e-06 2023-06-10T21:30:45.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805621+0000", 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1746999999999999e-05 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.805631+0000", 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3880000000000006e-06 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822642+0000", 2023-06-10T21:30:45.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0170114130000001 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822728+0000", 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6032999999999994e-05 2023-06-10T21:30:45.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.822844+0000", 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011603300000000001 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.824327+0000", 2023-06-10T21:30:45.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014828739999999999 2023-06-10T21:30:45.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825354+0000", 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001026875 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825416+0000", 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1940999999999995e-05 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.825496+0000", 2023-06-10T21:30:45.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9591000000000004e-05 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.650 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54574 10.5 10:a853286f:::benchmark_data_smithi064_312713_object54573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.72846728299999, 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0150581569999999, 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.651 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54574 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806372+0000", 2023-06-10T21:30:45.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806370+0000", 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806397+0000", 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6996999999999999e-05 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806413+0000", 2023-06-10T21:30:45.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6099999999999998e-05 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.806422+0000", 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.833e-06 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809434+0000", 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030114529999999999 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.809442+0000", 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1799999999999996e-06 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818062+0000", 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0086199140000001 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818198+0000", 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013567700000000001 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0", 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818376+0000", 2023-06-10T21:30:45.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000178813 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.820582+0000", 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022051589999999999 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821290+0000", 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070819899999999996 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821344+0000", 2023-06-10T21:30:45.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4304e-05 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.821430+0000", 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6263999999999999e-05 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.662 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54575 10.e 10:708b6ab1:::benchmark_data_smithi064_312713_object54574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.71911316199999, 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0012291179999999, 2023-06-10T21:30:45.663 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54575 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815726+0000", 2023-06-10T21:30:45.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815725+0000", 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815752+0000", 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7001000000000001e-05 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815769+0000", 2023-06-10T21:30:45.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7203000000000001e-05 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.815779+0000", 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6020000000000006e-06 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820630+0000", 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048513219999999999 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820642+0000", 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2081e-05 2023-06-10T21:30:45.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817919+0000", 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99727731900000005 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817927+0000", 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7549999999999994e-06 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810577+0000", 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99265023100000005 2023-06-10T21:30:45.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810664+0000", 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6879999999999995e-05 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.810771+0000", 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106632 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816578+0000", 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058069870000000004 2023-06-10T21:30:45.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816627+0000", 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8791000000000001e-05 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816709+0000", 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2793000000000005e-05 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816771+0000", 2023-06-10T21:30:45.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1524000000000003e-05 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816955+0000", 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018453799999999999 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.676 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54576 10.f 10:f052f063:::benchmark_data_smithi064_312713_object54575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.71402012300001, 2023-06-10T21:30:45.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1709000860000001, 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54576 2023-06-10T21:30:45.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820819+0000", 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820817+0000", 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820849+0000", 2023-06-10T21:30:45.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1776000000000002e-05 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820866+0000", 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6892999999999999e-05 2023-06-10T21:30:45.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820875+0000", 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5739999999999996e-06 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820944+0000", 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9340999999999999e-05 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.820957+0000", 2023-06-10T21:30:45.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3226e-05 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818068+0000", 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99711057199999997 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.818077+0000", 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7480000000000005e-06 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.978558+0000", 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1604804230000001 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979614+0000", 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001056162 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.979766+0000", 2023-06-10T21:30:45.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015153399999999999 2023-06-10T21:30:45.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.984553+0000", 2023-06-10T21:30:45.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047875230000000001 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.986763+0000", 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022097850000000001 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990555+0000", 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037920319999999999 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.990588+0000", 2023-06-10T21:30:45.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3130999999999998e-05 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.991719+0000", 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001131331 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.692 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54580 10.e 10:70306020:::benchmark_data_smithi064_312713_object54579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e219)", 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 258.70603083999998, 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9886503549999999, 2023-06-10T21:30:45.693 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54580 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.694 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828809+0000", 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828807+0000", 2023-06-10T21:30:45.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828835+0000", 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7953000000000002e-05 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828857+0000", 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2255999999999998e-05 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828867+0000", 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9520000000000006e-06 2023-06-10T21:30:45.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828928+0000", 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1260999999999995e-05 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:26.828938+0000", 2023-06-10T21:30:45.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0195e-05 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817952+0000", 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98901349999999999 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817959+0000", 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3690000000000001e-06 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812558+0000", 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.994598918 2023-06-10T21:30:45.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812679+0000", 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012119499999999999 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.812787+0000", 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010732 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816636+0000", 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038490880000000001 2023-06-10T21:30:45.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817109+0000", 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047353 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817227+0000", 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011818 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817283+0000", 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5247000000000001e-05 2023-06-10T21:30:45.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817459+0000", 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017632800000000001 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:54582 10.e 10:78d4d384:::benchmark_data_smithi064_312713_object54581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e220)", 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 257.74544076299998, 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.028370658, 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.708 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 54582 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:30:45.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789399+0000", 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789397+0000", 2023-06-10T21:30:45.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:30:45.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.789430+0000", 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3229999999999999e-05 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813125+0000", 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023694667999999999 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.813150+0000", 2023-06-10T21:30:45.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5534999999999999e-05 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817984+0000", 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048335940000000001 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:27.817990+0000", 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8869999999999997e-06 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814293+0000", 2023-06-10T21:30:45.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99630294900000005 2023-06-10T21:30:45.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814412+0000", 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011934099999999999 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.814525+0000", 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011297400000000001 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.816642+0000", 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021171509999999998 2023-06-10T21:30:45.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817165+0000", 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052269299999999996 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817520+0000", 2023-06-10T21:30:45.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035547299999999998 2023-06-10T21:30:45.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817583+0000", 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2090999999999999e-05 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:26:28.817769+0000", 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018677799999999999 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.723 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93470 10.f 10:f601cf75:::benchmark_data_smithi064_312713_object2714:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.41753279700001, 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.39052785000000001, 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.724 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93470 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117307+0000", 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117305+0000", 2023-06-10T21:30:45.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117310+0000", 2023-06-10T21:30:45.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2850000000000002e-06 2023-06-10T21:30:45.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117320+0000", 2023-06-10T21:30:45.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2159999999999995e-06 2023-06-10T21:30:45.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.117327+0000", 2023-06-10T21:30:45.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2470000000000002e-06 2023-06-10T21:30:45.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.132977+0000", 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015650496 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.133683+0000", 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070519099999999998 2023-06-10T21:30:45.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166028+0000", 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.032345157999999999 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166124+0000", 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5873999999999997e-05 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.166229+0000", 2023-06-10T21:30:45.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010504499999999999 2023-06-10T21:30:45.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172050+0000", 2023-06-10T21:30:45.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058210919999999999 2023-06-10T21:30:45.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.175754+0000", 2023-06-10T21:30:45.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037038639999999999 2023-06-10T21:30:45.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.506456+0000", 2023-06-10T21:30:45.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33070211799999999 2023-06-10T21:30:45.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.506511+0000", 2023-06-10T21:30:45.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5271e-05 2023-06-10T21:30:45.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507834+0000", 2023-06-10T21:30:45.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013234049999999999 2023-06-10T21:30:45.744 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.745 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.745 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.746 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93479 10.f 10:f21736d6:::benchmark_data_smithi064_312713_object2723:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:30:45.747 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:30:45.747 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.41472252299999, 2023-06-10T21:30:45.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.38120153600000001, 2023-06-10T21:30:45.747 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.748 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.748 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.748 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.749 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.749 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93479 2023-06-10T21:30:45.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.749 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:30:45.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120117+0000", 2023-06-10T21:30:45.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120115+0000", 2023-06-10T21:30:45.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120123+0000", 2023-06-10T21:30:45.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6690000000000006e-06 2023-06-10T21:30:45.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120136+0000", 2023-06-10T21:30:45.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3159000000000001e-05 2023-06-10T21:30:45.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.120146+0000", 2023-06-10T21:30:45.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2490000000000007e-06 2023-06-10T21:30:45.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.133759+0000", 2023-06-10T21:30:45.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013613142999999999 2023-06-10T21:30:45.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.134418+0000", 2023-06-10T21:30:45.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065979000000000005 2023-06-10T21:30:45.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.162952+0000", 2023-06-10T21:30:45.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028533912000000002 2023-06-10T21:30:45.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.163054+0000", 2023-06-10T21:30:45.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101817 2023-06-10T21:30:45.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.163196+0000", 2023-06-10T21:30:45.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014156100000000001 2023-06-10T21:30:45.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.170963+0000", 2023-06-10T21:30:45.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077668759999999998 2023-06-10T21:30:45.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172078+0000", 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001115436 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.369431+0000", 2023-06-10T21:30:45.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19735330300000001 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.369467+0000", 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5976999999999998e-05 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.501318+0000", 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13185106799999999 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93515 10.7 10:e0fa7a28:::benchmark_data_smithi064_312713_object2759:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.364925736, 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35528036099999999, 2023-06-10T21:30:45.775 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.776 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.776 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.776 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.777 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.777 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93515 2023-06-10T21:30:45.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.777 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:30:45.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169914+0000", 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169913+0000", 2023-06-10T21:30:45.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:30:45.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169916+0000", 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7259999999999998e-06 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169922+0000", 2023-06-10T21:30:45.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6049999999999997e-06 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.169927+0000", 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9960000000000001e-06 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.171491+0000", 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001564735 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172025+0000", 2023-06-10T21:30:45.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053375000000000004 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505189+0000", 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.333163863 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505285+0000", 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6084999999999999e-05 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.505417+0000", 2023-06-10T21:30:45.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013150400000000001 2023-06-10T21:30:45.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.514118+0000", 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0087012889999999992 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.518148+0000", 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040298829999999997 2023-06-10T21:30:45.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.523254+0000", 2023-06-10T21:30:45.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051061709999999996 2023-06-10T21:30:45.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.523312+0000", 2023-06-10T21:30:45.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8529000000000002e-05 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.525194+0000", 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018816250000000001 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:93518 10.f 10:f6658ccc:::benchmark_data_smithi064_312713_object2762:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 211.36275394500001, 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.353940329, 2023-06-10T21:30:45.793 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 93518 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.794 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172085+0000", 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172084+0000", 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172090+0000", 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1209999999999998e-06 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172100+0000", 2023-06-10T21:30:45.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0366999999999999e-05 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172107+0000", 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4460000000000004e-06 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172140+0000", 2023-06-10T21:30:45.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2665e-05 2023-06-10T21:30:45.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.172649+0000", 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050871100000000002 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507880+0000", 2023-06-10T21:30:45.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33523090900000002 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.507987+0000", 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010741 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.508127+0000", 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140283 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.514508+0000", 2023-06-10T21:30:45.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063808509999999999 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.518298+0000", 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00378926 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.524434+0000", 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061361949999999997 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.524481+0000", 2023-06-10T21:30:45.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7307e-05 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:14.526026+0000", 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015447740000000001 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:97037 10.f 10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 206.02365127799999, 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35586957400000002, 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.806 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 97037 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.807 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511188+0000", 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:30:45.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511190+0000", 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9340000000000002e-06 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511198+0000", 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0910000000000004e-06 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511203+0000", 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.412e-06 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511240+0000", 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7057000000000001e-05 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:30:45.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.511951+0000", 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00071124399999999998 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839245+0000", 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32729350600000001 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839346+0000", 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010158 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4,6", 2023-06-10T21:30:45.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.839487+0000", 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014055299999999999 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.845225+0000", 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0057377060000000004 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.845297+0000", 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2552000000000006e-05 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.865475+0000", 2023-06-10T21:30:45.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020177835000000002 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.865540+0000", 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5288000000000002e-05 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:19.867058+0000", 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001517432 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111470 10.a 10:51c5c507:::benchmark_data_smithi064_312713_object20714:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.22254170599999, 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0089041089999999, 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111470 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312298+0000", 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312296+0000", 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312303+0000", 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4030000000000004e-06 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312316+0000", 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3351e-05 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312327+0000", 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0559e-05 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.315119+0000", 2023-06-10T21:30:45.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027926600000000002 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.315131+0000", 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2006e-05 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.321118+0000", 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.005986448 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.321202+0000", 2023-06-10T21:30:45.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4065000000000002e-05 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111477 10.8 10:1f95a548:::benchmark_data_smithi064_312713_object20721:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:30:45.827 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.22250508100001, 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0239321720000001, 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.829 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111477 2023-06-10T21:30:45.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.829 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:30:45.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312334+0000", 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312337+0000", 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5159999999999999e-06 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312345+0000", 2023-06-10T21:30:45.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5569999999999998e-06 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312351+0000", 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0390000000000001e-06 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312451+0000", 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9871000000000007e-05 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312462+0000", 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1076e-05 2023-06-10T21:30:45.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330040+0000", 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.017578477 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330149+0000", 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010885199999999999 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336202+0000", 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006053353 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336267+0000", 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4151000000000003e-05 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111482 10.5 10.2e274505 (undecoded) ondisk+write+known_if_redirected e232)", 2023-06-10T21:30:45.838 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.215314366, 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001240895, 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.839 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111482 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319525+0000", 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319524+0000", 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319532+0000", 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1629999999999996e-06 2023-06-10T21:30:45.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319546+0000", 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4496e-05 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319555+0000", 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2e-06 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319609+0000", 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3497e-05 2023-06-10T21:30:45.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.319622+0000", 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325e-05 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.320740+0000", 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001117627 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.320766+0000", 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6193000000000001e-05 2023-06-10T21:30:45.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111485 10.8 10:16819874:::benchmark_data_smithi064_312713_object20729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.205969872, 2023-06-10T21:30:45.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0079469780000001, 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111485 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.848 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328870+0000", 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328868+0000", 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:30:45.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328874+0000", 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3410000000000002e-06 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:30:45.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328886+0000", 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2594e-05 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.328893+0000", 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5359999999999998e-06 2023-06-10T21:30:45.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.339898+0000", 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011005173 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:30:45.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.339912+0000", 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4032999999999999e-05 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330385+0000", 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99047314200000003 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330481+0000", 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6095999999999996e-05 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2023-06-10T21:30:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.330564+0000", 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2441000000000006e-05 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.335813+0000", 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005249592 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336705+0000", 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00089199799999999997 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336749+0000", 2023-06-10T21:30:45.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3569000000000002e-05 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.336817+0000", 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7805000000000006e-05 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:30:45.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:30:45.859 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:30:45.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:45.860 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:45 smithi152 ceph-mon[156819]: pgmap v945: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 181 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 467 op/s; 35081/8730 objects degraded (401.844%); 0 B/s, 32 objects/s recovering 2023-06-10T21:30:45.861 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:30:45.861 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 35081/8730 objects degraded (401.844%), 7 pgs degraded, 8 pgs undersized","count":15},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1318,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":264,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":2910,"data_bytes":189667238,"bytes_used":3498622976,"bytes_avail":572471590912,"bytes_total":575970213888,"degraded_objects":35081,"degraded_total":8730,"degraded_ratio":4.0184421534936998,"recovering_objects_per_sec":32,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":323,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":467},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":24,"modified":"2023-06-10T21:29:57.688996+0000","services":{}},"progress_events":{"07c9c57a-628e-4e92-b056-3138ef282b37":{"message":"Global Recovery Event (4m)\n [=========================...] (remaining: 25s)","progress":0.91011238098144531,"add_to_ceph_s":true},"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:30:45.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:46.528 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:30:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2188987069' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:30:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/8730 objects degraded (401.844%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:46.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2188987069' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:30:46.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/8730 objects degraded (401.844%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2188987069' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:30:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/8730 objects degraded (401.844%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:47.102 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 90753 objects 2023-06-10T21:30:47.103 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :216.719 2023-06-10T21:30:47.206 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:47.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:47.311 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:47.339 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:47.384 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:47.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:47.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:47.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:47.561 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:30:47.587 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:47 smithi152 ceph-mon[156819]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:47.587 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:30:47.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:47 smithi064 ceph-mon[186706]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:47.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:30:47.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:47 smithi064 ceph-mon[180297]: pgmap v946: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:47.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:30:47.806 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:30:47.807 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:30:47.930 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:47.955 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:47.963 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:47.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:48.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:30:48.088 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:30:48.104 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:30:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:48.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:30:48.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:48.305 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:48.409 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:48.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:48.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:30:48.554 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:48.703 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:30:48.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:30:48.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:48.840 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:48.991 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:48.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:48.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:49.095 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:49.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[180297]: pgmap v947: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:49.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[186706]: pgmap v947: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:49.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:49 smithi152 ceph-mon[156819]: pgmap v947: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 98 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 35081/4743 objects degraded (739.637%); 0 B/s, 38 objects/s recovering 2023-06-10T21:30:49.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:49.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:30:50.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:50.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:50.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:50.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:50.430 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:50.469 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:50.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:50.475 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:30:50.475 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:30:50.475 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:30:50.475 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:30:50.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:50.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:51.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:51.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:51.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:51 smithi064 ceph-mon[180297]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:51.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:51.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:51 smithi064 ceph-mon[186706]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:51.214 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:51.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:30:51.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:30:51.261 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:51.314 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:51.409 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:30:51.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:51.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:30:51.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:51 smithi152 ceph-mon[156819]: pgmap v948: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:51.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:51.436 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-06-10T21:30:51.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-06-10T21:30:51.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:51.547 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:51.749 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:30:51.823 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:51.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:51.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:30:51.974 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:52.052 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:30:52.085 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:30:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:30:52.136 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:30:52.161 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:30:52.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:30:52.204 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:30:52.222 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.224 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.225 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.225 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:30:52.250 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3406415918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:52.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3406415918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:52.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3406415918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:30:52.523 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:30:52.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:52.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:30:52.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:52.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:53.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:30:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:30:52] "GET /metrics HTTP/1.1" 200 223721 "" "Prometheus/2.33.4" 2023-06-10T21:30:53.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[180297]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:53.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/728837420' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.176 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:30:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[186706]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/728837420' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:53 smithi152 ceph-mon[156819]: pgmap v949: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/728837420' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:30:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:30:53.473 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:30:53.473 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:30:53.521 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:53.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:53.634 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:53.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:54.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:30:54.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:54.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:54.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:30:54.411 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:54 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:54 smithi152 ceph-mon[156819]: osdmap e265: 8 total, 8 up, 6 in 2023-06-10T21:30:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:54 smithi152 ceph-mon[156819]: failed to encode map e265 with expected crc 2023-06-10T21:30:54.501 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:30:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[180297]: osdmap e265: 8 total, 8 up, 6 in 2023-06-10T21:30:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[180297]: failed to encode map e265 with expected crc 2023-06-10T21:30:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:30:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:30:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[186706]: osdmap e265: 8 total, 8 up, 6 in 2023-06-10T21:30:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:54 smithi064 ceph-mon[186706]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: osdmap e266: 8 total, 8 up, 6 in 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:55 smithi152 ceph-mon[156819]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: osdmap e266: 8 total, 8 up, 6 in 2023-06-10T21:30:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[186706]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: pgmap v951: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 288 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 73 objects/s recovering 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e265 with expected crc 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: osdmap e266: 8 total, 8 up, 6 in 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:55 smithi064 ceph-mon[180297]: failed to encode map e266 with expected crc 2023-06-10T21:30:55.869 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:30:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:55.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:30:55.915 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:30:55.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:30:56.011 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:56.067 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:30:56.067 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:30:56.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:30:56.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:56.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:30:56.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:56.314 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:56.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:56.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:56.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:30:56.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:30:56.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:56.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:56 smithi064 ceph-mon[186706]: pgmap v953: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 181 objects/s recovering 2023-06-10T21:30:56.845 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:56.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:56 smithi064 ceph-mon[180297]: pgmap v953: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 181 objects/s recovering 2023-06-10T21:30:56.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:56.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:30:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:30:56.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:30:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:56 smithi152 ceph-mon[156819]: pgmap v953: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 35081/72 objects degraded (48723.611%); 0 B/s, 181 objects/s recovering 2023-06-10T21:30:56.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 35081/72 objects degraded (48723.611%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:30:56.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:30:57.011 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:30:57.059 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:30:57.134 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:30:57.135 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:30:57.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:30:57.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:30:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:57 smithi152 ceph-mon[156819]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:58.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:57 smithi064 ceph-mon[186706]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:58.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:57 smithi064 ceph-mon[180297]: from='client.46184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:30:58.569 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:30:58.570 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:30:59.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:30:58 smithi064 ceph-mon[186706]: pgmap v954: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 35081/72 objects degraded (48723.611%); 0 B/s, 117 objects/s recovering 2023-06-10T21:30:59.368 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:30:58 smithi064 ceph-mon[180297]: pgmap v954: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 35081/72 objects degraded (48723.611%); 0 B/s, 117 objects/s recovering 2023-06-10T21:30:59.370 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:30:58 smithi152 ceph-mon[156819]: pgmap v954: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 35081/72 objects degraded (48723.611%); 0 B/s, 117 objects/s recovering 2023-06-10T21:30:59.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:30:59.728 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:00.500 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:00.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:00.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:00.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:00.635 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:01.038 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:00 smithi152 ceph-mon[156819]: pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 210 objects/s recovering 2023-06-10T21:31:01.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:00 smithi064 ceph-mon[180297]: pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 210 objects/s recovering 2023-06-10T21:31:01.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:00 smithi064 ceph-mon[186706]: pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 210 objects/s recovering 2023-06-10T21:31:01.237 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:01.238 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:01.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:01.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:01.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:01.697 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:01.736 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":953,"stamp":"2023-06-10T21:30:55.732516+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":181769,"num_write_kb":5810639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22653,"num_bytes_recovered":1323376156,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127001,"ondisk_log_size":127001,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2817984,"kb_used_data":675344,"kb_used_omap":30,"kb_used_meta":2142561,"kb_avail":559652928,"statfs":{"total":575970213888,"available":573084598272,"internally_reserved":0,"allocated":691552256,"data_stored":687018929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2193982502},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-101974040,"num_objects":-1557,"num_object_clones":0,"num_object_copies":-4671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1557,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1453,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002314"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":257,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.165976+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:30:53.165976+0000","last_peered":"2023-06-10T21:30:53.165976+0000","last_clean":"2023-06-10T21:30:53.165976+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:30:53.165976+0000","last_undegraded":"2023-06-10T21:30:53.165976+0000","last_fullsized":"2023-06-10T21:30:53.165976+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":196,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.955013+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:30:29.955013+0000","last_peered":"2023-06-10T21:30:29.955013+0000","last_clean":"2023-06-10T21:30:29.955013+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:30:29.955013+0000","last_undegraded":"2023-06-10T21:30:29.955013+0000","last_fullsized":"2023-06-10T21:30:29.955013+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":171,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660420+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:30:29.660420+0000","last_peered":"2023-06-10T21:30:29.660420+0000","last_clean":"2023-06-10T21:30:29.660420+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:30:29.660420+0000","last_undegraded":"2023-06-10T21:30:29.660420+0000","last_fullsized":"2023-06-10T21:30:29.660420+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:32.806438+0000","last_change":"2023-06-10T21:28:57.902734+0000","last_active":"2023-06-10T21:30:32.806438+0000","last_peered":"2023-06-10T21:30:32.806438+0000","last_clean":"2023-06-10T21:30:32.806438+0000","last_became_active":"2023-06-10T21:28:57.902279+0000","last_became_peered":"2023-06-10T21:28:57.902279+0000","last_unstale":"2023-06-10T21:30:32.806438+0000","last_undegraded":"2023-06-10T21:30:32.806438+0000","last_fullsized":"2023-06-10T21:30:32.806438+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:46.251698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660507+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:30:29.660507+0000","last_peered":"2023-06-10T21:30:29.660507+0000","last_clean":"2023-06-10T21:30:29.660507+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:30:29.660507+0000","last_undegraded":"2023-06-10T21:30:29.660507+0000","last_fullsized":"2023-06-10T21:30:29.660507+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":229,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.813616+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:30:37.813616+0000","last_peered":"2023-06-10T21:30:37.813616+0000","last_clean":"2023-06-10T21:30:37.813616+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:30:37.813616+0000","last_undegraded":"2023-06-10T21:30:37.813616+0000","last_fullsized":"2023-06-10T21:30:37.813616+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":118,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.887688+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:30:29.887688+0000","last_peered":"2023-06-10T21:30:29.887688+0000","last_clean":"2023-06-10T21:30:29.887688+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:30:29.887688+0000","last_undegraded":"2023-06-10T21:30:29.887688+0000","last_fullsized":"2023-06-10T21:30:29.887688+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":210,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660008+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:30:29.660008+0000","last_peered":"2023-06-10T21:30:29.660008+0000","last_clean":"2023-06-10T21:30:29.660008+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:30:29.660008+0000","last_undegraded":"2023-06-10T21:30:29.660008+0000","last_fullsized":"2023-06-10T21:30:29.660008+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.153978+0000","last_change":"2023-06-10T21:28:57.899055+0000","last_active":"2023-06-10T21:30:54.153978+0000","last_peered":"2023-06-10T21:30:54.153978+0000","last_clean":"2023-06-10T21:30:54.153978+0000","last_became_active":"2023-06-10T21:28:57.898692+0000","last_became_peered":"2023-06-10T21:28:57.898692+0000","last_unstale":"2023-06-10T21:30:54.153978+0000","last_undegraded":"2023-06-10T21:30:54.153978+0000","last_fullsized":"2023-06-10T21:30:54.153978+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":189,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660696+0000","last_change":"2023-06-10T21:28:57.895548+0000","last_active":"2023-06-10T21:30:29.660696+0000","last_peered":"2023-06-10T21:30:29.660696+0000","last_clean":"2023-06-10T21:30:29.660696+0000","last_became_active":"2023-06-10T21:28:57.895217+0000","last_became_peered":"2023-06-10T21:28:57.895217+0000","last_unstale":"2023-06-10T21:30:29.660696+0000","last_undegraded":"2023-06-10T21:30:29.660696+0000","last_fullsized":"2023-06-10T21:30:29.660696+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:46:40.560609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":251,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154129+0000","last_change":"2023-06-10T21:28:57.899846+0000","last_active":"2023-06-10T21:30:54.154129+0000","last_peered":"2023-06-10T21:30:54.154129+0000","last_clean":"2023-06-10T21:30:54.154129+0000","last_became_active":"2023-06-10T21:28:57.898805+0000","last_became_peered":"2023-06-10T21:28:57.898805+0000","last_unstale":"2023-06-10T21:30:54.154129+0000","last_undegraded":"2023-06-10T21:30:54.154129+0000","last_fullsized":"2023-06-10T21:30:54.154129+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.165886+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:30:53.165886+0000","last_peered":"2023-06-10T21:30:53.165886+0000","last_clean":"2023-06-10T21:30:53.165886+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:30:53.165886+0000","last_undegraded":"2023-06-10T21:30:53.165886+0000","last_fullsized":"2023-06-10T21:30:53.165886+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":136,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154328+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:30:54.154328+0000","last_peered":"2023-06-10T21:30:54.154328+0000","last_clean":"2023-06-10T21:30:54.154328+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:30:54.154328+0000","last_undegraded":"2023-06-10T21:30:54.154328+0000","last_fullsized":"2023-06-10T21:30:54.154328+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.165963+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:30:53.165963+0000","last_peered":"2023-06-10T21:30:53.165963+0000","last_clean":"2023-06-10T21:30:53.165963+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:30:53.165963+0000","last_undegraded":"2023-06-10T21:30:53.165963+0000","last_fullsized":"2023-06-10T21:30:53.165963+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":265,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154146+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:30:54.154146+0000","last_peered":"2023-06-10T21:30:54.154146+0000","last_clean":"2023-06-10T21:30:54.154146+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:30:54.154146+0000","last_undegraded":"2023-06-10T21:30:54.154146+0000","last_fullsized":"2023-06-10T21:30:54.154146+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660602+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:30:29.660602+0000","last_peered":"2023-06-10T21:30:29.660602+0000","last_clean":"2023-06-10T21:30:29.660602+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:30:29.660602+0000","last_undegraded":"2023-06-10T21:30:29.660602+0000","last_fullsized":"2023-06-10T21:30:29.660602+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":101,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.811341+0000","last_change":"2023-06-10T21:28:57.904420+0000","last_active":"2023-06-10T21:30:41.811341+0000","last_peered":"2023-06-10T21:30:41.811341+0000","last_clean":"2023-06-10T21:30:41.811341+0000","last_became_active":"2023-06-10T21:28:57.904027+0000","last_became_peered":"2023-06-10T21:28:57.904027+0000","last_unstale":"2023-06-10T21:30:41.811341+0000","last_undegraded":"2023-06-10T21:30:41.811341+0000","last_fullsized":"2023-06-10T21:30:41.811341+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:15:36.667597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":194,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154277+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:30:54.154277+0000","last_peered":"2023-06-10T21:30:54.154277+0000","last_clean":"2023-06-10T21:30:54.154277+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:30:54.154277+0000","last_undegraded":"2023-06-10T21:30:54.154277+0000","last_fullsized":"2023-06-10T21:30:54.154277+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154122+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:30:54.154122+0000","last_peered":"2023-06-10T21:30:54.154122+0000","last_clean":"2023-06-10T21:30:54.154122+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:30:54.154122+0000","last_undegraded":"2023-06-10T21:30:54.154122+0000","last_fullsized":"2023-06-10T21:30:54.154122+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.887685+0000","last_change":"2023-06-10T21:28:57.902565+0000","last_active":"2023-06-10T21:30:29.887685+0000","last_peered":"2023-06-10T21:30:29.887685+0000","last_clean":"2023-06-10T21:30:29.887685+0000","last_became_active":"2023-06-10T21:28:57.901902+0000","last_became_peered":"2023-06-10T21:28:57.901902+0000","last_unstale":"2023-06-10T21:30:29.887685+0000","last_undegraded":"2023-06-10T21:30:29.887685+0000","last_fullsized":"2023-06-10T21:30:29.887685+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:54.534642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":238,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.809860+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:30:41.809860+0000","last_peered":"2023-06-10T21:30:41.809860+0000","last_clean":"2023-06-10T21:30:41.809860+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:30:41.809860+0000","last_undegraded":"2023-06-10T21:30:41.809860+0000","last_fullsized":"2023-06-10T21:30:41.809860+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949539+0000","last_change":"2023-06-10T21:28:57.889771+0000","last_active":"2023-06-10T21:30:29.949539+0000","last_peered":"2023-06-10T21:30:29.949539+0000","last_clean":"2023-06-10T21:30:29.949539+0000","last_became_active":"2023-06-10T21:28:57.887036+0000","last_became_peered":"2023-06-10T21:28:57.887036+0000","last_unstale":"2023-06-10T21:30:29.949539+0000","last_undegraded":"2023-06-10T21:30:29.949539+0000","last_fullsized":"2023-06-10T21:30:29.949539+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:05.952139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"264'11358","reported_seq":13108,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:46.914901+0000","last_change":"2023-06-10T21:28:34.624898+0000","last_active":"2023-06-10T21:30:46.914901+0000","last_peered":"2023-06-10T21:30:46.914901+0000","last_clean":"2023-06-10T21:30:46.914901+0000","last_became_active":"2023-06-10T21:28:34.501130+0000","last_became_peered":"2023-06-10T21:28:34.501130+0000","last_unstale":"2023-06-10T21:30:46.914901+0000","last_undegraded":"2023-06-10T21:30:46.914901+0000","last_fullsized":"2023-06-10T21:30:46.914901+0000","mapping_epoch":244,"log_start":"224'5200","ondisk_log_start":"224'5200","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6158,"log_dups_size":0,"ondisk_log_size":6158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:11:04.611578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11358,"num_write_kb":363456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":832,"num_bytes_recovered":53870592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":275,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154390+0000","last_change":"2023-06-10T21:28:34.522514+0000","last_active":"2023-06-10T21:30:54.154390+0000","last_peered":"2023-06-10T21:30:54.154390+0000","last_clean":"2023-06-10T21:30:54.154390+0000","last_became_active":"2023-06-10T21:28:34.522038+0000","last_became_peered":"2023-06-10T21:28:34.522038+0000","last_unstale":"2023-06-10T21:30:54.154390+0000","last_undegraded":"2023-06-10T21:30:54.154390+0000","last_fullsized":"2023-06-10T21:30:54.154390+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154415+0000","last_change":"2023-06-10T21:28:57.899217+0000","last_active":"2023-06-10T21:30:54.154415+0000","last_peered":"2023-06-10T21:30:54.154415+0000","last_clean":"2023-06-10T21:30:54.154415+0000","last_became_active":"2023-06-10T21:28:57.898541+0000","last_became_peered":"2023-06-10T21:28:57.898541+0000","last_unstale":"2023-06-10T21:30:54.154415+0000","last_undegraded":"2023-06-10T21:30:54.154415+0000","last_fullsized":"2023-06-10T21:30:54.154415+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"264'11420","reported_seq":19942,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:53.166902+0000","last_change":"2023-06-10T21:28:59.502716+0000","last_active":"2023-06-10T21:30:53.166902+0000","last_peered":"2023-06-10T21:30:53.166902+0000","last_clean":"2023-06-10T21:28:56.868806+0000","last_became_active":"2023-06-10T21:28:59.444403+0000","last_became_peered":"2023-06-10T21:28:59.444403+0000","last_unstale":"2023-06-10T21:30:53.166902+0000","last_undegraded":"2023-06-10T21:28:59.066225+0000","last_fullsized":"2023-06-10T21:28:58.880029+0000","mapping_epoch":251,"log_start":"203'1400","ondisk_log_start":"203'1400","created":194,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10020,"log_dups_size":0,"ondisk_log_size":10020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:26.444508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11420,"num_write_kb":365440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4198,"num_bytes_recovered":275054592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2314}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":235,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154070+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:30:54.154070+0000","last_peered":"2023-06-10T21:30:54.154070+0000","last_clean":"2023-06-10T21:30:54.154070+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:30:54.154070+0000","last_undegraded":"2023-06-10T21:30:54.154070+0000","last_fullsized":"2023-06-10T21:30:54.154070+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":135,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660314+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:30:29.660314+0000","last_peered":"2023-06-10T21:30:29.660314+0000","last_clean":"2023-06-10T21:30:29.660314+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:30:29.660314+0000","last_undegraded":"2023-06-10T21:30:29.660314+0000","last_fullsized":"2023-06-10T21:30:29.660314+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.4","version":"264'11422","reported_seq":11510,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:47.080117+0000","last_change":"2023-06-10T21:28:59.404926+0000","last_active":"2023-06-10T21:30:47.080117+0000","last_peered":"2023-06-10T21:30:47.080117+0000","last_clean":"2023-06-10T21:26:26.766125+0000","last_became_active":"2023-06-10T21:28:59.373005+0000","last_became_peered":"2023-06-10T21:28:59.373005+0000","last_unstale":"2023-06-10T21:30:47.080117+0000","last_undegraded":"2023-06-10T21:28:58.922286+0000","last_fullsized":"2023-06-10T21:28:58.878960+0000","mapping_epoch":251,"log_start":"224'5200","ondisk_log_start":"224'5200","created":194,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:21.872755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11422,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":-2219}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":260,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154019+0000","last_change":"2023-06-10T21:28:57.900006+0000","last_active":"2023-06-10T21:30:54.154019+0000","last_peered":"2023-06-10T21:30:54.154019+0000","last_clean":"2023-06-10T21:30:54.154019+0000","last_became_active":"2023-06-10T21:28:57.898709+0000","last_became_peered":"2023-06-10T21:28:57.898709+0000","last_unstale":"2023-06-10T21:30:54.154019+0000","last_undegraded":"2023-06-10T21:30:54.154019+0000","last_fullsized":"2023-06-10T21:30:54.154019+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":142,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154231+0000","last_change":"2023-06-10T21:28:34.551591+0000","last_active":"2023-06-10T21:30:54.154231+0000","last_peered":"2023-06-10T21:30:54.154231+0000","last_clean":"2023-06-10T21:30:54.154231+0000","last_became_active":"2023-06-10T21:28:34.551254+0000","last_became_peered":"2023-06-10T21:28:34.551254+0000","last_unstale":"2023-06-10T21:30:54.154231+0000","last_undegraded":"2023-06-10T21:30:54.154231+0000","last_fullsized":"2023-06-10T21:30:54.154231+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"264'11262","reported_seq":11370,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:53.166496+0000","last_change":"2023-06-10T21:28:59.445842+0000","last_active":"2023-06-10T21:30:53.166496+0000","last_peered":"2023-06-10T21:30:53.166496+0000","last_clean":"2023-06-10T21:26:26.764779+0000","last_became_active":"2023-06-10T21:28:59.419763+0000","last_became_peered":"2023-06-10T21:28:59.419763+0000","last_unstale":"2023-06-10T21:30:53.166496+0000","last_undegraded":"2023-06-10T21:28:58.932194+0000","last_fullsized":"2023-06-10T21:28:58.879691+0000","mapping_epoch":251,"log_start":"203'1200","ondisk_log_start":"203'1200","created":194,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:46.862125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4698,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11262,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":-2288}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660742+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:30:29.660742+0000","last_peered":"2023-06-10T21:30:29.660742+0000","last_clean":"2023-06-10T21:30:29.660742+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:30:29.660742+0000","last_undegraded":"2023-06-10T21:30:29.660742+0000","last_fullsized":"2023-06-10T21:30:29.660742+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":86,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.814282+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:30:41.814282+0000","last_peered":"2023-06-10T21:30:41.814282+0000","last_clean":"2023-06-10T21:30:41.814282+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:30:41.814282+0000","last_undegraded":"2023-06-10T21:30:41.814282+0000","last_fullsized":"2023-06-10T21:30:41.814282+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"264'11520","reported_seq":11615,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:47.091232+0000","last_change":"2023-06-10T21:28:34.940046+0000","last_active":"2023-06-10T21:30:47.091232+0000","last_peered":"2023-06-10T21:30:47.091232+0000","last_clean":"2023-06-10T21:30:47.091232+0000","last_became_active":"2023-06-10T21:28:34.501595+0000","last_became_peered":"2023-06-10T21:28:34.501595+0000","last_unstale":"2023-06-10T21:30:47.091232+0000","last_undegraded":"2023-06-10T21:30:47.091232+0000","last_fullsized":"2023-06-10T21:30:47.091232+0000","mapping_epoch":244,"log_start":"224'5302","ondisk_log_start":"224'5302","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6218,"log_dups_size":0,"ondisk_log_size":6218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:59:45.255711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11520,"num_write_kb":368640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":213,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949528+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:30:29.949528+0000","last_peered":"2023-06-10T21:30:29.949528+0000","last_clean":"2023-06-10T21:30:29.949528+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:30:29.949528+0000","last_undegraded":"2023-06-10T21:30:29.949528+0000","last_fullsized":"2023-06-10T21:30:29.949528+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":133,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660815+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:30:29.660815+0000","last_peered":"2023-06-10T21:30:29.660815+0000","last_clean":"2023-06-10T21:30:29.660815+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:30:29.660815+0000","last_undegraded":"2023-06-10T21:30:29.660815+0000","last_fullsized":"2023-06-10T21:30:29.660815+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.3","version":"264'11056","reported_seq":22102,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.227271+0000","last_change":"2023-06-10T21:30:54.227271+0000","last_active":"2023-06-10T21:30:54.227271+0000","last_peered":"2023-06-10T21:30:54.227271+0000","last_clean":"2023-06-10T21:30:54.227271+0000","last_became_active":"2023-06-10T21:30:54.182622+0000","last_became_peered":"2023-06-10T21:30:54.182622+0000","last_unstale":"2023-06-10T21:30:54.227271+0000","last_undegraded":"2023-06-10T21:30:54.227271+0000","last_fullsized":"2023-06-10T21:30:54.227271+0000","mapping_epoch":265,"log_start":"203'1000","ondisk_log_start":"203'1000","created":194,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:03:17.959358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11056,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5460,"num_bytes_recovered":201523200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":36,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949457+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:30:29.949457+0000","last_peered":"2023-06-10T21:30:29.949457+0000","last_clean":"2023-06-10T21:30:29.949457+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:30:29.949457+0000","last_undegraded":"2023-06-10T21:30:29.949457+0000","last_fullsized":"2023-06-10T21:30:29.949457+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":256,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166063+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:30:53.166063+0000","last_peered":"2023-06-10T21:30:53.166063+0000","last_clean":"2023-06-10T21:30:53.166063+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:30:53.166063+0000","last_undegraded":"2023-06-10T21:30:53.166063+0000","last_fullsized":"2023-06-10T21:30:53.166063+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"264'11472","reported_seq":16900,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154283+0000","last_change":"2023-06-10T21:27:45.367487+0000","last_active":"2023-06-10T21:30:54.154283+0000","last_peered":"2023-06-10T21:30:54.154283+0000","last_clean":"2023-06-10T21:30:54.154283+0000","last_became_active":"2023-06-10T21:27:45.353438+0000","last_became_peered":"2023-06-10T21:27:45.353438+0000","last_unstale":"2023-06-10T21:30:54.154283+0000","last_undegraded":"2023-06-10T21:30:54.154283+0000","last_fullsized":"2023-06-10T21:30:54.154283+0000","mapping_epoch":234,"log_start":"222'3700","ondisk_log_start":"222'3700","created":194,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7772,"log_dups_size":0,"ondisk_log_size":7772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11472,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3133,"num_bytes_recovered":205324288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166221+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:30:53.166221+0000","last_peered":"2023-06-10T21:30:53.166221+0000","last_clean":"2023-06-10T21:30:53.166221+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:30:53.166221+0000","last_undegraded":"2023-06-10T21:30:53.166221+0000","last_fullsized":"2023-06-10T21:30:53.166221+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":36,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.955380+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:30:29.955380+0000","last_peered":"2023-06-10T21:30:29.955380+0000","last_clean":"2023-06-10T21:30:29.955380+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:30:29.955380+0000","last_undegraded":"2023-06-10T21:30:29.955380+0000","last_fullsized":"2023-06-10T21:30:29.955380+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.1","version":"264'11338","reported_seq":11453,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.153891+0000","last_change":"2023-06-10T21:28:34.572666+0000","last_active":"2023-06-10T21:30:54.153891+0000","last_peered":"2023-06-10T21:30:54.153891+0000","last_clean":"2023-06-10T21:30:54.153891+0000","last_became_active":"2023-06-10T21:28:34.522012+0000","last_became_peered":"2023-06-10T21:28:34.522012+0000","last_unstale":"2023-06-10T21:30:54.153891+0000","last_undegraded":"2023-06-10T21:30:54.153891+0000","last_fullsized":"2023-06-10T21:30:54.153891+0000","mapping_epoch":244,"log_start":"221'3600","ondisk_log_start":"221'3600","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7738,"log_dups_size":0,"ondisk_log_size":7738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11338,"num_write_kb":362816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":198,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660351+0000","last_change":"2023-06-10T21:28:57.894408+0000","last_active":"2023-06-10T21:30:29.660351+0000","last_peered":"2023-06-10T21:30:29.660351+0000","last_clean":"2023-06-10T21:30:29.660351+0000","last_became_active":"2023-06-10T21:28:57.893595+0000","last_became_peered":"2023-06-10T21:28:57.893595+0000","last_unstale":"2023-06-10T21:30:29.660351+0000","last_undegraded":"2023-06-10T21:30:29.660351+0000","last_fullsized":"2023-06-10T21:30:29.660351+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:03.720304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":172,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949933+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:30:29.949933+0000","last_peered":"2023-06-10T21:30:29.949933+0000","last_clean":"2023-06-10T21:30:29.949933+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:30:29.949933+0000","last_undegraded":"2023-06-10T21:30:29.949933+0000","last_fullsized":"2023-06-10T21:30:29.949933+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.c","version":"264'11422","reported_seq":11534,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154121+0000","last_change":"2023-06-10T21:28:34.926507+0000","last_active":"2023-06-10T21:30:54.154121+0000","last_peered":"2023-06-10T21:30:54.154121+0000","last_clean":"2023-06-10T21:30:54.154121+0000","last_became_active":"2023-06-10T21:28:34.541984+0000","last_became_peered":"2023-06-10T21:28:34.541984+0000","last_unstale":"2023-06-10T21:30:54.154121+0000","last_undegraded":"2023-06-10T21:30:54.154121+0000","last_fullsized":"2023-06-10T21:30:54.154121+0000","mapping_epoch":244,"log_start":"221'3700","ondisk_log_start":"221'3700","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11422,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":86,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812810+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:30:37.812810+0000","last_peered":"2023-06-10T21:30:37.812810+0000","last_clean":"2023-06-10T21:30:37.812810+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:30:37.812810+0000","last_undegraded":"2023-06-10T21:30:37.812810+0000","last_fullsized":"2023-06-10T21:30:37.812810+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.955171+0000","last_change":"2023-06-10T21:28:57.946656+0000","last_active":"2023-06-10T21:30:29.955171+0000","last_peered":"2023-06-10T21:30:29.955171+0000","last_clean":"2023-06-10T21:30:29.955171+0000","last_became_active":"2023-06-10T21:28:57.946186+0000","last_became_peered":"2023-06-10T21:28:57.946186+0000","last_unstale":"2023-06-10T21:30:29.955171+0000","last_undegraded":"2023-06-10T21:30:29.955171+0000","last_fullsized":"2023-06-10T21:30:29.955171+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:31.259349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":106,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812706+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:30:37.812706+0000","last_peered":"2023-06-10T21:30:37.812706+0000","last_clean":"2023-06-10T21:30:37.812706+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:30:37.812706+0000","last_undegraded":"2023-06-10T21:30:37.812706+0000","last_fullsized":"2023-06-10T21:30:37.812706+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.a","version":"264'11342","reported_seq":13670,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:47.042934+0000","last_change":"2023-06-10T21:28:59.454017+0000","last_active":"2023-06-10T21:30:47.042934+0000","last_peered":"2023-06-10T21:30:47.042934+0000","last_clean":"2023-06-10T21:28:56.879563+0000","last_became_active":"2023-06-10T21:28:59.417345+0000","last_became_peered":"2023-06-10T21:28:59.417345+0000","last_unstale":"2023-06-10T21:30:47.042934+0000","last_undegraded":"2023-06-10T21:28:58.916372+0000","last_fullsized":"2023-06-10T21:28:58.878970+0000","mapping_epoch":251,"log_start":"224'5200","ondisk_log_start":"224'5200","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:55:23.819643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5415,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11342,"num_write_kb":362944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1110,"num_bytes_recovered":71827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2","objects":-2920}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":132,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812721+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:30:37.812721+0000","last_peered":"2023-06-10T21:30:37.812721+0000","last_clean":"2023-06-10T21:30:37.812721+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:30:37.812721+0000","last_undegraded":"2023-06-10T21:30:37.812721+0000","last_fullsized":"2023-06-10T21:30:37.812721+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":206,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812765+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:30:37.812765+0000","last_peered":"2023-06-10T21:30:37.812765+0000","last_clean":"2023-06-10T21:30:37.812765+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:30:37.812765+0000","last_undegraded":"2023-06-10T21:30:37.812765+0000","last_fullsized":"2023-06-10T21:30:37.812765+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166601+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:30:53.166601+0000","last_peered":"2023-06-10T21:30:53.166601+0000","last_clean":"2023-06-10T21:30:53.166601+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:30:53.166601+0000","last_undegraded":"2023-06-10T21:30:53.166601+0000","last_fullsized":"2023-06-10T21:30:53.166601+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"264'11230","reported_seq":13020,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166394+0000","last_change":"2023-06-10T21:26:01.630050+0000","last_active":"2023-06-10T21:30:53.166394+0000","last_peered":"2023-06-10T21:30:53.166394+0000","last_clean":"2023-06-10T21:30:53.166394+0000","last_became_active":"2023-06-10T21:26:01.626322+0000","last_became_peered":"2023-06-10T21:26:01.626322+0000","last_unstale":"2023-06-10T21:30:53.166394+0000","last_undegraded":"2023-06-10T21:30:53.166394+0000","last_fullsized":"2023-06-10T21:30:53.166394+0000","mapping_epoch":210,"log_start":"203'1200","ondisk_log_start":"203'1200","created":194,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:03.741223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11230,"num_write_kb":359360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":840,"num_bytes_recovered":55050240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":108,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166297+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:30:53.166297+0000","last_peered":"2023-06-10T21:30:53.166297+0000","last_clean":"2023-06-10T21:30:53.166297+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:30:53.166297+0000","last_undegraded":"2023-06-10T21:30:53.166297+0000","last_fullsized":"2023-06-10T21:30:53.166297+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":106,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.165781+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:30:53.165781+0000","last_peered":"2023-06-10T21:30:53.165781+0000","last_clean":"2023-06-10T21:30:53.165781+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:30:53.165781+0000","last_undegraded":"2023-06-10T21:30:53.165781+0000","last_fullsized":"2023-06-10T21:30:53.165781+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":137,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.814226+0000","last_change":"2023-06-10T21:28:34.518545+0000","last_active":"2023-06-10T21:30:41.814226+0000","last_peered":"2023-06-10T21:30:41.814226+0000","last_clean":"2023-06-10T21:30:41.814226+0000","last_became_active":"2023-06-10T21:28:34.517742+0000","last_became_peered":"2023-06-10T21:28:34.517742+0000","last_unstale":"2023-06-10T21:30:41.814226+0000","last_undegraded":"2023-06-10T21:30:41.814226+0000","last_fullsized":"2023-06-10T21:30:41.814226+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:13.761246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"264'11324","reported_seq":17726,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:47.096298+0000","last_change":"2023-06-10T21:28:34.988147+0000","last_active":"2023-06-10T21:30:47.096298+0000","last_peered":"2023-06-10T21:30:47.096298+0000","last_clean":"2023-06-10T21:30:47.096298+0000","last_became_active":"2023-06-10T21:28:34.506722+0000","last_became_peered":"2023-06-10T21:28:34.506722+0000","last_unstale":"2023-06-10T21:30:47.096298+0000","last_undegraded":"2023-06-10T21:30:47.096298+0000","last_fullsized":"2023-06-10T21:30:47.096298+0000","mapping_epoch":244,"log_start":"224'5200","ondisk_log_start":"224'5200","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6124,"log_dups_size":0,"ondisk_log_size":6124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:56:33.672188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3157,"num_bytes_recovered":206438400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.165880+0000","last_change":"2023-06-10T21:28:57.907800+0000","last_active":"2023-06-10T21:30:53.165880+0000","last_peered":"2023-06-10T21:30:53.165880+0000","last_clean":"2023-06-10T21:30:53.165880+0000","last_became_active":"2023-06-10T21:28:57.907509+0000","last_became_peered":"2023-06-10T21:28:57.907509+0000","last_unstale":"2023-06-10T21:30:53.165880+0000","last_undegraded":"2023-06-10T21:30:53.165880+0000","last_fullsized":"2023-06-10T21:30:53.165880+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:56.001988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":553,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154423+0000","last_change":"2023-06-10T21:28:34.503678+0000","last_active":"2023-06-10T21:30:54.154423+0000","last_peered":"2023-06-10T21:30:54.154423+0000","last_clean":"2023-06-10T21:30:54.154423+0000","last_became_active":"2023-06-10T21:28:34.503254+0000","last_became_peered":"2023-06-10T21:28:34.503254+0000","last_unstale":"2023-06-10T21:30:54.154423+0000","last_undegraded":"2023-06-10T21:30:54.154423+0000","last_fullsized":"2023-06-10T21:30:54.154423+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":199,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660850+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:30:29.660850+0000","last_peered":"2023-06-10T21:30:29.660850+0000","last_clean":"2023-06-10T21:30:29.660850+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:30:29.660850+0000","last_undegraded":"2023-06-10T21:30:29.660850+0000","last_fullsized":"2023-06-10T21:30:29.660850+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154192+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:30:54.154192+0000","last_peered":"2023-06-10T21:30:54.154192+0000","last_clean":"2023-06-10T21:30:54.154192+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:30:54.154192+0000","last_undegraded":"2023-06-10T21:30:54.154192+0000","last_fullsized":"2023-06-10T21:30:54.154192+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"264'11504","reported_seq":11602,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:47.048665+0000","last_change":"2023-06-10T21:28:34.951633+0000","last_active":"2023-06-10T21:30:47.048665+0000","last_peered":"2023-06-10T21:30:47.048665+0000","last_clean":"2023-06-10T21:30:47.048665+0000","last_became_active":"2023-06-10T21:28:34.509441+0000","last_became_peered":"2023-06-10T21:28:34.509441+0000","last_unstale":"2023-06-10T21:30:47.048665+0000","last_undegraded":"2023-06-10T21:30:47.048665+0000","last_fullsized":"2023-06-10T21:30:47.048665+0000","mapping_epoch":244,"log_start":"224'5300","ondisk_log_start":"224'5300","created":194,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6204,"log_dups_size":0,"ondisk_log_size":6204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:34:59.163129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11504,"num_write_kb":368128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":285,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154332+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:30:54.154332+0000","last_peered":"2023-06-10T21:30:54.154332+0000","last_clean":"2023-06-10T21:30:54.154332+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:30:54.154332+0000","last_undegraded":"2023-06-10T21:30:54.154332+0000","last_fullsized":"2023-06-10T21:30:54.154332+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":101,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812675+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:30:37.812675+0000","last_peered":"2023-06-10T21:30:37.812675+0000","last_clean":"2023-06-10T21:30:37.812675+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:30:37.812675+0000","last_undegraded":"2023-06-10T21:30:37.812675+0000","last_fullsized":"2023-06-10T21:30:37.812675+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":86,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.814986+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:30:41.814986+0000","last_peered":"2023-06-10T21:30:41.814986+0000","last_clean":"2023-06-10T21:30:41.814986+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:30:41.814986+0000","last_undegraded":"2023-06-10T21:30:41.814986+0000","last_fullsized":"2023-06-10T21:30:41.814986+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"264'11448","reported_seq":12058,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:47.108088+0000","last_change":"2023-06-10T21:28:59.491171+0000","last_active":"2023-06-10T21:30:47.108088+0000","last_peered":"2023-06-10T21:30:47.108088+0000","last_clean":"2023-06-10T21:28:56.884482+0000","last_became_active":"2023-06-10T21:28:59.418669+0000","last_became_peered":"2023-06-10T21:28:59.418669+0000","last_unstale":"2023-06-10T21:30:47.108088+0000","last_undegraded":"2023-06-10T21:28:58.914273+0000","last_fullsized":"2023-06-10T21:28:58.879027+0000","mapping_epoch":251,"log_start":"224'5300","ondisk_log_start":"224'5300","created":194,"last_epoch_clean":246,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:47:23.775373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5473,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11448,"num_write_kb":366273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":255,"num_bytes_recovered":16449536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-3012}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154104+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:30:54.154104+0000","last_peered":"2023-06-10T21:30:54.154104+0000","last_clean":"2023-06-10T21:30:54.154104+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:30:54.154104+0000","last_undegraded":"2023-06-10T21:30:54.154104+0000","last_fullsized":"2023-06-10T21:30:54.154104+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":106,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166526+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:30:53.166526+0000","last_peered":"2023-06-10T21:30:53.166526+0000","last_clean":"2023-06-10T21:30:53.166526+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:30:53.166526+0000","last_undegraded":"2023-06-10T21:30:53.166526+0000","last_fullsized":"2023-06-10T21:30:53.166526+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812780+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:30:37.812780+0000","last_peered":"2023-06-10T21:30:37.812780+0000","last_clean":"2023-06-10T21:30:37.812780+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:30:37.812780+0000","last_undegraded":"2023-06-10T21:30:37.812780+0000","last_fullsized":"2023-06-10T21:30:37.812780+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"264'11296","reported_seq":11958,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:53.167201+0000","last_change":"2023-06-10T21:28:59.476469+0000","last_active":"2023-06-10T21:30:53.167201+0000","last_peered":"2023-06-10T21:30:53.167201+0000","last_clean":"2023-06-10T21:28:56.852537+0000","last_became_active":"2023-06-10T21:28:59.439769+0000","last_became_peered":"2023-06-10T21:28:59.439769+0000","last_unstale":"2023-06-10T21:30:53.167201+0000","last_undegraded":"2023-06-10T21:28:58.939041+0000","last_fullsized":"2023-06-10T21:28:58.883972+0000","mapping_epoch":251,"log_start":"203'1200","ondisk_log_start":"203'1200","created":194,"last_epoch_clean":228,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10096,"log_dups_size":0,"ondisk_log_size":10096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:10:03.064078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5381,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11296,"num_write_kb":361472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":267,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2942}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":213,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949636+0000","last_change":"2023-06-10T21:28:57.899002+0000","last_active":"2023-06-10T21:30:29.949636+0000","last_peered":"2023-06-10T21:30:29.949636+0000","last_clean":"2023-06-10T21:30:29.949636+0000","last_became_active":"2023-06-10T21:28:57.898639+0000","last_became_peered":"2023-06-10T21:28:57.898639+0000","last_unstale":"2023-06-10T21:30:29.949636+0000","last_undegraded":"2023-06-10T21:30:29.949636+0000","last_fullsized":"2023-06-10T21:30:29.949636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:12:01.141117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":36,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949949+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:30:29.949949+0000","last_peered":"2023-06-10T21:30:29.949949+0000","last_clean":"2023-06-10T21:30:29.949949+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:30:29.949949+0000","last_undegraded":"2023-06-10T21:30:29.949949+0000","last_fullsized":"2023-06-10T21:30:29.949949+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660265+0000","last_change":"2023-06-10T21:28:57.895650+0000","last_active":"2023-06-10T21:30:29.660265+0000","last_peered":"2023-06-10T21:30:29.660265+0000","last_clean":"2023-06-10T21:30:29.660265+0000","last_became_active":"2023-06-10T21:28:57.895320+0000","last_became_peered":"2023-06-10T21:28:57.895320+0000","last_unstale":"2023-06-10T21:30:29.660265+0000","last_undegraded":"2023-06-10T21:30:29.660265+0000","last_fullsized":"2023-06-10T21:30:29.660265+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:35:20.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"264'11094","reported_seq":17887,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:30:53.167779+0000","last_change":"2023-06-10T21:28:59.500885+0000","last_active":"2023-06-10T21:30:53.167779+0000","last_peered":"2023-06-10T21:30:53.167779+0000","last_clean":"2023-06-10T21:28:56.851941+0000","last_became_active":"2023-06-10T21:28:59.444754+0000","last_became_peered":"2023-06-10T21:28:59.444754+0000","last_unstale":"2023-06-10T21:30:53.167779+0000","last_undegraded":"2023-06-10T21:28:59.081148+0000","last_fullsized":"2023-06-10T21:28:58.879700+0000","mapping_epoch":251,"log_start":"203'1000","ondisk_log_start":"203'1000","created":194,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:25:27.158506+0000","last_clean_scrub_stamp":"2023-06-10T21:25:27.158506+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:11:59.029499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4601,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11094,"num_write_kb":355008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3339,"num_bytes_recovered":218628096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2211}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":256,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154398+0000","last_change":"2023-06-10T21:28:57.899350+0000","last_active":"2023-06-10T21:30:54.154398+0000","last_peered":"2023-06-10T21:30:54.154398+0000","last_clean":"2023-06-10T21:30:54.154398+0000","last_became_active":"2023-06-10T21:28:57.898711+0000","last_became_peered":"2023-06-10T21:28:57.898711+0000","last_unstale":"2023-06-10T21:30:54.154398+0000","last_undegraded":"2023-06-10T21:30:54.154398+0000","last_fullsized":"2023-06-10T21:30:54.154398+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":90,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-06-10T21:30:41.815149+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:30:41.815149+0000","last_peered":"2023-06-10T21:30:41.815149+0000","last_clean":"2023-06-10T21:30:41.815149+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:30:41.815149+0000","last_undegraded":"2023-06-10T21:30:41.815149+0000","last_fullsized":"2023-06-10T21:30:41.815149+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":200,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.812655+0000","last_change":"2023-06-10T21:28:34.521765+0000","last_active":"2023-06-10T21:30:37.812655+0000","last_peered":"2023-06-10T21:30:37.812655+0000","last_clean":"2023-06-10T21:30:37.812655+0000","last_became_active":"2023-06-10T21:28:34.521409+0000","last_became_peered":"2023-06-10T21:28:34.521409+0000","last_unstale":"2023-06-10T21:30:37.812655+0000","last_undegraded":"2023-06-10T21:30:37.812655+0000","last_fullsized":"2023-06-10T21:30:37.812655+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:05:44.771841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":85,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.887958+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:30:29.887958+0000","last_peered":"2023-06-10T21:30:29.887958+0000","last_clean":"2023-06-10T21:30:29.887958+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:30:29.887958+0000","last_undegraded":"2023-06-10T21:30:29.887958+0000","last_fullsized":"2023-06-10T21:30:29.887958+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":106,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:53.166440+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:30:53.166440+0000","last_peered":"2023-06-10T21:30:53.166440+0000","last_clean":"2023-06-10T21:30:53.166440+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:30:53.166440+0000","last_undegraded":"2023-06-10T21:30:53.166440+0000","last_fullsized":"2023-06-10T21:30:53.166440+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":66,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.154021+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:30:54.154021+0000","last_peered":"2023-06-10T21:30:54.154021+0000","last_clean":"2023-06-10T21:30:54.154021+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:30:54.154021+0000","last_undegraded":"2023-06-10T21:30:54.154021+0000","last_fullsized":"2023-06-10T21:30:54.154021+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949419+0000","last_change":"2023-06-10T21:28:57.898684+0000","last_active":"2023-06-10T21:30:29.949419+0000","last_peered":"2023-06-10T21:30:29.949419+0000","last_clean":"2023-06-10T21:30:29.949419+0000","last_became_active":"2023-06-10T21:28:57.898387+0000","last_became_peered":"2023-06-10T21:28:57.898387+0000","last_unstale":"2023-06-10T21:30:29.949419+0000","last_undegraded":"2023-06-10T21:30:29.949419+0000","last_fullsized":"2023-06-10T21:30:29.949419+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:55:36.556314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":40,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660592+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:30:29.660592+0000","last_peered":"2023-06-10T21:30:29.660592+0000","last_clean":"2023-06-10T21:30:29.660592+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:30:29.660592+0000","last_undegraded":"2023-06-10T21:30:29.660592+0000","last_fullsized":"2023-06-10T21:30:29.660592+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":200,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.949406+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:30:29.949406+0000","last_peered":"2023-06-10T21:30:29.949406+0000","last_clean":"2023-06-10T21:30:29.949406+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:30:29.949406+0000","last_undegraded":"2023-06-10T21:30:29.949406+0000","last_fullsized":"2023-06-10T21:30:29.949406+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660340+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:30:29.660340+0000","last_peered":"2023-06-10T21:30:29.660340+0000","last_clean":"2023-06-10T21:30:29.660340+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:30:29.660340+0000","last_undegraded":"2023-06-10T21:30:29.660340+0000","last_fullsized":"2023-06-10T21:30:29.660340+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":249,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660565+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:30:29.660565+0000","last_peered":"2023-06-10T21:30:29.660565+0000","last_clean":"2023-06-10T21:30:29.660565+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:30:29.660565+0000","last_undegraded":"2023-06-10T21:30:29.660565+0000","last_fullsized":"2023-06-10T21:30:29.660565+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":151,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:37.813589+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:30:37.813589+0000","last_peered":"2023-06-10T21:30:37.813589+0000","last_clean":"2023-06-10T21:30:37.813589+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:30:37.813589+0000","last_undegraded":"2023-06-10T21:30:37.813589+0000","last_fullsized":"2023-06-10T21:30:37.813589+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":36,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-06-10T21:30:29.660433+0000","last_change":"2023-06-10T21:28:57.894606+0000","last_active":"2023-06-10T21:30:29.660433+0000","last_peered":"2023-06-10T21:30:29.660433+0000","last_clean":"2023-06-10T21:30:29.660433+0000","last_became_active":"2023-06-10T21:28:57.894004+0000","last_became_peered":"2023-06-10T21:28:57.894004+0000","last_unstale":"2023-06-10T21:30:29.660433+0000","last_undegraded":"2023-06-10T21:30:29.660433+0000","last_fullsized":"2023-06-10T21:30:29.660433+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:28:45.258464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35081,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181508,"num_write_kb":5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22620,"num_bytes_recovered":1321598976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1722683392,"data_stored":1722679320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126806,"ondisk_log_size":126806,"up":48,"acting":41,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":6076,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430606,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522036,"kb_used_data":2924,"kb_used_omap":0,"kb_used_meta":519104,"kb_avail":93223116,"statfs":{"total":95995035648,"available":95460470784,"internally_reserved":0,"allocated":2994176,"data_stored":2447747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":531562496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":8.0749999999999993,"15min":9.1630000000000003},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.3239999999999998,"5min":89.245000000000005,"15min":89.245000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.327,"5min":8.3049999999999997,"15min":9.468},"min":{"1min":0.23899999999999999,"5min":0.223,"15min":0.223},"max":{"1min":4.5449999999999999,"5min":90.027000000000001,"15min":90.027000000000001},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.2069999999999999,"15min":2.8370000000000002},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0629999999999997,"5min":93.093999999999994,"15min":130.19399999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.796,"15min":3.1589999999999998},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.673,"5min":93.119,"15min":212.20599999999999},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6040000000000001,"15min":2.3929999999999998},"min":{"1min":0.377,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8129999999999997,"5min":160.99299999999999,"15min":160.99299999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.998,"5min":4.327,"15min":3.0030000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0529999999999999,"5min":160.53700000000001,"15min":212.13399999999999},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:29:59 2023","interfaces":[{"interface":"back","average":{"1min":2.5,"5min":2.5,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":9.2490000000000006,"5min":9.2490000000000006,"15min":9.2490000000000006},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.833,"5min":1.833,"15min":1.833},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.242,"5min":4.242,"15min":4.242},"last":0.60499999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.65900000000000003,"15min":0.89400000000000002},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.45200000000000001,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.749,"15min":1.3129999999999999},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48599999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.97799999999999998,"15min":2.145},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6280000000000001,"5min":12.048,"15min":211.99799999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.67000000000000004,"15min":1.946},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.56599999999999995,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.97899999999999998,"15min":1.294},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.8849999999999998,"5min":13.073,"15min":13.073},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.626,"15min":1.093},"min":{"1min":0.371,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.897,"5min":3.8239999999999998,"15min":10.874000000000001},"last":0.626}]}]},{"osd":6,"up_from":0,"seq":180388626834,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566276,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":447708,"kb_used_data":3220,"kb_used_omap":4,"kb_used_meta":444475,"kb_avail":93297444,"statfs":{"total":95995035648,"available":95536582656,"internally_reserved":0,"allocated":3297280,"data_stored":2473196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":455143310},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3860000000000001,"5min":12.260999999999999,"15min":13.24},"min":{"1min":0.32600000000000001,"5min":0.221,"15min":0.221},"max":{"1min":42.567999999999998,"5min":107.825,"15min":107.825},"last":154.31399999999999},{"interface":"front","average":{"1min":7.7030000000000003,"5min":12.196999999999999,"15min":12.946},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":42.372999999999998,"5min":110.928,"15min":110.928},"last":154.27199999999999}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":9.843,"5min":5.5069999999999997,"15min":2.8380000000000001},"min":{"1min":0.39100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":107.39700000000001,"5min":107.857,"15min":107.857},"last":104.334},{"interface":"front","average":{"1min":11.939,"5min":5.8049999999999997,"15min":2.637},"min":{"1min":0.61099999999999999,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":106.907,"5min":107.958,"15min":107.958},"last":104.98699999999999}]},{"osd":3,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.976,"5min":1.976,"15min":1.976},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.445,"5min":11.445,"15min":11.445},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.1819999999999999,"5min":3.1819999999999999,"15min":3.1819999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":10.856,"5min":10.856,"15min":10.856},"last":0.24299999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3839999999999999,"15min":1.71},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":16.059999999999999,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.432},{"interface":"front","average":{"1min":1.641,"5min":1.03,"15min":1.5920000000000001},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.8949999999999996,"5min":6.8949999999999996,"15min":14.460000000000001},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.4490000000000001,"15min":4.2670000000000003},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":5.0700000000000003,"5min":6.9550000000000001,"15min":383.83100000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.389,"15min":4.1070000000000002},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.29699999999999999},"max":{"1min":16.146999999999998,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.387,"5min":1.905,"15min":2.1589999999999998},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.33800000000000002},"max":{"1min":6.6859999999999999,"5min":13.465,"15min":14.858000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":2,"5min":1.7270000000000001,"15min":1.956},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":11.476000000000001,"5min":13.48,"15min":13.48},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.6839999999999999,"15min":2.7810000000000001},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.32600000000000001},"max":{"1min":16.113,"5min":16.113,"15min":162.399},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.3280000000000001,"15min":2.367},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9459999999999997,"5min":6.9459999999999997,"15min":162.27199999999999},"last":0.379}]}]},{"osd":0,"up_from":244,"seq":1047972020253,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174836,"kb_used_data":3308,"kb_used_omap":10,"kb_used_meta":171509,"kb_avail":93570316,"statfs":{"total":95995035648,"available":95816003584,"internally_reserved":0,"allocated":3387392,"data_stored":2480418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":175625268},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":5.7510000000000003,"5min":4.6779999999999999,"15min":4.4989999999999997},"min":{"1min":0.32300000000000001,"5min":0.191,"15min":0.191},"max":{"1min":16.716999999999999,"5min":19.806999999999999,"15min":19.806999999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":4.8090000000000002,"5min":5.8109999999999999,"15min":5.9779999999999998},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":12.032,"5min":29.841000000000001,"15min":29.841000000000001},"last":0.312}]},{"osd":2,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":12.528,"5min":7.2670000000000003,"15min":6.3899999999999997},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":205.10900000000001,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":14.632999999999999,"5min":7.0110000000000001,"15min":5.7409999999999997},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":208.34299999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":6.2869999999999999,"5min":6.2869999999999999,"15min":6.2869999999999999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":17.088000000000001,"5min":17.088000000000001,"15min":17.088000000000001},"last":0.371},{"interface":"front","average":{"1min":7.2839999999999998,"5min":7.2839999999999998,"15min":7.2839999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":32.822000000000003,"5min":32.822000000000003,"15min":32.822000000000003},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.4750000000000001,"5min":4.7720000000000002,"15min":4.9880000000000004},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.135999999999999,"5min":29.709,"15min":29.709},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.9809999999999999,"5min":4.016,"15min":4.1879999999999997},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.065,"5min":13.872,"15min":13.872},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":4.1070000000000002,"15min":4.2930000000000001},"min":{"1min":0.39600000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":13.116,"5min":14.105,"15min":14.105},"last":0.443},{"interface":"front","average":{"1min":3.7440000000000002,"5min":3.6400000000000001,"15min":3.6219999999999999},"min":{"1min":0.47899999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.083,"5min":19.846,"15min":19.846},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.5270000000000001,"5min":4.9589999999999996,"15min":5.1970000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.18,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":3.0419999999999998,"5min":4.1459999999999999,"15min":4.3300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":13.147,"5min":13.82,"15min":13.82},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":4.0179999999999998,"15min":4.0640000000000001},"min":{"1min":0.44400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.121,"5min":19.866,"15min":19.866},"last":0.50900000000000001},{"interface":"front","average":{"1min":3.5289999999999999,"5min":4.9569999999999999,"15min":5.1950000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":10.218999999999999,"5min":29.77,"15min":29.77},"last":0.626}]}]},{"osd":2,"up_from":120,"seq":515396075665,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325920,"kb_used_data":2648,"kb_used_omap":7,"kb_used_meta":323256,"kb_avail":93419232,"statfs":{"total":95995035648,"available":95661293568,"internally_reserved":0,"allocated":2711552,"data_stored":1891592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":331014225},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:37 2023","interfaces":[{"interface":"back","average":{"1min":5.0330000000000004,"5min":4.7690000000000001,"15min":4.7249999999999996},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":17.765999999999998,"5min":17.765999999999998,"15min":17.765999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":6.5289999999999999,"5min":5.6189999999999998,"15min":5.4669999999999996},"min":{"1min":0.54100000000000004,"5min":0.252,"15min":0.252},"max":{"1min":18.123999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":5.0439999999999996,"5min":2.3580000000000001,"15min":1.71},"min":{"1min":0.35699999999999998,"5min":0.19,"15min":0.19},"max":{"1min":10.736000000000001,"5min":11.6,"15min":29.486000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":3.5430000000000001,"5min":1.776,"15min":1.488},"min":{"1min":0.28899999999999998,"5min":0.214,"15min":0.214},"max":{"1min":11.336,"5min":14.334,"15min":28.013999999999999},"last":0.28000000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:29:57 2023","interfaces":[{"interface":"back","average":{"1min":5.2409999999999997,"5min":5.2409999999999997,"15min":5.2409999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":14.272,"5min":14.272,"15min":14.272},"last":0.35899999999999999},{"interface":"front","average":{"1min":6.2939999999999996,"5min":6.2939999999999996,"15min":6.2939999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":16.411000000000001,"5min":16.411000000000001,"15min":16.411000000000001},"last":0.25600000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.044,"5min":5.5250000000000004,"15min":2.6320000000000001},"min":{"1min":0.314,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.0969999999999995,"5min":224.00999999999999,"15min":224.00999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":4.6299999999999999,"5min":5.7709999999999999,"15min":2.7799999999999998},"min":{"1min":0.377,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":17.535,"5min":223.28399999999999,"15min":223.28399999999999},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":1.595,"15min":1.339},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.473000000000001,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.0609999999999999,"5min":1.788,"15min":1.542},"min":{"1min":0.442,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.1989999999999998,"5min":16.125,"15min":16.125},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0830000000000002,"5min":2.077,"15min":1.599},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.1690000000000005,"5min":16.183,"15min":16.183},"last":0.51700000000000002},{"interface":"front","average":{"1min":4.3399999999999999,"5min":2.306,"15min":1.782},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":10.625999999999999,"5min":11.029,"15min":17.244},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":3.02,"15min":1.7330000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.44,"5min":113.913,"15min":113.913},"last":0.498},{"interface":"front","average":{"1min":3.0609999999999999,"5min":3.3149999999999999,"15min":2.036},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.141,"5min":113.937,"15min":113.937},"last":0.48499999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824024,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":786920,"kb_used_data":660896,"kb_used_omap":7,"kb_used_meta":126008,"kb_avail":92958232,"statfs":{"total":95995035648,"available":95189229568,"internally_reserved":0,"allocated":676757504,"data_stored":675861707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":129032211},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":5.5490000000000004,"15min":5.5490000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":56.828000000000003,"5min":56.828000000000003,"15min":56.828000000000003},"last":0.28199999999999997},{"interface":"front","average":{"1min":5.3330000000000002,"5min":5.3330000000000002,"15min":5.3330000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":57.600000000000001,"5min":57.600000000000001,"15min":57.600000000000001},"last":0.254}]},{"osd":1,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":3.6389999999999998,"15min":3.6389999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":16.925999999999998,"5min":16.925999999999998,"15min":16.925999999999998},"last":0.65600000000000003},{"interface":"front","average":{"1min":3.4849999999999999,"5min":3.4849999999999999,"15min":3.4849999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.2579999999999991,"5min":8.2579999999999991,"15min":8.2579999999999991},"last":0.376}]},{"osd":2,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":12.922000000000001,"5min":12.922000000000001,"15min":12.922000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":87.555999999999997,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.504},{"interface":"front","average":{"1min":14.343,"5min":14.343,"15min":14.343},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":80.566999999999993,"5min":80.566999999999993,"15min":80.566999999999993},"last":0.69799999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.7690000000000001,"5min":7.7690000000000001,"15min":7.7690000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.7389999999999999,"5min":2.7389999999999999,"15min":2.7389999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7149999999999999,"5min":8.7149999999999999,"15min":8.7149999999999999},"last":0.79200000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.393,"15min":1.393},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.2480000000000002,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":7.8319999999999999,"5min":7.8319999999999999,"15min":7.8319999999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":8.6370000000000005,"5min":8.6370000000000005,"15min":8.6370000000000005},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.4399999999999999,"15min":1.4399999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.3449999999999998,"5min":6.3449999999999998,"15min":6.3449999999999998},"last":0.443}]},{"osd":7,"last update":"Sat Jun 10 21:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.778,"5min":2.778,"15min":2.778},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.6809999999999992,"5min":8.6809999999999992,"15min":8.6809999999999992},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.3029999999999999,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.52300000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019290,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560564,"kb_used_data":2348,"kb_used_omap":0,"kb_used_meta":558208,"kb_avail":93184588,"statfs":{"total":95995035648,"available":95421018112,"internally_reserved":0,"allocated":2404352,"data_stored":1864269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":571604992},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":8.8949999999999996,"15min":10.083},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8889999999999993,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":6.008,"5min":9.1530000000000005,"15min":9.6769999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":38.420000000000002,"5min":144.303,"15min":144.303},"last":0.57099999999999995}]},{"osd":1,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.496,"15min":2.9900000000000002},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6,"5min":15.063000000000001,"15min":182.035},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.718,"15min":2.8940000000000001},"min":{"1min":0.34799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.4760000000000009,"5min":15.052,"15min":181.73099999999999},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.0369999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.319000000000001,"5min":22.614000000000001,"15min":22.614000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.748,"15min":1.377},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":18.329999999999998,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.498}]},{"osd":3,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.3919999999999999,"15min":2.3919999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":9.2379999999999995,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.044,"5min":2.044,"15min":2.044},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.9640000000000004,"5min":5.9640000000000004,"15min":5.9640000000000004},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.76200000000000001,"15min":1.1399999999999999},"min":{"1min":0.30299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.80000000000000004,"5min":6.742,"15min":13.941000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.88900000000000001,"15min":1.2070000000000001},"min":{"1min":0.311,"5min":0.247,"15min":0.247},"max":{"1min":0.85199999999999998,"5min":6.0819999999999999,"15min":11.904999999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.82299999999999995,"15min":1.179},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.749,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.92300000000000004,"15min":1.2569999999999999},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":1.1990000000000001,"5min":8.3109999999999999,"15min":13.845000000000001},"last":0.44}]},{"osd":7,"last update":"Sat Jun 10 21:30:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.206,"15min":1.448},"min":{"1min":0.32900000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.7689999999999999,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.0980000000000001,"15min":1.3180000000000001},"min":{"1min":0.35099999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.052,"5min":8.9900000000000002,"15min":9.5060000000000002},"last":0.80400000000000005}]}]},{"osd":5,"up_from":0,"seq":154618823062,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":667353088,"data_stored":667353088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3473408,"data_stored":3473408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1051856896,"data_stored":1051852824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:31:01.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:31:01.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:01.892 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:01.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:01.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:02.031 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:02.106 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:02.146 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:02.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:02.234 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:02.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:02.280 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:02.358 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:02.362 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:02.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:02.468 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:02.495 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:02.537 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:02.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:02.644 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:02.672 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:02.866 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:02 smithi064 ceph-mon[186706]: pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 196 objects/s recovering 2023-06-10T21:31:02.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:02 smithi064 ceph-mon[180297]: pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 196 objects/s recovering 2023-06-10T21:31:02.869 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:02.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:02.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:03.016 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:03.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:02] "GET /metrics HTTP/1.1" 200 223721 "" "Prometheus/2.33.4" 2023-06-10T21:31:03.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:02 smithi152 ceph-mon[156819]: pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 196 objects/s recovering 2023-06-10T21:31:03.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:03.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:03.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:03.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:03.419 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:03.796 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:03.797 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:03.813 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_5' removed 2023-06-10T21:31:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4087786' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:03 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:03.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:04.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4087786' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:04.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:04.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4087786' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:04.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:31:04.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:04.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:04.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:04.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:31:04.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:31:04.854 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:05.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:04 smithi152 ceph-mon[156819]: pgmap v957: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 235 objects/s recovering 2023-06-10T21:31:05.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:04 smithi152 ceph-mon[156819]: failed to encode map e267 with expected crc 2023-06-10T21:31:05.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:04 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:31:05.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:04 smithi152 ceph-mon[156819]: osdmap e267: 8 total, 8 up, 6 in 2023-06-10T21:31:05.070 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:04 smithi152 ceph-mon[156819]: failed to encode map e267 with expected crc 2023-06-10T21:31:05.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[186706]: pgmap v957: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 235 objects/s recovering 2023-06-10T21:31:05.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[186706]: failed to encode map e267 with expected crc 2023-06-10T21:31:05.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:31:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[186706]: osdmap e267: 8 total, 8 up, 6 in 2023-06-10T21:31:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[186706]: failed to encode map e267 with expected crc 2023-06-10T21:31:05.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[180297]: pgmap v957: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 32562/72 objects degraded (45225.000%); 0 B/s, 235 objects/s recovering 2023-06-10T21:31:05.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[180297]: failed to encode map e267 with expected crc 2023-06-10T21:31:05.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:31:05.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[180297]: osdmap e267: 8 total, 8 up, 6 in 2023-06-10T21:31:05.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:04 smithi064 ceph-mon[180297]: failed to encode map e267 with expected crc 2023-06-10T21:31:06.138 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:05 smithi152 ceph-mon[156819]: failed to encode map e267 with expected crc 2023-06-10T21:31:06.139 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:31:06.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:05 smithi064 ceph-mon[180297]: failed to encode map e267 with expected crc 2023-06-10T21:31:06.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:31:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:05 smithi064 ceph-mon[186706]: failed to encode map e267 with expected crc 2023-06-10T21:31:06.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-10T21:31:06.367 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:06.368 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:06.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:06.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:06.781 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:06.809 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:06.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:06 smithi152 ceph-mon[156819]: pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:06 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized) 2023-06-10T21:31:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:31:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:31:06.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:07.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:31:07.103 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[186706]: pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:07.104 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized) 2023-06-10T21:31:07.105 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:31:07.105 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:31:07.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:07.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[180297]: pgmap v959: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:07.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 32562/72 objects degraded (45225.000%), 7 pgs degraded, 7 pgs undersized) 2023-06-10T21:31:07.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:31:07.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:31:07.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:07.107 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:07.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:07.211 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:07.240 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:07.361 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:07.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:07.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:07.396 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:31:07.397 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:31:07.398 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-06-10T21:31:07.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_6 16 2023-06-10T21:31:07.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:07.510 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:07.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:07.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:07.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:07.700 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:07.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:07.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:07.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:07.872 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:07.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:07.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:07.995 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:07.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:08.094 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:31:08.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:08.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:31:08.128 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:08.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:31:08.153 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:08.167 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:31:08.188 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:31:08.211 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:31:08.231 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:31:08.250 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:31:08.251 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.252 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:31:08.286 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:08.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:08.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:08.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:08.427 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:08.628 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:08.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:08.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:08.767 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:08.813 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:08.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:08.868 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:31:08.869 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:31:08.869 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:31:08.869 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:31:08.869 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:31:08.895 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:31:08.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[186706]: pgmap v960: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:08.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3476573697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:08.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3816407445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:08.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:08.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[180297]: pgmap v960: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:08.915 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3476573697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:08.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3816407445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:08.916 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:08 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:08.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:08.949 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:09.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:08 smithi152 ceph-mon[156819]: pgmap v960: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:09.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3476573697' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:09.021 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3816407445' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:09.021 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:08 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:09.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:31:09.863 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[186706]: failed to encode map e268 with expected crc 2023-06-10T21:31:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:09.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[186706]: osdmap e268: 8 total, 8 up, 6 in 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[186706]: failed to encode map e268 with expected crc 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[180297]: failed to encode map e268 with expected crc 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[180297]: osdmap e268: 8 total, 8 up, 6 in 2023-06-10T21:31:09.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:09 smithi064 ceph-mon[180297]: failed to encode map e268 with expected crc 2023-06-10T21:31:10.086 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:09 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:10.086 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:09 smithi152 ceph-mon[156819]: failed to encode map e268 with expected crc 2023-06-10T21:31:10.086 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:09 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:10.086 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:09 smithi152 ceph-mon[156819]: osdmap e268: 8 total, 8 up, 6 in 2023-06-10T21:31:10.087 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:09 smithi152 ceph-mon[156819]: failed to encode map e268 with expected crc 2023-06-10T21:31:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:10.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:10.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:10.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:10.405 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:10.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:10.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:10.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:10.553 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:10.580 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:10.621 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:10.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:10.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:10.762 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:10.955 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:10.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:10.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:11.060 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:11.087 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:10 smithi152 ceph-mon[156819]: pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:11.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:11.093 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:10 smithi064 ceph-mon[180297]: pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:11.093 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:10 smithi064 ceph-mon[186706]: pgmap v962: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:11.139 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:11.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:11.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:11.277 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:11.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:11.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:11.478 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:31:11.478 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:31:11.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:31:11.478 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:31:11.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.479 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166812 9.0 9:0ada01af:::benchmark_data_smithi064_247471_object57824:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:31:11.479 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:31:11.479 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 446.14916882099999, 2023-06-10T21:31:11.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0161496670000001, 2023-06-10T21:31:11.480 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.480 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.481 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.481 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.482 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.482 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166812 2023-06-10T21:31:11.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:31:11.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:31:11.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322918+0000", 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9450000000000004e-06 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322924+0000", 2023-06-10T21:31:11.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0769999999999997e-06 2023-06-10T21:31:11.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322934+0000", 2023-06-10T21:31:11.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5289999999999997e-06 2023-06-10T21:31:11.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322940+0000", 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7850999999999995e-05 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323038+0000", 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0801e-05 2023-06-10T21:31:11.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323049+0000", 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0047669809999999 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.327816+0000", 2023-06-10T21:31:11.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023938549999999999 2023-06-10T21:31:11.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.330210+0000", 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000173513 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.330383+0000", 2023-06-10T21:31:11.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030409370000000001 2023-06-10T21:31:11.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333424+0000", 2023-06-10T21:31:11.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054907360000000004 2023-06-10T21:31:11.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.338915+0000", 2023-06-10T21:31:11.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9415e-05 2023-06-10T21:31:11.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.338964+0000", 2023-06-10T21:31:11.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9045000000000001e-05 2023-06-10T21:31:11.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339023+0000", 2023-06-10T21:31:11.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5683000000000003e-05 2023-06-10T21:31:11.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339069+0000", 2023-06-10T21:31:11.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0161496670000001 2023-06-10T21:31:11.501 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.501 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.501 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.503 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166815 9.0 9:03235a20:::benchmark_data_smithi064_247471_object57827:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:31:11.503 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:31:11.503 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 446.14913448300001, 2023-06-10T21:31:11.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0162916630000001, 2023-06-10T21:31:11.503 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166815 2023-06-10T21:31:11.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.505 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:31:11.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:31:11.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:11.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322953+0000", 2023-06-10T21:31:11.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7860000000000001e-06 2023-06-10T21:31:11.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322956+0000", 2023-06-10T21:31:11.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8239999999999996e-06 2023-06-10T21:31:11.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322962+0000", 2023-06-10T21:31:11.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2969999999999997e-06 2023-06-10T21:31:11.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322966+0000", 2023-06-10T21:31:11.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111656 2023-06-10T21:31:11.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323078+0000", 2023-06-10T21:31:11.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5869999999999998e-06 2023-06-10T21:31:11.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323084+0000", 2023-06-10T21:31:11.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0103796089999999 2023-06-10T21:31:11.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333464+0000", 2023-06-10T21:31:11.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0339000000000004e-05 2023-06-10T21:31:11.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333544+0000", 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9686999999999996e-05 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333644+0000", 2023-06-10T21:31:11.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054681870000000002 2023-06-10T21:31:11.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339112+0000", 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7512000000000003e-05 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339150+0000", 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8546000000000001e-05 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339178+0000", 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3045e-05 2023-06-10T21:31:11.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339211+0000", 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4028e-05 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339245+0000", 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0162916630000001 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.525 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166826 9.0 9:04443790:::benchmark_data_smithi064_247471_object57838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:31:11.525 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:31:11.525 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 446.13745364499999, 2023-06-10T21:31:11.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014370735, 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.526 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166826 2023-06-10T21:31:11.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.527 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334633+0000", 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2560000000000008e-06 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334642+0000", 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6484e-05 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334659+0000", 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9730000000000002e-06 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334665+0000", 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8026999999999999e-05 2023-06-10T21:31:11.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334723+0000", 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4628e-05 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334737+0000", 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0014236080000001 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336161+0000", 2023-06-10T21:31:11.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00050002300000000005 2023-06-10T21:31:11.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336661+0000", 2023-06-10T21:31:11.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105979 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336767+0000", 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024868210000000002 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339254+0000", 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096393050000000008 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348893+0000", 2023-06-10T21:31:11.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5008e-05 2023-06-10T21:31:11.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348938+0000", 2023-06-10T21:31:11.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.243e-05 2023-06-10T21:31:11.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348970+0000", 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4765999999999999e-05 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349005+0000", 2023-06-10T21:31:11.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014370735 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.540 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166843 9.0 9:06ac37a2:::benchmark_data_smithi064_247471_object57855:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 446.10850461400003, 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98562871200000002, 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166843 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:31:11.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363582+0000", 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3459999999999999e-06 2023-06-10T21:31:11.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363588+0000", 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7246000000000001e-05 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363606+0000", 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4109999999999999e-06 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363613+0000", 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2362999999999999e-05 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363675+0000", 2023-06-10T21:31:11.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1939e-05 2023-06-10T21:31:11.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363687+0000", 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97560709499999998 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339295+0000", 2023-06-10T21:31:11.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0881000000000001e-05 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339365+0000", 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7531000000000001e-05 2023-06-10T21:31:11.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339433+0000", 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094019989999999994 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348835+0000", 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022852899999999999 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349063+0000", 2023-06-10T21:31:11.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0859000000000001e-05 2023-06-10T21:31:11.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349124+0000", 2023-06-10T21:31:11.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1295999999999998e-05 2023-06-10T21:31:11.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349176+0000", 2023-06-10T21:31:11.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6529000000000002e-05 2023-06-10T21:31:11.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349212+0000", 2023-06-10T21:31:11.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98562871200000002 2023-06-10T21:31:11.557 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.558 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.558 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.559 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166894 9.0 9:058fc9da:::benchmark_data_smithi064_247471_object57906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:31:11.559 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:31:11.559 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 445.901899722, 2023-06-10T21:31:11.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78374433300000002, 2023-06-10T21:31:11.560 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.560 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.560 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.560 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.561 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.561 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166894 2023-06-10T21:31:11.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.561 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570186+0000", 2023-06-10T21:31:11.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.789e-06 2023-06-10T21:31:11.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570195+0000", 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9400000000000001e-05 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570215+0000", 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5470000000000002e-06 2023-06-10T21:31:11.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570221+0000", 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2519000000000003e-05 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570274+0000", 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8919999999999995e-06 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570284+0000", 2023-06-10T21:31:11.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77249051099999999 2023-06-10T21:31:11.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.342774+0000", 2023-06-10T21:31:11.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013168899999999999 2023-06-10T21:31:11.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.342906+0000", 2023-06-10T21:31:11.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000158128 2023-06-10T21:31:11.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.343064+0000", 2023-06-10T21:31:11.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059511130000000001 2023-06-10T21:31:11.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349015+0000", 2023-06-10T21:31:11.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041870379999999997 2023-06-10T21:31:11.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353202+0000", 2023-06-10T21:31:11.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062741800000000003 2023-06-10T21:31:11.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353829+0000", 2023-06-10T21:31:11.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9527000000000001e-05 2023-06-10T21:31:11.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353889+0000", 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3646999999999997e-05 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353933+0000", 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78374433300000002 2023-06-10T21:31:11.576 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166905 9.0 9:01f9f1d0:::benchmark_data_smithi064_247471_object57917:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:31:11.577 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 445.88077597900002, 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76680837999999996, 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.578 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166905 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:31:11.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:31:11.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591311+0000", 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3470000000000004e-06 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591317+0000", 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1277000000000001e-05 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591328+0000", 2023-06-10T21:31:11.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1000000000000003e-06 2023-06-10T21:31:11.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591333+0000", 2023-06-10T21:31:11.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1403999999999999e-05 2023-06-10T21:31:11.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591375+0000", 2023-06-10T21:31:11.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.636e-06 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591383+0000", 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75494583900000001 2023-06-10T21:31:11.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346329+0000", 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6804000000000005e-05 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346406+0000", 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8774000000000002e-05 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346485+0000", 2023-06-10T21:31:11.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027402619999999998 2023-06-10T21:31:11.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349225+0000", 2023-06-10T21:31:11.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047691249999999999 2023-06-10T21:31:11.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353994+0000", 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040310190000000003 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358025+0000", 2023-06-10T21:31:11.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9750000000000003e-05 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358075+0000", 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5550999999999998e-05 2023-06-10T21:31:11.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358120+0000", 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76680837999999996 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199852 9.0 9:0219fd3c:::benchmark_data_smithi064_247471_object90864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 389.93949747900001, 2023-06-10T21:31:11.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30757901700000001, 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:31:11.595 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199852 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:31:11.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532589+0000", 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2009999999999999e-06 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532597+0000", 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5182e-05 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532612+0000", 2023-06-10T21:31:11.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5780000000000004e-06 2023-06-10T21:31:11.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532619+0000", 2023-06-10T21:31:11.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8724999999999999e-05 2023-06-10T21:31:11.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532658+0000", 2023-06-10T21:31:11.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118538 2023-06-10T21:31:11.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532776+0000", 2023-06-10T21:31:11.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119059 2023-06-10T21:31:11.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:11.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532895+0000", 2023-06-10T21:31:11.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041169240000000001 2023-06-10T21:31:11.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.537012+0000", 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013852160000000001 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.550864+0000", 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28918148999999999 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840046+0000", 2023-06-10T21:31:11.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6958000000000004e-05 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840123+0000", 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6752000000000002e-05 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840170+0000", 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30757901700000001 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13226 10.a 10:57a745d6:::benchmark_data_smithi064_312713_object13225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.22271442300001, 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.416610181, 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.609 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13226 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:31:11.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249372+0000", 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015133799999999999 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249524+0000", 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0832000000000001e-05 2023-06-10T21:31:11.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249534+0000", 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6910000000000002e-06 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249540+0000", 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000273119 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249813+0000", 2023-06-10T21:31:11.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114503 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249928+0000", 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014703399999999999 2023-06-10T21:31:11.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.250075+0000", 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00083144899999999995 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.250906+0000", 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00097972100000000002 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.251886+0000", 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014250977999999999 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266137+0000", 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0433e-05 2023-06-10T21:31:11.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266147+0000", 2023-06-10T21:31:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36071926900000001 2023-06-10T21:31:11.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.626867+0000", 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1214e-05 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.626878+0000", 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0389994629999999 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665877+0000", 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106587 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665984+0000", 2023-06-10T21:31:11.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.416610181 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.624 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13230 10.a 10:56398bc8:::benchmark_data_smithi064_312713_object13229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.21952023599999, 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.413825726, 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.625 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13230 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:31:11.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:11.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252567+0000", 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8826999999999999e-05 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252606+0000", 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3930000000000008e-06 2023-06-10T21:31:11.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252614+0000", 2023-06-10T21:31:11.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5149999999999997e-06 2023-06-10T21:31:11.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252620+0000", 2023-06-10T21:31:11.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0380000000000003e-05 2023-06-10T21:31:11.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252660+0000", 2023-06-10T21:31:11.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102132 2023-06-10T21:31:11.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252762+0000", 2023-06-10T21:31:11.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015171800000000001 2023-06-10T21:31:11.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:31:11.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252914+0000", 2023-06-10T21:31:11.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031348970000000002 2023-06-10T21:31:11.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.256049+0000", 2023-06-10T21:31:11.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001809043 2023-06-10T21:31:11.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.257858+0000", 2023-06-10T21:31:11.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083186690000000008 2023-06-10T21:31:11.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266177+0000", 2023-06-10T21:31:11.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1099999999999997e-06 2023-06-10T21:31:11.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266184+0000", 2023-06-10T21:31:11.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36171270300000002 2023-06-10T21:31:11.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627896+0000", 2023-06-10T21:31:11.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.901e-06 2023-06-10T21:31:11.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627905+0000", 2023-06-10T21:31:11.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0384316309999999 2023-06-10T21:31:11.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666337+0000", 2023-06-10T21:31:11.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6521999999999999e-05 2023-06-10T21:31:11.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666394+0000", 2023-06-10T21:31:11.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.413825726 2023-06-10T21:31:11.648 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.648 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.648 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13234 10.a 10:5db0cd49:::benchmark_data_smithi064_312713_object13233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.21706951700003, 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4210149110000001, 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.649 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13234 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.650 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255017+0000", 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8182000000000001e-05 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255055+0000", 2023-06-10T21:31:11.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9320000000000008e-06 2023-06-10T21:31:11.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255065+0000", 2023-06-10T21:31:11.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.468e-06 2023-06-10T21:31:11.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255070+0000", 2023-06-10T21:31:11.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4666999999999998e-05 2023-06-10T21:31:11.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255115+0000", 2023-06-10T21:31:11.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012085 2023-06-10T21:31:11.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255236+0000", 2023-06-10T21:31:11.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144413 2023-06-10T21:31:11.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:31:11.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255380+0000", 2023-06-10T21:31:11.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00088783000000000004 2023-06-10T21:31:11.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.256268+0000", 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017292430000000001 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.257997+0000", 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082125259999999995 2023-06-10T21:31:11.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266210+0000", 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8719999999999996e-06 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266217+0000", 2023-06-10T21:31:11.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36172231999999999 2023-06-10T21:31:11.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627939+0000", 2023-06-10T21:31:11.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3130000000000001e-06 2023-06-10T21:31:11.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627947+0000", 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0384744699999999 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666421+0000", 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00080771 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667229+0000", 2023-06-10T21:31:11.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087343360000000005 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.675963+0000", 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0401999999999994e-05 2023-06-10T21:31:11.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676033+0000", 2023-06-10T21:31:11.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4210149110000001 2023-06-10T21:31:11.668 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13237 10.a 10:52604c11:::benchmark_data_smithi064_312713_object13236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:31:11.669 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.21338833999999, 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.417774082, 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.670 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13237 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:31:11.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:11.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258699+0000", 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2926999999999998e-05 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258732+0000", 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4180000000000003e-06 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258741+0000", 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9060000000000001e-06 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258746+0000", 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074915210000000001 2023-06-10T21:31:11.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266238+0000", 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2489999999999998e-06 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266244+0000", 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36174977400000002 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627994+0000", 2023-06-10T21:31:11.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9409999999999996e-06 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628003+0000", 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0392761150000001 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667279+0000", 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118165 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667397+0000", 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121712 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667519+0000", 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002026927 2023-06-10T21:31:11.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669546+0000", 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053423530000000002 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674888+0000", 2023-06-10T21:31:11.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011963340000000001 2023-06-10T21:31:11.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676084+0000", 2023-06-10T21:31:11.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.015e-05 2023-06-10T21:31:11.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676125+0000", 2023-06-10T21:31:11.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034917100000000001 2023-06-10T21:31:11.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676474+0000", 2023-06-10T21:31:11.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.417774082 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.686 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13245 10.a 10:577fa6ae:::benchmark_data_smithi064_312713_object13244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.20401137900001, 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4089718099999999, 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.688 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.688 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13245 2023-06-10T21:31:11.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.689 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:31:11.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:31:11.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268076+0000", 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0794999999999998e-05 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268116+0000", 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.151e-05 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268128+0000", 2023-06-10T21:31:11.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5259999999999999e-06 2023-06-10T21:31:11.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268134+0000", 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00186199 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.269996+0000", 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9990000000000003e-06 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.270006+0000", 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35802179899999997 2023-06-10T21:31:11.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628028+0000", 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9039999999999992e-06 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628036+0000", 2023-06-10T21:31:11.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040087349 2023-06-10T21:31:11.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668124+0000", 2023-06-10T21:31:11.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014886000000000001 2023-06-10T21:31:11.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668272+0000", 2023-06-10T21:31:11.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021805899999999999 2023-06-10T21:31:11.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:31:11.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668490+0000", 2023-06-10T21:31:11.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001063549 2023-06-10T21:31:11.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669554+0000", 2023-06-10T21:31:11.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053829209999999997 2023-06-10T21:31:11.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674937+0000", 2023-06-10T21:31:11.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001595632 2023-06-10T21:31:11.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676533+0000", 2023-06-10T21:31:11.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8981000000000003e-05 2023-06-10T21:31:11.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676592+0000", 2023-06-10T21:31:11.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045696000000000002 2023-06-10T21:31:11.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677048+0000", 2023-06-10T21:31:11.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4089718099999999 2023-06-10T21:31:11.710 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.710 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.710 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13248 10.7 10:e5a76f8f:::benchmark_data_smithi064_312713_object13247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.20022191200002, 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3988524149999999, 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.711 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.712 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.712 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.712 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.712 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13248 2023-06-10T21:31:11.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.713 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:31:11.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:31:11.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271864+0000", 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104888 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271969+0000", 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8029999999999998e-05 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271987+0000", 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8689999999999998e-06 2023-06-10T21:31:11.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271994+0000", 2023-06-10T21:31:11.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031337380000000001 2023-06-10T21:31:11.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.275128+0000", 2023-06-10T21:31:11.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1113e-05 2023-06-10T21:31:11.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.275139+0000", 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35235471299999999 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627494+0000", 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2689e-05 2023-06-10T21:31:11.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627506+0000", 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.036269895 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.663776+0000", 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012765699999999999 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.663904+0000", 2023-06-10T21:31:11.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000138367 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.664042+0000", 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035549879999999998 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667597+0000", 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120649 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667718+0000", 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023842529999999998 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670102+0000", 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4106000000000001e-05 2023-06-10T21:31:11.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670146+0000", 2023-06-10T21:31:11.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00057218899999999997 2023-06-10T21:31:11.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670719+0000", 2023-06-10T21:31:11.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3988524149999999 2023-06-10T21:31:11.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.729 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13271 10.a 10:5728c7b4:::benchmark_data_smithi064_312713_object13270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.18554411600002, 2023-06-10T21:31:11.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3910280420000001, 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13271 2023-06-10T21:31:11.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.732 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:31:11.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:31:11.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:31:11.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286543+0000", 2023-06-10T21:31:11.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.872e-05 2023-06-10T21:31:11.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286602+0000", 2023-06-10T21:31:11.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3389999999999993e-06 2023-06-10T21:31:11.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286610+0000", 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8359999999999999e-06 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286615+0000", 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2231999999999996e-05 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286677+0000", 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3670000000000003e-06 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286686+0000", 2023-06-10T21:31:11.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.343093494 2023-06-10T21:31:11.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.629780+0000", 2023-06-10T21:31:11.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6810000000000004e-06 2023-06-10T21:31:11.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.629789+0000", 2023-06-10T21:31:11.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0391585459999999 2023-06-10T21:31:11.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668947+0000", 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012422299999999999 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669071+0000", 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121916 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669193+0000", 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053272599999999997 2023-06-10T21:31:11.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669726+0000", 2023-06-10T21:31:11.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052600779999999996 2023-06-10T21:31:11.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674986+0000", 2023-06-10T21:31:11.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002138872 2023-06-10T21:31:11.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677125+0000", 2023-06-10T21:31:11.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4203999999999998e-05 2023-06-10T21:31:11.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677179+0000", 2023-06-10T21:31:11.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039273699999999999 2023-06-10T21:31:11.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677572+0000", 2023-06-10T21:31:11.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3910280420000001 2023-06-10T21:31:11.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.753 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13274 10.7 10:e090e432:::benchmark_data_smithi064_312713_object13273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.18305808299999, 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.384448001, 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.754 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13274 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6611999999999999e-05 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289056+0000", 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4040000000000003e-06 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289062+0000", 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1099999999999999e-06 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289065+0000", 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7024999999999997e-05 2023-06-10T21:31:11.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289122+0000", 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0210000000000001e-05 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289132+0000", 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33959756699999999 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628729+0000", 2023-06-10T21:31:11.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9420000000000007e-06 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628739+0000", 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0367496140000001 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665489+0000", 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016709400000000001 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665656+0000", 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037900500000000002 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666035+0000", 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016913609999999999 2023-06-10T21:31:11.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667726+0000", 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030370800000000002 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670764+0000", 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022778709999999999 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673041+0000", 2023-06-10T21:31:11.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9855999999999997e-05 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673101+0000", 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000376707 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673478+0000", 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.384448001 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13276 10.7 10:e43325ff:::benchmark_data_smithi064_312713_object13275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 326.18046167400001, 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.383298761, 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:31:11.769 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13276 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:31:11.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:31:11.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291625+0000", 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8028000000000007e-05 2023-06-10T21:31:11.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291693+0000", 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5150999999999999e-05 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:31:11.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291708+0000", 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3890000000000002e-06 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291715+0000", 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0940000000000003e-05 2023-06-10T21:31:11.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291775+0000", 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3091000000000001e-05 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291789+0000", 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33697318300000001 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628762+0000", 2023-06-10T21:31:11.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2109999999999998e-06 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628770+0000", 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.037982368 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666752+0000", 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013466800000000001 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:31:11.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666887+0000", 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019130800000000001 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667078+0000", 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065763300000000002 2023-06-10T21:31:11.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667736+0000", 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030568829999999998 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670793+0000", 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036619259999999998 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674455+0000", 2023-06-10T21:31:11.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6567000000000002e-05 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674511+0000", 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041383799999999998 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:31:11.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674925+0000", 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.383298761 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:31:11.783 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:31:11.784 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:31:11.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:11.787 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:31:11.788 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:31:11.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:11.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:11.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:12.014 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:12.044 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:12.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:31:12.102 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:12.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:12.151 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_6' created 2023-06-10T21:31:12.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:31:12.207 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:12.235 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:12.281 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:12.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:12.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:12.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:12 smithi064 ceph-mon[186706]: from='client.46208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:12.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:12 smithi064 ceph-mon[180297]: from='client.46208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:12.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:12 smithi152 ceph-mon[156819]: from='client.46208 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:12.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:12.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:12.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:12.517 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:12.533 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":962,"stamp":"2023-06-10T21:31:09.737333+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":1777180,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":231,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2671148,"kb_used_data":520764,"kb_used_omap":30,"kb_used_meta":2150305,"kb_avail":559799764,"statfs":{"total":575970213888,"available":573234958336,"internally_reserved":0,"allocated":533262336,"data_stored":528665007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2201912358},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":43,"apply_latency_ms":43,"commit_latency_ns":43000000,"apply_latency_ns":43000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-32562,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-181508,"num_write_kb":-5808193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-23369,"num_bytes_recovered":-1321598976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002450"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":260,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.272790+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:31:04.272790+0000","last_peered":"2023-06-10T21:31:04.272790+0000","last_clean":"2023-06-10T21:31:04.272790+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:31:04.272790+0000","last_undegraded":"2023-06-10T21:31:04.272790+0000","last_fullsized":"2023-06-10T21:31:04.272790+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":199,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178651+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:30:54.178651+0000","last_peered":"2023-06-10T21:30:54.178651+0000","last_clean":"2023-06-10T21:30:54.178651+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:30:54.178651+0000","last_undegraded":"2023-06-10T21:30:54.178651+0000","last_fullsized":"2023-06-10T21:30:54.178651+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177802+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:30:54.177802+0000","last_peered":"2023-06-10T21:30:54.177802+0000","last_clean":"2023-06-10T21:30:54.177802+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:30:54.177802+0000","last_undegraded":"2023-06-10T21:30:54.177802+0000","last_fullsized":"2023-06-10T21:30:54.177802+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":104,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.183568+0000","last_change":"2023-06-10T21:28:57.902734+0000","last_active":"2023-06-10T21:30:54.183568+0000","last_peered":"2023-06-10T21:30:54.183568+0000","last_clean":"2023-06-10T21:30:54.183568+0000","last_became_active":"2023-06-10T21:28:57.902279+0000","last_became_peered":"2023-06-10T21:28:57.902279+0000","last_unstale":"2023-06-10T21:30:54.183568+0000","last_undegraded":"2023-06-10T21:30:54.183568+0000","last_fullsized":"2023-06-10T21:30:54.183568+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:46.251698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":239,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177902+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:30:54.177902+0000","last_peered":"2023-06-10T21:30:54.177902+0000","last_clean":"2023-06-10T21:30:54.177902+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:30:54.177902+0000","last_undegraded":"2023-06-10T21:30:54.177902+0000","last_fullsized":"2023-06-10T21:30:54.177902+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":232,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.184247+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:30:54.184247+0000","last_peered":"2023-06-10T21:30:54.184247+0000","last_clean":"2023-06-10T21:30:54.184247+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:30:54.184247+0000","last_undegraded":"2023-06-10T21:30:54.184247+0000","last_fullsized":"2023-06-10T21:30:54.184247+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:57.827702+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:30:57.827702+0000","last_peered":"2023-06-10T21:30:57.827702+0000","last_clean":"2023-06-10T21:30:57.827702+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:30:57.827702+0000","last_undegraded":"2023-06-10T21:30:57.827702+0000","last_fullsized":"2023-06-10T21:30:57.827702+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":213,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177455+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:30:54.177455+0000","last_peered":"2023-06-10T21:30:54.177455+0000","last_clean":"2023-06-10T21:30:54.177455+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:30:54.177455+0000","last_undegraded":"2023-06-10T21:30:54.177455+0000","last_fullsized":"2023-06-10T21:30:54.177455+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":156,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585391+0000","last_change":"2023-06-10T21:28:57.899055+0000","last_active":"2023-06-10T21:30:54.585391+0000","last_peered":"2023-06-10T21:30:54.585391+0000","last_clean":"2023-06-10T21:30:54.585391+0000","last_became_active":"2023-06-10T21:28:57.898692+0000","last_became_peered":"2023-06-10T21:28:57.898692+0000","last_unstale":"2023-06-10T21:30:54.585391+0000","last_undegraded":"2023-06-10T21:30:54.585391+0000","last_fullsized":"2023-06-10T21:30:54.585391+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":192,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178001+0000","last_change":"2023-06-10T21:28:57.895548+0000","last_active":"2023-06-10T21:30:54.178001+0000","last_peered":"2023-06-10T21:30:54.178001+0000","last_clean":"2023-06-10T21:30:54.178001+0000","last_became_active":"2023-06-10T21:28:57.895217+0000","last_became_peered":"2023-06-10T21:28:57.895217+0000","last_unstale":"2023-06-10T21:30:54.178001+0000","last_undegraded":"2023-06-10T21:30:54.178001+0000","last_fullsized":"2023-06-10T21:30:54.178001+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:46:40.560609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":252,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585437+0000","last_change":"2023-06-10T21:28:57.899846+0000","last_active":"2023-06-10T21:30:54.585437+0000","last_peered":"2023-06-10T21:30:54.585437+0000","last_clean":"2023-06-10T21:30:54.585437+0000","last_became_active":"2023-06-10T21:28:57.898805+0000","last_became_peered":"2023-06-10T21:28:57.898805+0000","last_unstale":"2023-06-10T21:30:54.585437+0000","last_undegraded":"2023-06-10T21:30:54.585437+0000","last_fullsized":"2023-06-10T21:30:54.585437+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":229,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.149324+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:31:04.149324+0000","last_peered":"2023-06-10T21:31:04.149324+0000","last_clean":"2023-06-10T21:31:04.149324+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:31:04.149324+0000","last_undegraded":"2023-06-10T21:31:04.149324+0000","last_fullsized":"2023-06-10T21:31:04.149324+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":137,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585324+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:30:54.585324+0000","last_peered":"2023-06-10T21:30:54.585324+0000","last_clean":"2023-06-10T21:30:54.585324+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:30:54.585324+0000","last_undegraded":"2023-06-10T21:30:54.585324+0000","last_fullsized":"2023-06-10T21:30:54.585324+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.147745+0000","last_change":"2023-06-10T21:26:15.745053+0000","last_active":"2023-06-10T21:31:04.147745+0000","last_peered":"2023-06-10T21:31:04.147745+0000","last_clean":"2023-06-10T21:31:04.147745+0000","last_became_active":"2023-06-10T21:26:15.744736+0000","last_became_peered":"2023-06-10T21:26:15.744736+0000","last_unstale":"2023-06-10T21:31:04.147745+0000","last_undegraded":"2023-06-10T21:31:04.147745+0000","last_fullsized":"2023-06-10T21:31:04.147745+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:31:16.695327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585211+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:30:54.585211+0000","last_peered":"2023-06-10T21:30:54.585211+0000","last_clean":"2023-06-10T21:30:54.585211+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:30:54.585211+0000","last_undegraded":"2023-06-10T21:30:54.585211+0000","last_fullsized":"2023-06-10T21:30:54.585211+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178829+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:30:54.178829+0000","last_peered":"2023-06-10T21:30:54.178829+0000","last_clean":"2023-06-10T21:30:54.178829+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:30:54.178829+0000","last_undegraded":"2023-06-10T21:30:54.178829+0000","last_fullsized":"2023-06-10T21:30:54.178829+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":104,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.181469+0000","last_change":"2023-06-10T21:28:57.904420+0000","last_active":"2023-06-10T21:30:54.181469+0000","last_peered":"2023-06-10T21:30:54.181469+0000","last_clean":"2023-06-10T21:30:54.181469+0000","last_became_active":"2023-06-10T21:28:57.904027+0000","last_became_peered":"2023-06-10T21:28:57.904027+0000","last_unstale":"2023-06-10T21:30:54.181469+0000","last_undegraded":"2023-06-10T21:30:54.181469+0000","last_fullsized":"2023-06-10T21:30:54.181469+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:15:36.667597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585492+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:30:54.585492+0000","last_peered":"2023-06-10T21:30:54.585492+0000","last_clean":"2023-06-10T21:30:54.585492+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:30:54.585492+0000","last_undegraded":"2023-06-10T21:30:54.585492+0000","last_fullsized":"2023-06-10T21:30:54.585492+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":243,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585193+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:30:54.585193+0000","last_peered":"2023-06-10T21:30:54.585193+0000","last_clean":"2023-06-10T21:30:54.585193+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:30:54.585193+0000","last_undegraded":"2023-06-10T21:30:54.585193+0000","last_fullsized":"2023-06-10T21:30:54.585193+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":104,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:57.827620+0000","last_change":"2023-06-10T21:28:57.902565+0000","last_active":"2023-06-10T21:30:57.827620+0000","last_peered":"2023-06-10T21:30:57.827620+0000","last_clean":"2023-06-10T21:30:57.827620+0000","last_became_active":"2023-06-10T21:28:57.901902+0000","last_became_peered":"2023-06-10T21:28:57.901902+0000","last_unstale":"2023-06-10T21:30:57.827620+0000","last_undegraded":"2023-06-10T21:30:57.827620+0000","last_fullsized":"2023-06-10T21:30:57.827620+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:54.534642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":241,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.180613+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:30:54.180613+0000","last_peered":"2023-06-10T21:30:54.180613+0000","last_clean":"2023-06-10T21:30:54.180613+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:30:54.180613+0000","last_undegraded":"2023-06-10T21:30:54.180613+0000","last_fullsized":"2023-06-10T21:30:54.180613+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":199,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177957+0000","last_change":"2023-06-10T21:28:57.889771+0000","last_active":"2023-06-10T21:30:54.177957+0000","last_peered":"2023-06-10T21:30:54.177957+0000","last_clean":"2023-06-10T21:30:54.177957+0000","last_became_active":"2023-06-10T21:28:57.887036+0000","last_became_peered":"2023-06-10T21:28:57.887036+0000","last_unstale":"2023-06-10T21:30:54.177957+0000","last_undegraded":"2023-06-10T21:30:54.177957+0000","last_fullsized":"2023-06-10T21:30:54.177957+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:05.952139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":276,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585562+0000","last_change":"2023-06-10T21:28:34.522514+0000","last_active":"2023-06-10T21:30:54.585562+0000","last_peered":"2023-06-10T21:30:54.585562+0000","last_clean":"2023-06-10T21:30:54.585562+0000","last_became_active":"2023-06-10T21:28:34.522038+0000","last_became_peered":"2023-06-10T21:28:34.522038+0000","last_unstale":"2023-06-10T21:30:54.585562+0000","last_undegraded":"2023-06-10T21:30:54.585562+0000","last_fullsized":"2023-06-10T21:30:54.585562+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585029+0000","last_change":"2023-06-10T21:28:57.899217+0000","last_active":"2023-06-10T21:30:54.585029+0000","last_peered":"2023-06-10T21:30:54.585029+0000","last_clean":"2023-06-10T21:30:54.585029+0000","last_became_active":"2023-06-10T21:28:57.898541+0000","last_became_peered":"2023-06-10T21:28:57.898541+0000","last_unstale":"2023-06-10T21:30:54.585029+0000","last_undegraded":"2023-06-10T21:30:54.585029+0000","last_fullsized":"2023-06-10T21:30:54.585029+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":236,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585436+0000","last_change":"2023-06-10T21:14:42.439256+0000","last_active":"2023-06-10T21:30:54.585436+0000","last_peered":"2023-06-10T21:30:54.585436+0000","last_clean":"2023-06-10T21:30:54.585436+0000","last_became_active":"2023-06-10T21:14:42.438881+0000","last_became_peered":"2023-06-10T21:14:42.438881+0000","last_unstale":"2023-06-10T21:30:54.585436+0000","last_undegraded":"2023-06-10T21:30:54.585436+0000","last_fullsized":"2023-06-10T21:30:54.585436+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:05.356777+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:31:05.356777+0000","last_peered":"2023-06-10T21:31:05.356777+0000","last_clean":"2023-06-10T21:31:05.356777+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:31:05.356777+0000","last_undegraded":"2023-06-10T21:31:05.356777+0000","last_fullsized":"2023-06-10T21:31:05.356777+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":261,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585346+0000","last_change":"2023-06-10T21:28:57.900006+0000","last_active":"2023-06-10T21:30:54.585346+0000","last_peered":"2023-06-10T21:30:54.585346+0000","last_clean":"2023-06-10T21:30:54.585346+0000","last_became_active":"2023-06-10T21:28:57.898709+0000","last_became_peered":"2023-06-10T21:28:57.898709+0000","last_unstale":"2023-06-10T21:30:54.585346+0000","last_undegraded":"2023-06-10T21:30:54.585346+0000","last_fullsized":"2023-06-10T21:30:54.585346+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585072+0000","last_change":"2023-06-10T21:28:34.551591+0000","last_active":"2023-06-10T21:30:54.585072+0000","last_peered":"2023-06-10T21:30:54.585072+0000","last_clean":"2023-06-10T21:30:54.585072+0000","last_became_active":"2023-06-10T21:28:34.551254+0000","last_became_peered":"2023-06-10T21:28:34.551254+0000","last_unstale":"2023-06-10T21:30:54.585072+0000","last_undegraded":"2023-06-10T21:30:54.585072+0000","last_fullsized":"2023-06-10T21:30:54.585072+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178965+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:30:54.178965+0000","last_peered":"2023-06-10T21:30:54.178965+0000","last_clean":"2023-06-10T21:30:54.178965+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:30:54.178965+0000","last_undegraded":"2023-06-10T21:30:54.178965+0000","last_fullsized":"2023-06-10T21:30:54.178965+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":89,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.181750+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:30:54.181750+0000","last_peered":"2023-06-10T21:30:54.181750+0000","last_clean":"2023-06-10T21:30:54.181750+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:30:54.181750+0000","last_undegraded":"2023-06-10T21:30:54.181750+0000","last_fullsized":"2023-06-10T21:30:54.181750+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":216,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177818+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:30:54.177818+0000","last_peered":"2023-06-10T21:30:54.177818+0000","last_clean":"2023-06-10T21:30:54.177818+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:30:54.177818+0000","last_undegraded":"2023-06-10T21:30:54.177818+0000","last_fullsized":"2023-06-10T21:30:54.177818+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.179011+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:30:54.179011+0000","last_peered":"2023-06-10T21:30:54.179011+0000","last_clean":"2023-06-10T21:30:54.179011+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:30:54.179011+0000","last_undegraded":"2023-06-10T21:30:54.179011+0000","last_fullsized":"2023-06-10T21:30:54.179011+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":40,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:05.369229+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:31:05.369229+0000","last_peered":"2023-06-10T21:31:05.369229+0000","last_clean":"2023-06-10T21:31:05.369229+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:31:05.369229+0000","last_undegraded":"2023-06-10T21:31:05.369229+0000","last_fullsized":"2023-06-10T21:31:05.369229+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":258,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.148125+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:31:04.148125+0000","last_peered":"2023-06-10T21:31:04.148125+0000","last_clean":"2023-06-10T21:31:04.148125+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:31:04.148125+0000","last_undegraded":"2023-06-10T21:31:04.148125+0000","last_fullsized":"2023-06-10T21:31:04.148125+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":244,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.147939+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:31:04.147939+0000","last_peered":"2023-06-10T21:31:04.147939+0000","last_clean":"2023-06-10T21:31:04.147939+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:31:04.147939+0000","last_undegraded":"2023-06-10T21:31:04.147939+0000","last_fullsized":"2023-06-10T21:31:04.147939+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":39,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178872+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:30:54.178872+0000","last_peered":"2023-06-10T21:30:54.178872+0000","last_clean":"2023-06-10T21:30:54.178872+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:30:54.178872+0000","last_undegraded":"2023-06-10T21:30:54.178872+0000","last_fullsized":"2023-06-10T21:30:54.178872+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":202,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:05.356820+0000","last_change":"2023-06-10T21:28:57.894408+0000","last_active":"2023-06-10T21:31:05.356820+0000","last_peered":"2023-06-10T21:31:05.356820+0000","last_clean":"2023-06-10T21:31:05.356820+0000","last_became_active":"2023-06-10T21:28:57.893595+0000","last_became_peered":"2023-06-10T21:28:57.893595+0000","last_unstale":"2023-06-10T21:31:05.356820+0000","last_undegraded":"2023-06-10T21:31:05.356820+0000","last_fullsized":"2023-06-10T21:31:05.356820+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:32:03.720304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":175,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178316+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:30:54.178316+0000","last_peered":"2023-06-10T21:30:54.178316+0000","last_clean":"2023-06-10T21:30:54.178316+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:30:54.178316+0000","last_undegraded":"2023-06-10T21:30:54.178316+0000","last_fullsized":"2023-06-10T21:30:54.178316+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":89,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.182198+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:30:54.182198+0000","last_peered":"2023-06-10T21:30:54.182198+0000","last_clean":"2023-06-10T21:30:54.182198+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:30:54.182198+0000","last_undegraded":"2023-06-10T21:30:54.182198+0000","last_fullsized":"2023-06-10T21:30:54.182198+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":251,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:31:01.303198+0000","last_change":"2023-06-10T21:28:57.946656+0000","last_active":"2023-06-10T21:31:01.303198+0000","last_peered":"2023-06-10T21:31:01.303198+0000","last_clean":"2023-06-10T21:31:01.303198+0000","last_became_active":"2023-06-10T21:28:57.946186+0000","last_became_peered":"2023-06-10T21:28:57.946186+0000","last_unstale":"2023-06-10T21:31:01.303198+0000","last_undegraded":"2023-06-10T21:31:01.303198+0000","last_fullsized":"2023-06-10T21:31:01.303198+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:31.259349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":109,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.180955+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:30:54.180955+0000","last_peered":"2023-06-10T21:30:54.180955+0000","last_clean":"2023-06-10T21:30:54.180955+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:30:54.180955+0000","last_undegraded":"2023-06-10T21:30:54.180955+0000","last_fullsized":"2023-06-10T21:30:54.180955+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":135,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.180997+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:30:54.180997+0000","last_peered":"2023-06-10T21:30:54.180997+0000","last_clean":"2023-06-10T21:30:54.180997+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:30:54.180997+0000","last_undegraded":"2023-06-10T21:30:54.180997+0000","last_fullsized":"2023-06-10T21:30:54.180997+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":209,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.181518+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:30:54.181518+0000","last_peered":"2023-06-10T21:30:54.181518+0000","last_clean":"2023-06-10T21:30:54.181518+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:30:54.181518+0000","last_undegraded":"2023-06-10T21:30:54.181518+0000","last_fullsized":"2023-06-10T21:30:54.181518+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.148241+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:31:04.148241+0000","last_peered":"2023-06-10T21:31:04.148241+0000","last_clean":"2023-06-10T21:31:04.148241+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:31:04.148241+0000","last_undegraded":"2023-06-10T21:31:04.148241+0000","last_fullsized":"2023-06-10T21:31:04.148241+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":110,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.148029+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:31:04.148029+0000","last_peered":"2023-06-10T21:31:04.148029+0000","last_clean":"2023-06-10T21:31:04.148029+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:31:04.148029+0000","last_undegraded":"2023-06-10T21:31:04.148029+0000","last_fullsized":"2023-06-10T21:31:04.148029+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.147868+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:31:04.147868+0000","last_peered":"2023-06-10T21:31:04.147868+0000","last_clean":"2023-06-10T21:31:04.147868+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:31:04.147868+0000","last_undegraded":"2023-06-10T21:31:04.147868+0000","last_fullsized":"2023-06-10T21:31:04.147868+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":140,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.184195+0000","last_change":"2023-06-10T21:28:34.518545+0000","last_active":"2023-06-10T21:30:54.184195+0000","last_peered":"2023-06-10T21:30:54.184195+0000","last_clean":"2023-06-10T21:30:54.184195+0000","last_became_active":"2023-06-10T21:28:34.517742+0000","last_became_peered":"2023-06-10T21:28:34.517742+0000","last_unstale":"2023-06-10T21:30:54.184195+0000","last_undegraded":"2023-06-10T21:30:54.184195+0000","last_fullsized":"2023-06-10T21:30:54.184195+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:32:13.761246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":272,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.147915+0000","last_change":"2023-06-10T21:28:57.907800+0000","last_active":"2023-06-10T21:31:04.147915+0000","last_peered":"2023-06-10T21:31:04.147915+0000","last_clean":"2023-06-10T21:31:04.147915+0000","last_became_active":"2023-06-10T21:28:57.907509+0000","last_became_peered":"2023-06-10T21:28:57.907509+0000","last_unstale":"2023-06-10T21:31:04.147915+0000","last_undegraded":"2023-06-10T21:31:04.147915+0000","last_fullsized":"2023-06-10T21:31:04.147915+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:56.001988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":554,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585008+0000","last_change":"2023-06-10T21:28:34.503678+0000","last_active":"2023-06-10T21:30:54.585008+0000","last_peered":"2023-06-10T21:30:54.585008+0000","last_clean":"2023-06-10T21:30:54.585008+0000","last_became_active":"2023-06-10T21:28:34.503254+0000","last_became_peered":"2023-06-10T21:28:34.503254+0000","last_unstale":"2023-06-10T21:30:54.585008+0000","last_undegraded":"2023-06-10T21:30:54.585008+0000","last_fullsized":"2023-06-10T21:30:54.585008+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":203,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:31:01.303133+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:31:01.303133+0000","last_peered":"2023-06-10T21:31:01.303133+0000","last_clean":"2023-06-10T21:31:01.303133+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:31:01.303133+0000","last_undegraded":"2023-06-10T21:31:01.303133+0000","last_fullsized":"2023-06-10T21:31:01.303133+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585459+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:30:54.585459+0000","last_peered":"2023-06-10T21:30:54.585459+0000","last_clean":"2023-06-10T21:30:54.585459+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:30:54.585459+0000","last_undegraded":"2023-06-10T21:30:54.585459+0000","last_fullsized":"2023-06-10T21:30:54.585459+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":286,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.584933+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:30:54.584933+0000","last_peered":"2023-06-10T21:30:54.584933+0000","last_clean":"2023-06-10T21:30:54.584933+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:30:54.584933+0000","last_undegraded":"2023-06-10T21:30:54.584933+0000","last_fullsized":"2023-06-10T21:30:54.584933+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.180798+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:30:54.180798+0000","last_peered":"2023-06-10T21:30:54.180798+0000","last_clean":"2023-06-10T21:30:54.180798+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:30:54.180798+0000","last_undegraded":"2023-06-10T21:30:54.180798+0000","last_fullsized":"2023-06-10T21:30:54.180798+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":89,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.183316+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:30:54.183316+0000","last_peered":"2023-06-10T21:30:54.183316+0000","last_clean":"2023-06-10T21:30:54.183316+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:30:54.183316+0000","last_undegraded":"2023-06-10T21:30:54.183316+0000","last_fullsized":"2023-06-10T21:30:54.183316+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":271,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585378+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:30:54.585378+0000","last_peered":"2023-06-10T21:30:54.585378+0000","last_clean":"2023-06-10T21:30:54.585378+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:30:54.585378+0000","last_undegraded":"2023-06-10T21:30:54.585378+0000","last_fullsized":"2023-06-10T21:30:54.585378+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.148159+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:31:04.148159+0000","last_peered":"2023-06-10T21:31:04.148159+0000","last_clean":"2023-06-10T21:31:04.148159+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:31:04.148159+0000","last_undegraded":"2023-06-10T21:31:04.148159+0000","last_fullsized":"2023-06-10T21:31:04.148159+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":85,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.181602+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:30:54.181602+0000","last_peered":"2023-06-10T21:30:54.181602+0000","last_clean":"2023-06-10T21:30:54.181602+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:30:54.181602+0000","last_undegraded":"2023-06-10T21:30:54.181602+0000","last_fullsized":"2023-06-10T21:30:54.181602+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":217,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:05.369867+0000","last_change":"2023-06-10T21:28:57.899002+0000","last_active":"2023-06-10T21:31:05.369867+0000","last_peered":"2023-06-10T21:31:05.369867+0000","last_clean":"2023-06-10T21:31:05.369867+0000","last_became_active":"2023-06-10T21:28:57.898639+0000","last_became_peered":"2023-06-10T21:28:57.898639+0000","last_unstale":"2023-06-10T21:31:05.369867+0000","last_undegraded":"2023-06-10T21:31:05.369867+0000","last_fullsized":"2023-06-10T21:31:05.369867+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:12:01.141117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":40,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:05.369838+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:31:05.369838+0000","last_peered":"2023-06-10T21:31:05.369838+0000","last_clean":"2023-06-10T21:31:05.369838+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:31:05.369838+0000","last_undegraded":"2023-06-10T21:31:05.369838+0000","last_fullsized":"2023-06-10T21:31:05.369838+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":206,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:56.302428+0000","last_change":"2023-06-10T21:28:57.895650+0000","last_active":"2023-06-10T21:30:56.302428+0000","last_peered":"2023-06-10T21:30:56.302428+0000","last_clean":"2023-06-10T21:30:56.302428+0000","last_became_active":"2023-06-10T21:28:57.895320+0000","last_became_peered":"2023-06-10T21:28:57.895320+0000","last_unstale":"2023-06-10T21:30:56.302428+0000","last_undegraded":"2023-06-10T21:30:56.302428+0000","last_fullsized":"2023-06-10T21:30:56.302428+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:35:20.312084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":257,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585547+0000","last_change":"2023-06-10T21:28:57.899350+0000","last_active":"2023-06-10T21:30:54.585547+0000","last_peered":"2023-06-10T21:30:54.585547+0000","last_clean":"2023-06-10T21:30:54.585547+0000","last_became_active":"2023-06-10T21:28:57.898711+0000","last_became_peered":"2023-06-10T21:28:57.898711+0000","last_unstale":"2023-06-10T21:30:54.585547+0000","last_undegraded":"2023-06-10T21:30:54.585547+0000","last_fullsized":"2023-06-10T21:30:54.585547+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":93,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.183491+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:30:54.183491+0000","last_peered":"2023-06-10T21:30:54.183491+0000","last_clean":"2023-06-10T21:30:54.183491+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:30:54.183491+0000","last_undegraded":"2023-06-10T21:30:54.183491+0000","last_fullsized":"2023-06-10T21:30:54.183491+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":203,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.180702+0000","last_change":"2023-06-10T21:28:34.521765+0000","last_active":"2023-06-10T21:30:54.180702+0000","last_peered":"2023-06-10T21:30:54.180702+0000","last_clean":"2023-06-10T21:30:54.180702+0000","last_became_active":"2023-06-10T21:28:34.521409+0000","last_became_peered":"2023-06-10T21:28:34.521409+0000","last_unstale":"2023-06-10T21:30:54.180702+0000","last_undegraded":"2023-06-10T21:30:54.180702+0000","last_fullsized":"2023-06-10T21:30:54.180702+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:05:44.771841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":89,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:31:02.828261+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:31:02.828261+0000","last_peered":"2023-06-10T21:31:02.828261+0000","last_clean":"2023-06-10T21:31:02.828261+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:31:02.828261+0000","last_undegraded":"2023-06-10T21:31:02.828261+0000","last_fullsized":"2023-06-10T21:31:02.828261+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":108,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-06-10T21:31:04.148064+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:31:04.148064+0000","last_peered":"2023-06-10T21:31:04.148064+0000","last_clean":"2023-06-10T21:31:04.148064+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:31:04.148064+0000","last_undegraded":"2023-06-10T21:31:04.148064+0000","last_fullsized":"2023-06-10T21:31:04.148064+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":67,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.585124+0000","last_change":"2023-06-10T21:26:15.725760+0000","last_active":"2023-06-10T21:30:54.585124+0000","last_peered":"2023-06-10T21:30:54.585124+0000","last_clean":"2023-06-10T21:30:54.585124+0000","last_became_active":"2023-06-10T21:26:15.636720+0000","last_became_peered":"2023-06-10T21:26:15.636720+0000","last_unstale":"2023-06-10T21:30:54.585124+0000","last_undegraded":"2023-06-10T21:30:54.585124+0000","last_fullsized":"2023-06-10T21:30:54.585124+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":138,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177751+0000","last_change":"2023-06-10T21:28:57.898684+0000","last_active":"2023-06-10T21:30:54.177751+0000","last_peered":"2023-06-10T21:30:54.177751+0000","last_clean":"2023-06-10T21:30:54.177751+0000","last_became_active":"2023-06-10T21:28:57.898387+0000","last_became_peered":"2023-06-10T21:28:57.898387+0000","last_unstale":"2023-06-10T21:30:54.177751+0000","last_undegraded":"2023-06-10T21:30:54.177751+0000","last_fullsized":"2023-06-10T21:30:54.177751+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:55:36.556314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":43,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178492+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:30:54.178492+0000","last_peered":"2023-06-10T21:30:54.178492+0000","last_clean":"2023-06-10T21:30:54.178492+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:30:54.178492+0000","last_undegraded":"2023-06-10T21:30:54.178492+0000","last_fullsized":"2023-06-10T21:30:54.178492+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":203,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.178295+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:30:54.178295+0000","last_peered":"2023-06-10T21:30:54.178295+0000","last_clean":"2023-06-10T21:30:54.178295+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:30:54.178295+0000","last_undegraded":"2023-06-10T21:30:54.178295+0000","last_fullsized":"2023-06-10T21:30:54.178295+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":249,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:56.302662+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:30:56.302662+0000","last_peered":"2023-06-10T21:30:56.302662+0000","last_clean":"2023-06-10T21:30:56.302662+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:30:56.302662+0000","last_undegraded":"2023-06-10T21:30:56.302662+0000","last_fullsized":"2023-06-10T21:30:56.302662+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":252,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177917+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:30:54.177917+0000","last_peered":"2023-06-10T21:30:54.177917+0000","last_clean":"2023-06-10T21:30:54.177917+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:30:54.177917+0000","last_undegraded":"2023-06-10T21:30:54.177917+0000","last_fullsized":"2023-06-10T21:30:54.177917+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":154,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.184164+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:30:54.184164+0000","last_peered":"2023-06-10T21:30:54.184164+0000","last_clean":"2023-06-10T21:30:54.184164+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:30:54.184164+0000","last_undegraded":"2023-06-10T21:30:54.184164+0000","last_fullsized":"2023-06-10T21:30:54.184164+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":39,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-06-10T21:30:54.177882+0000","last_change":"2023-06-10T21:28:57.894606+0000","last_active":"2023-06-10T21:30:54.177882+0000","last_peered":"2023-06-10T21:30:54.177882+0000","last_clean":"2023-06-10T21:30:54.177882+0000","last_became_active":"2023-06-10T21:28:57.894004+0000","last_became_peered":"2023-06-10T21:28:57.894004+0000","last_unstale":"2023-06-10T21:30:54.177882+0000","last_undegraded":"2023-06-10T21:30:54.177882+0000","last_fullsized":"2023-06-10T21:30:54.177882+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:28:45.258464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":6076,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430609,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":523720,"kb_used_data":2944,"kb_used_omap":0,"kb_used_meta":520768,"kb_avail":93221432,"statfs":{"total":95995035648,"available":95458746368,"internally_reserved":0,"allocated":3014656,"data_stored":2463340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":533266432},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:36 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":8.0749999999999993,"15min":9.1630000000000003},"min":{"1min":0.23999999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.3239999999999998,"5min":89.245000000000005,"15min":89.245000000000005},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.327,"5min":8.3049999999999997,"15min":9.468},"min":{"1min":0.23899999999999999,"5min":0.223,"15min":0.223},"max":{"1min":4.5449999999999999,"5min":90.027000000000001,"15min":90.027000000000001},"last":0.58899999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:30:09 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.2069999999999999,"15min":2.8370000000000002},"min":{"1min":0.27100000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0629999999999997,"5min":93.093999999999994,"15min":130.19399999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.796,"15min":3.1589999999999998},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.673,"5min":93.119,"15min":212.20599999999999},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6040000000000001,"15min":2.3929999999999998},"min":{"1min":0.377,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.8129999999999997,"5min":160.99299999999999,"15min":160.99299999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.998,"5min":4.327,"15min":3.0030000000000001},"min":{"1min":0.317,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.0529999999999999,"5min":160.53700000000001,"15min":212.13399999999999},"last":0.622}]},{"osd":3,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":2.383,"15min":2.4609999999999999},"min":{"1min":0.38500000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":10.550000000000001,"5min":10.550000000000001,"15min":10.550000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.7769999999999999,"15min":1.8140000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.6480000000000001,"5min":4.242,"15min":4.242},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.65900000000000003,"15min":0.89400000000000002},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.45200000000000001,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.749,"15min":1.3129999999999999},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48599999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.373}]},{"osd":5,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.97799999999999998,"15min":2.145},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6280000000000001,"5min":12.048,"15min":211.99799999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.67000000000000004,"15min":1.946},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.56599999999999995,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.97899999999999998,"15min":1.294},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.8849999999999998,"5min":13.073,"15min":13.073},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.626,"15min":1.093},"min":{"1min":0.371,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.897,"5min":3.8239999999999998,"15min":10.874000000000001},"last":0.38400000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626837,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566279,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":447744,"kb_used_data":3256,"kb_used_omap":4,"kb_used_meta":444475,"kb_avail":93297408,"statfs":{"total":95995035648,"available":95536545792,"internally_reserved":0,"allocated":3334144,"data_stored":2497359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":455143310},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:38 2023","interfaces":[{"interface":"back","average":{"1min":6.3860000000000001,"5min":12.260999999999999,"15min":13.24},"min":{"1min":0.32600000000000001,"5min":0.221,"15min":0.221},"max":{"1min":42.567999999999998,"5min":107.825,"15min":107.825},"last":0.39200000000000002},{"interface":"front","average":{"1min":7.7030000000000003,"5min":12.196999999999999,"15min":12.946},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":42.372999999999998,"5min":110.928,"15min":110.928},"last":0.501}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":9.843,"5min":5.5069999999999997,"15min":2.8380000000000001},"min":{"1min":0.39100000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":107.39700000000001,"5min":107.857,"15min":107.857},"last":0.28000000000000003},{"interface":"front","average":{"1min":11.939,"5min":5.8049999999999997,"15min":2.637},"min":{"1min":0.61099999999999999,"5min":0.22500000000000001,"15min":0.218},"max":{"1min":106.907,"5min":107.958,"15min":107.958},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.1059999999999999,"15min":2.0190000000000001},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.9879999999999995,"5min":11.445,"15min":11.445},"last":0.41099999999999998},{"interface":"front","average":{"1min":5.1050000000000004,"5min":3.5659999999999998,"15min":3.3100000000000001},"min":{"1min":0.24299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":23.93,"5min":23.93,"15min":23.93},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1299999999999999,"5min":1.3839999999999999,"15min":1.71},"min":{"1min":0.28399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":16.059999999999999,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.641,"5min":1.03,"15min":1.5920000000000001},"min":{"1min":0.28299999999999997,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.8949999999999996,"5min":6.8949999999999996,"15min":14.460000000000001},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.4490000000000001,"15min":4.2670000000000003},"min":{"1min":0.40200000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":5.0700000000000003,"5min":6.9550000000000001,"15min":383.83100000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.389,"15min":4.1070000000000002},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.29699999999999999},"max":{"1min":16.146999999999998,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.387,"5min":1.905,"15min":2.1589999999999998},"min":{"1min":0.42799999999999999,"5min":0.34300000000000003,"15min":0.33800000000000002},"max":{"1min":6.6859999999999999,"5min":13.465,"15min":14.858000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":2,"5min":1.7270000000000001,"15min":1.956},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":11.476000000000001,"5min":13.48,"15min":13.48},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.6839999999999999,"15min":2.7810000000000001},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.32600000000000001},"max":{"1min":16.113,"5min":16.113,"15min":162.399},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.9490000000000001,"5min":1.3280000000000001,"15min":2.367},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9459999999999997,"5min":6.9459999999999997,"15min":162.27199999999999},"last":0.61299999999999999}]}]},{"osd":0,"up_from":244,"seq":1047972020256,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174872,"kb_used_data":3344,"kb_used_omap":10,"kb_used_meta":171509,"kb_avail":93570280,"statfs":{"total":95995035648,"available":95815966720,"internally_reserved":0,"allocated":3424256,"data_stored":2504434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":175625268},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":5.7510000000000003,"5min":4.6779999999999999,"15min":4.4989999999999997},"min":{"1min":0.32300000000000001,"5min":0.191,"15min":0.191},"max":{"1min":16.716999999999999,"5min":19.806999999999999,"15min":19.806999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":4.8090000000000002,"5min":5.8109999999999999,"15min":5.9779999999999998},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":12.032,"5min":29.841000000000001,"15min":29.841000000000001},"last":0.379}]},{"osd":2,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":12.528,"5min":7.2670000000000003,"15min":6.3899999999999997},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":205.10900000000001,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":14.632999999999999,"5min":7.0110000000000001,"15min":5.7409999999999997},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":208.34299999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:31:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":5.827,"15min":6.133},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":27.952999999999999,"5min":27.952999999999999,"15min":27.952999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":4.8230000000000004,"5min":6.7910000000000004,"15min":7.1189999999999998},"min":{"1min":0.32600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":17.184000000000001,"5min":32.822000000000003,"15min":32.822000000000003},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.4750000000000001,"5min":4.7720000000000002,"15min":4.9880000000000004},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.135999999999999,"5min":29.709,"15min":29.709},"last":0.495},{"interface":"front","average":{"1min":2.9809999999999999,"5min":4.016,"15min":4.1879999999999997},"min":{"1min":0.38100000000000001,"5min":0.308,"15min":0.308},"max":{"1min":13.065,"5min":13.872,"15min":13.872},"last":0.504}]},{"osd":5,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9940000000000002,"5min":4.1070000000000002,"15min":4.2930000000000001},"min":{"1min":0.39600000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":13.116,"5min":14.105,"15min":14.105},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.7440000000000002,"5min":3.6400000000000001,"15min":3.6219999999999999},"min":{"1min":0.47899999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.083,"5min":19.846,"15min":19.846},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.5270000000000001,"5min":4.9589999999999996,"15min":5.1970000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.18,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.0419999999999998,"5min":4.1459999999999999,"15min":4.3300000000000001},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":13.147,"5min":13.82,"15min":13.82},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:30:41 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":4.0179999999999998,"15min":4.0640000000000001},"min":{"1min":0.44400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.121,"5min":19.866,"15min":19.866},"last":0.54900000000000004},{"interface":"front","average":{"1min":3.5289999999999999,"5min":4.9569999999999999,"15min":5.1950000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":10.218999999999999,"5min":29.77,"15min":29.77},"last":0.51400000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075668,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325948,"kb_used_data":2676,"kb_used_omap":7,"kb_used_meta":323256,"kb_avail":93419204,"statfs":{"total":95995035648,"available":95661264896,"internally_reserved":0,"allocated":2740224,"data_stored":1907836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":331014225},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:37 2023","interfaces":[{"interface":"back","average":{"1min":5.0330000000000004,"5min":4.7690000000000001,"15min":4.7249999999999996},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":17.765999999999998,"5min":17.765999999999998,"15min":17.765999999999998},"last":0.25},{"interface":"front","average":{"1min":6.5289999999999999,"5min":5.6189999999999998,"15min":5.4669999999999996},"min":{"1min":0.54100000000000004,"5min":0.252,"15min":0.252},"max":{"1min":18.123999999999999,"5min":20.792999999999999,"15min":20.792999999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":5.0439999999999996,"5min":2.3580000000000001,"15min":1.71},"min":{"1min":0.35699999999999998,"5min":0.19,"15min":0.19},"max":{"1min":10.736000000000001,"5min":11.6,"15min":29.486000000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":3.5430000000000001,"5min":1.776,"15min":1.488},"min":{"1min":0.28899999999999998,"5min":0.214,"15min":0.214},"max":{"1min":11.336,"5min":14.334,"15min":28.013999999999999},"last":0.309}]},{"osd":3,"last update":"Sat Jun 10 21:30:57 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":4.79,"15min":5.0899999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":17.417999999999999,"5min":17.417999999999999,"15min":17.417999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.8359999999999999,"5min":5.6020000000000003,"15min":6.0629999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.6620000000000008,"5min":16.411000000000001,"15min":16.411000000000001},"last":0.32100000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.044,"5min":5.5250000000000004,"15min":2.6320000000000001},"min":{"1min":0.314,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":8.0969999999999995,"5min":224.00999999999999,"15min":224.00999999999999},"last":0.374},{"interface":"front","average":{"1min":4.6299999999999999,"5min":5.7709999999999999,"15min":2.7799999999999998},"min":{"1min":0.377,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":17.535,"5min":223.28399999999999,"15min":223.28399999999999},"last":0.443}]},{"osd":5,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":1.595,"15min":1.339},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":10.473000000000001,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":3.0609999999999999,"5min":1.788,"15min":1.542},"min":{"1min":0.442,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.1989999999999998,"5min":16.125,"15min":16.125},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0830000000000002,"5min":2.077,"15min":1.599},"min":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":8.1690000000000005,"5min":16.183,"15min":16.183},"last":0.54500000000000004},{"interface":"front","average":{"1min":4.3399999999999999,"5min":2.306,"15min":1.782},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":10.625999999999999,"5min":11.029,"15min":17.244},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":3.02,"15min":1.7330000000000001},"min":{"1min":0.40699999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":10.44,"5min":113.913,"15min":113.913},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.0609999999999999,"5min":3.3149999999999999,"15min":2.036},"min":{"1min":0.42799999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":8.141,"5min":113.937,"15min":113.937},"last":0.433}]}]},{"osd":3,"up_from":250,"seq":1073741824027,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":633872,"kb_used_data":506184,"kb_used_omap":7,"kb_used_meta":127672,"kb_avail":93111280,"statfs":{"total":95995035648,"available":95345950720,"internally_reserved":0,"allocated":518332416,"data_stored":517420074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":130736147},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":4.9409999999999998,"15min":5.3460000000000001},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":19.916,"5min":56.828000000000003,"15min":56.828000000000003},"last":0.23100000000000001},{"interface":"front","average":{"1min":1.881,"5min":4.6420000000000003,"15min":5.1020000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":9.1159999999999997,"5min":57.600000000000001,"15min":57.600000000000001},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":3.7090000000000001,"15min":3.6619999999999999},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":19.859000000000002,"5min":19.859000000000002,"15min":19.859000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.302,"5min":3.048,"15min":3.339},"min":{"1min":0.31900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.1219999999999999,"5min":8.2579999999999991,"15min":8.2579999999999991},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":10.744,"15min":12.196},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.218999999999999,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.27900000000000003},{"interface":"front","average":{"1min":4.758,"5min":12.426,"15min":13.704000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.832000000000001,"5min":80.566999999999993,"15min":80.566999999999993},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.397,"15min":1.381},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2330000000000005,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":3.484,"5min":2.8879999999999999,"15min":2.7879999999999998},"min":{"1min":0.372,"5min":0.311,"15min":0.311},"max":{"1min":19.728000000000002,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.69999999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.282,"15min":1.3560000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.2029999999999998,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.4079999999999999,"5min":1.425,"15min":1.4279999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":9.2599999999999998,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5190000000000001,"5min":2.915,"15min":2.8140000000000001},"min":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":19.805,"5min":19.805,"15min":19.805},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.349,"15min":1.409},"min":{"1min":0.443,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.2490000000000001,"5min":6.3449999999999998,"15min":6.3449999999999998},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":2.9249999999999998,"15min":2.827},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":19.777000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.3260000000000001,"15min":1.3839999999999999},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.2770000000000001,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.68700000000000006}]}]},{"osd":4,"up_from":30,"seq":128849019293,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":564992,"kb_used_data":2360,"kb_used_omap":0,"kb_used_meta":562624,"kb_avail":93180160,"statfs":{"total":95995035648,"available":95416483840,"internally_reserved":0,"allocated":2416640,"data_stored":1871964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":576126976},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":8.8949999999999996,"15min":10.083},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.8889999999999993,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":6.008,"5min":9.1530000000000005,"15min":9.6769999999999996},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":38.420000000000002,"5min":144.303,"15min":144.303},"last":0.57999999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:30:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.496,"15min":2.9900000000000002},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6,"5min":15.063000000000001,"15min":182.035},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.718,"15min":2.8940000000000001},"min":{"1min":0.34799999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.4760000000000009,"5min":15.052,"15min":181.73099999999999},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":2.859,"5min":2.0369999999999999,"15min":1.3919999999999999},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.319000000000001,"5min":22.614000000000001,"15min":22.614000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.9830000000000001,"5min":1.748,"15min":1.377},"min":{"1min":0.32300000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":18.329999999999998,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":2.262,"15min":2.3479999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8529999999999998,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.085,"15min":2.0569999999999999},"min":{"1min":0.35099999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3710000000000004,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:30:28 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.76200000000000001,"15min":1.1399999999999999},"min":{"1min":0.30299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.80000000000000004,"5min":6.742,"15min":13.941000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.434,"5min":0.88900000000000001,"15min":1.2070000000000001},"min":{"1min":0.311,"5min":0.247,"15min":0.247},"max":{"1min":0.85199999999999998,"5min":6.0819999999999999,"15min":11.904999999999999},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.82299999999999995,"15min":1.179},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.749,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.92300000000000004,"15min":1.2569999999999999},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":1.1990000000000001,"5min":8.3109999999999999,"15min":13.845000000000001},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.84599999999999997,"15min":1.393},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.65400000000000003,"5min":6.798,"15min":16.265000000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.91600000000000004,"15min":1.254},"min":{"1min":0.36799999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.80400000000000005,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":0.60899999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823064,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:31:12.553 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:31:12.554 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:31:12.554 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.6080948999873156 2023-06-10T21:31:12.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 0.6080948999873156 2023-06-10T21:31:12.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:12.626 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:12.719 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:12.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:12.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:31:12.824 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:12.858 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:12.953 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:12.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:13.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:13.090 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:13.099 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:12] "GET /metrics HTTP/1.1" 200 223691 "" "Prometheus/2.33.4" 2023-06-10T21:31:13.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-10T21:31:13.102 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[186706]: pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:13.104 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[186706]: failed to encode map e269 with expected crc 2023-06-10T21:31:13.105 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-10T21:31:13.106 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[186706]: osdmap e269: 8 total, 8 up, 6 in 2023-06-10T21:31:13.134 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:13.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:13.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-06-10T21:31:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-10T21:31:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:13 smithi152 ceph-mon[156819]: pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:13 smithi152 ceph-mon[156819]: failed to encode map e269 with expected crc 2023-06-10T21:31:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-10T21:31:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:13 smithi152 ceph-mon[156819]: osdmap e269: 8 total, 8 up, 6 in 2023-06-10T21:31:13.228 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:13.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:13.258 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:13.273 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:13.284 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:13.350 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:13.352 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-10T21:31:13.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[180297]: pgmap v963: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:13.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[180297]: failed to encode map e269 with expected crc 2023-06-10T21:31:13.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1080635073' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-10T21:31:13.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:13 smithi064 ceph-mon[180297]: osdmap e269: 8 total, 8 up, 6 in 2023-06-10T21:31:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:13.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:13.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:13.486 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:13.572 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:13.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:13.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:13.705 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:13.954 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:13.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:14.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:14.074 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:31:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:31:14.075 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:31:14.075 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:31:14.100 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:14.254 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: failed to encode map e269 with expected crc 2023-06-10T21:31:14.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: osdmap e270: 8 total, 8 up, 6 in 2023-06-10T21:31:14.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:14.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[186706]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: failed to encode map e269 with expected crc 2023-06-10T21:31:14.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: osdmap e270: 8 total, 8 up, 6 in 2023-06-10T21:31:14.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.256 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:14.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:14 smithi064 ceph-mon[180297]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: failed to encode map e269 with expected crc 2023-06-10T21:31:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: osdmap e270: 8 total, 8 up, 6 in 2023-06-10T21:31:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:14 smithi152 ceph-mon[156819]: failed to encode map e270 with expected crc 2023-06-10T21:31:14.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:31:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:15.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:15.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:15.099 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:15.106 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:15.136 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:15.218 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:15.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:15.323 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:15.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[180297]: failed to encode map e270 with expected crc 2023-06-10T21:31:15.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[180297]: pgmap v966: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:15.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[180297]: osdmap e271: 8 total, 8 up, 6 in 2023-06-10T21:31:15.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[180297]: failed to encode map e271 with expected crc 2023-06-10T21:31:15.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[186706]: failed to encode map e270 with expected crc 2023-06-10T21:31:15.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[186706]: pgmap v966: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:15.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[186706]: osdmap e271: 8 total, 8 up, 6 in 2023-06-10T21:31:15.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:15 smithi064 ceph-mon[186706]: failed to encode map e271 with expected crc 2023-06-10T21:31:15.353 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:15.369 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:15 smithi152 ceph-mon[156819]: failed to encode map e270 with expected crc 2023-06-10T21:31:15.370 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:15 smithi152 ceph-mon[156819]: pgmap v966: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:15.370 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:15 smithi152 ceph-mon[156819]: osdmap e271: 8 total, 8 up, 6 in 2023-06-10T21:31:15.370 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:15 smithi152 ceph-mon[156819]: failed to encode map e271 with expected crc 2023-06-10T21:31:15.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:15.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:16.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:16.051 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:16.145 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:16.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:16.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:16.284 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:16.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[186706]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[186706]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.300 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[186706]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[180297]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[180297]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:16 smithi064 ceph-mon[180297]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.349 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:16.355 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:16 smithi152 ceph-mon[156819]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.355 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:16 smithi152 ceph-mon[156819]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.355 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:16 smithi152 ceph-mon[156819]: failed to encode map e271 with expected crc 2023-06-10T21:31:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:16.383 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:16.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:16.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:16.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:16.552 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:31:16.552 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:31:16.552 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:31:16.553 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:31:16.553 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:31:16.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:16.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:16.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:16.868 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:16.897 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:17.135 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 0.608095 (9bac) 2023-06-10T21:31:17.136 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:31:17.143 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:17.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:17.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:17.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:31:17.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:17.275 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:17.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:31:17.319 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:31:17.324 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:17.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:17.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:17.339 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:31:17.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[186706]: pgmap v968: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:17.342 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]: dispatch 2023-06-10T21:31:17.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3088638667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[180297]: pgmap v968: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:17.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]: dispatch 2023-06-10T21:31:17.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3088638667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.359 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:31:17.377 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:31:17.396 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:31:17.413 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:31:17.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:17.432 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:31:17.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.434 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.434 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.434 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.435 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.435 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.436 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.436 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:31:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:17 smithi152 ceph-mon[156819]: pgmap v968: 89 pgs: 2 creating+peering, 6 unknown, 81 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]: dispatch 2023-06-10T21:31:17.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3088638667' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:17.465 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:17.562 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:31:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:31:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]': finished 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: osdmap e272: 8 total, 8 up, 6 in 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2320197678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:18.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[186706]: osdmap e273: 8 total, 8 up, 6 in 2023-06-10T21:31:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:31:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:31:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]': finished 2023-06-10T21:31:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: osdmap e272: 8 total, 8 up, 6 in 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2320197678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:18.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:18 smithi064 ceph-mon[180297]: osdmap e273: 8 total, 8 up, 6 in 2023-06-10T21:31:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:31:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:31:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2549836554' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.6080948999873156}]': finished 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: osdmap e272: 8 total, 8 up, 6 in 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: failed to encode map e272 with expected crc 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2320197678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:18 smithi152 ceph-mon[156819]: osdmap e273: 8 total, 8 up, 6 in 2023-06-10T21:31:19.122 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:19.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:19.140 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:19.141 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:19.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:19.260 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: failed to encode map e272 with expected crc 2023-06-10T21:31:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: pgmap v970: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[186706]: osdmap e274: 8 total, 8 up, 6 in 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: failed to encode map e272 with expected crc 2023-06-10T21:31:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: pgmap v970: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:19 smithi064 ceph-mon[180297]: osdmap e274: 8 total, 8 up, 6 in 2023-06-10T21:31:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: failed to encode map e272 with expected crc 2023-06-10T21:31:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: pgmap v970: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: failed to encode map e273 with expected crc 2023-06-10T21:31:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:19.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:19 smithi152 ceph-mon[156819]: osdmap e274: 8 total, 8 up, 6 in 2023-06-10T21:31:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:19.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:19.564 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:19.595 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:19.728 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:19.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:19.832 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:19.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:31:19.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:20.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:20.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:20.112 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:20.145 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:20.186 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:20.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:20.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:20.300 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:20.318 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:20.360 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:20.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:20 smithi064 ceph-mon[180297]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:20 smithi064 ceph-mon[180297]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:20 smithi064 ceph-mon[186706]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:20 smithi064 ceph-mon[186706]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:20 smithi152 ceph-mon[156819]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:20 smithi152 ceph-mon[156819]: failed to encode map e274 with expected crc 2023-06-10T21:31:20.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:20.492 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:20.534 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:20.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:20.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:20.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:20.871 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:20.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:20.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:21.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:21.047 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:21.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:21.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:21.154 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:21.161 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-06-10T21:31:21.183 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:21.224 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:21.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:21.333 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:21.358 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: pgmap v973: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:21.360 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.360 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.363 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:31:21.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: osdmap e275: 8 total, 8 up, 6 in 2023-06-10T21:31:21.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:31:21.365 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:21.401 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:21.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:21.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:21.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: pgmap v973: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:21.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: osdmap e275: 8 total, 8 up, 6 in 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: pgmap v973: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: failed to encode map e274 with expected crc 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: osdmap e275: 8 total, 8 up, 6 in 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: failed to encode map e275 with expected crc 2023-06-10T21:31:21.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:31:21.426 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:31:21.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:21.537 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:21.574 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:21.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:21.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:21.708 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:21.745 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:31:21.750 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:21.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:21.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:21.857 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:21.886 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:22.013 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:22.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:22.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:22.147 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:22.190 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:22.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:22.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:22.320 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:22.322 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:22.330 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:22.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:31:22.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[186706]: osdmap e276: 8 total, 8 up, 6 in 2023-06-10T21:31:22.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[186706]: failed to encode map e276 with expected crc 2023-06-10T21:31:22.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[186706]: failed to encode map e275 with expected crc 2023-06-10T21:31:22.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:31:22.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[180297]: osdmap e276: 8 total, 8 up, 6 in 2023-06-10T21:31:22.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[180297]: failed to encode map e276 with expected crc 2023-06-10T21:31:22.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:22 smithi064 ceph-mon[180297]: failed to encode map e275 with expected crc 2023-06-10T21:31:22.345 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:22.380 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:22.387 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:22.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:22.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2954473245' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:31:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:22 smithi152 ceph-mon[156819]: osdmap e276: 8 total, 8 up, 6 in 2023-06-10T21:31:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:22 smithi152 ceph-mon[156819]: failed to encode map e276 with expected crc 2023-06-10T21:31:22.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:22 smithi152 ceph-mon[156819]: failed to encode map e275 with expected crc 2023-06-10T21:31:22.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:22.538 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:23.020 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:23.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:23.042 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-06-10T21:31:23.065 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:31:23.066 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:31:23.113 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:31:23.114 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:31:23.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_385725 2023-06-10T21:31:23.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:31:23.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:31:23.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:22] "GET /metrics HTTP/1.1" 200 223672 "" "Prometheus/2.33.4" 2023-06-10T21:31:23.159 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:23.212 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:23.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:23.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:31:23.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:23.412 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:23.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[180297]: pgmap v976: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[180297]: failed to encode map e275 with expected crc 2023-06-10T21:31:23.428 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[180297]: failed to encode map e276 with expected crc 2023-06-10T21:31:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[186706]: pgmap v976: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:23.429 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[186706]: failed to encode map e275 with expected crc 2023-06-10T21:31:23.429 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:23 smithi064 ceph-mon[186706]: failed to encode map e276 with expected crc 2023-06-10T21:31:23.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:23 smithi152 ceph-mon[156819]: pgmap v976: 89 pgs: 8 peering, 1 unknown, 80 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail 2023-06-10T21:31:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:23 smithi152 ceph-mon[156819]: failed to encode map e275 with expected crc 2023-06-10T21:31:23.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:23 smithi152 ceph-mon[156819]: failed to encode map e276 with expected crc 2023-06-10T21:31:23.579 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:23.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:23.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:23.716 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:23.765 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:23.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:23.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:23.902 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:23.950 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:23.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:24.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:24.119 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2442 2426 151.569 151.625 0.00554251 0.00656534 2023-06-10T21:31:24.131 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:24.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:24 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:31:24.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:24 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:31:24.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:24 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:31:24.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:24 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:31:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:24 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-10T21:31:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:24 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:31:24.784 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:31:24.784 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:31:24.785 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-10T21:31:24.785 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-06-10T21:31:24.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 3 1 2023-06-10T21:31:25.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 4511 4495 140.429 129.312 0.00609081 0.00710136 2023-06-10T21:31:25.350 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:25.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:25.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:25.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:25.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:25 smithi064 ceph-mon[180297]: pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 102 KiB/s, 1 objects/s recovering 2023-06-10T21:31:25.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:25 smithi064 ceph-mon[186706]: pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 102 KiB/s, 1 objects/s recovering 2023-06-10T21:31:25.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:25.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:25 smithi152 ceph-mon[156819]: pgmap v977: 89 pgs: 89 active+clean; 582 KiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 102 KiB/s, 1 objects/s recovering 2023-06-10T21:31:26.114 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 6321 6305 131.325 113.125 0.00316777 0.007382 2023-06-10T21:31:26.728 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:26.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:26.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:26.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:26.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:27.053 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:27.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 8339 8323 130.022 126.125 0.00835617 0.00767567 2023-06-10T21:31:27.180 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:27.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:27.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:27.333 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:27.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:27.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:27.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:27.513 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:27 smithi152 ceph-mon[156819]: pgmap v978: 89 pgs: 89 active+clean; 83 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 12 MiB/s wr, 199 op/s; 87 KiB/s, 1 objects/s recovering 2023-06-10T21:31:27.516 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:27.567 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:27.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:27.608 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:27.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:27 smithi064 ceph-mon[186706]: pgmap v978: 89 pgs: 89 active+clean; 83 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 12 MiB/s wr, 199 op/s; 87 KiB/s, 1 objects/s recovering 2023-06-10T21:31:27.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:27 smithi064 ceph-mon[180297]: pgmap v978: 89 pgs: 89 active+clean; 83 MiB data, 2.6 GiB used, 534 GiB / 536 GiB avail; 12 MiB/s wr, 199 op/s; 87 KiB/s, 1 objects/s recovering 2023-06-10T21:31:27.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:27.702 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:27.750 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:27.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:27.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:27.886 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:27.931 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:27.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:28.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:28.102 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:28.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 10291 10275 128.416 122 0.00268759 0.00777436 2023-06-10T21:31:28.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:31:28.475 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:31:28.522 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:31:28.566 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:31:28.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:31:28.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:31:28.689 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:31:28.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:31:28.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:31:28.739 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:31:29.115 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 12098 12082 125.834 112.938 0.00662299 0.00793592 2023-06-10T21:31:29.233 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:29.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:29.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:29.271 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:31:29.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:29.373 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:29.420 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:29.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:29.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:29.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[186706]: pgmap v979: 89 pgs: 89 active+clean; 264 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 72 KiB/s, 0 objects/s recovering 2023-06-10T21:31:29.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3628640148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:29.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/453329709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:29.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:29.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[180297]: pgmap v979: 89 pgs: 89 active+clean; 264 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 72 KiB/s, 0 objects/s recovering 2023-06-10T21:31:29.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3628640148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/453329709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:29.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:29.600 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:29 smithi152 ceph-mon[156819]: pgmap v979: 89 pgs: 89 active+clean; 264 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 33 MiB/s wr, 525 op/s; 72 KiB/s, 0 objects/s recovering 2023-06-10T21:31:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3628640148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:29.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/453329709' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:29.708 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:31:30.116 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 14080 14064 125.553 123.875 0.0053619 0.007955 2023-06-10T21:31:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:31:30.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: osdmap e277: 8 total, 8 up, 6 in 2023-06-10T21:31:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[186706]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: osdmap e277: 8 total, 8 up, 6 in 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:30 smithi064 ceph-mon[180297]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:31:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: osdmap e277: 8 total, 8 up, 6 in 2023-06-10T21:31:30.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: failed to encode map e277 with expected crc 2023-06-10T21:31:30.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:30 smithi152 ceph-mon[156819]: failed to encode map e277 with expected crc 2023-06-10T21:31:31.116 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 16118 16102 125.763 127.375 0.00969483 0.00794329 2023-06-10T21:31:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:31 smithi064 ceph-mon[186706]: pgmap v981: 89 pgs: 89 active+clean; 607 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 67 KiB/s, 0 objects/s recovering 2023-06-10T21:31:31.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:31 smithi064 ceph-mon[180297]: pgmap v981: 89 pgs: 89 active+clean; 607 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 67 KiB/s, 0 objects/s recovering 2023-06-10T21:31:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:31 smithi152 ceph-mon[156819]: pgmap v981: 89 pgs: 89 active+clean; 607 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s; 67 KiB/s, 0 objects/s recovering 2023-06-10T21:31:31.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:31.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:31.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:31.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:32.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:32.111 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:32.116 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 17963 17947 124.601 115.312 0.00403516 0.00801911 2023-06-10T21:31:32.140 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.189 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:32.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:32.298 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:32.326 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.368 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:32.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:32.505 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.551 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:32.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:32.657 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:32.686 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.735 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:32.842 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:32.870 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:32.920 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:32.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:33.026 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:33.053 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:32] "GET /metrics HTTP/1.1" 200 223672 "" "Prometheus/2.33.4" 2023-06-10T21:31:33.057 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:33.117 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 19600 19584 122.361 102.312 0.00677309 0.00816615 2023-06-10T21:31:33.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: pgmap v982: 89 pgs: 89 active+clean; 872 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 58 KiB/s, 0 objects/s recovering 2023-06-10T21:31:33.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]: dispatch 2023-06-10T21:31:33.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]: dispatch 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]: dispatch 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:33.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: pgmap v982: 89 pgs: 89 active+clean; 872 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 58 KiB/s, 0 objects/s recovering 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]: dispatch 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]: dispatch 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]: dispatch 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:33.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: pgmap v982: 89 pgs: 89 active+clean; 872 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 58 KiB/s, 0 objects/s recovering 2023-06-10T21:31:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]: dispatch 2023-06-10T21:31:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]: dispatch 2023-06-10T21:31:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]: dispatch 2023-06-10T21:31:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:31:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:34.119 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 20586 20570 116.829 61.625 0.0134306 0.0085513 2023-06-10T21:31:34.435 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:34.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:34.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]': finished 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]': finished 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:31:34.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: osdmap e278: 8 total, 8 up, 6 in 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:34.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[186706]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]': finished 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]': finished 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:31:34.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: osdmap e278: 8 total, 8 up, 6 in 2023-06-10T21:31:34.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:34.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:34 smithi064 ceph-mon[180297]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.617 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [3, 2]}]': finished 2023-06-10T21:31:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 7]}]': finished 2023-06-10T21:31:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 7]}]': finished 2023-06-10T21:31:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:31:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:31:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: osdmap e278: 8 total, 8 up, 6 in 2023-06-10T21:31:34.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:34.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:34.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:34 smithi152 ceph-mon[156819]: failed to encode map e278 with expected crc 2023-06-10T21:31:35.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:35.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:35.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:35.119 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 21952 21936 114.197 85.375 0.00345939 0.00874949 2023-06-10T21:31:35.144 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:35.192 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:35.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:35.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:35.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:35.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:35.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: pgmap v984: 89 pgs: 89 active+clean; 947 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:31:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: osdmap e279: 8 total, 8 up, 6 in 2023-06-10T21:31:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:35 smithi152 ceph-mon[156819]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.571 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:35.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: pgmap v984: 89 pgs: 89 active+clean; 947 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:31:35.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: osdmap e279: 8 total, 8 up, 6 in 2023-06-10T21:31:35.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.582 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[186706]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:35.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:35.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: pgmap v984: 89 pgs: 89 active+clean; 947 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-10T21:31:35.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: osdmap e279: 8 total, 8 up, 6 in 2023-06-10T21:31:35.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:35 smithi064 ceph-mon[180297]: failed to encode map e279 with expected crc 2023-06-10T21:31:35.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:35.768 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:36.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:36.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:36.121 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:31:36.122 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:31:36.122 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:31:36.123 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:31:36.123 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:31:36.124 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 23439 23423 112.559 92.9375 0.00539752 0.00887417 2023-06-10T21:31:36.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:36.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:36.457 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:36.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:36.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:36.606 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:36.653 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:36.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:36.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:31:36.753 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:31:36.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:36 smithi152 ceph-mon[156819]: pgmap v986: 89 pgs: 1 peering, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:36 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:31:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3554560966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:36.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:36.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:37.019 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:37.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:37.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:37.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 24687 24671 110.088 78 0.0101519 0.00907648 2023-06-10T21:31:37.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:37.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[180297]: pgmap v986: 89 pgs: 1 peering, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:37.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:31:37.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3554560966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:37.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:37.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[186706]: pgmap v986: 89 pgs: 1 peering, 88 active+clean; 1.1 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:37.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:31:37.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3554560966' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:37.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:31:37.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:37.205 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:37.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:37.297 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-06-10T21:31:37.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:37.345 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:37.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:37.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:37.495 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:37.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:38.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 25621 25605 106.64 58.375 0.00969249 0.00936971 2023-06-10T21:31:38.390 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:38.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:31:38.394 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[186706]: osdmap e280: 8 total, 8 up, 6 in 2023-06-10T21:31:38.395 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[186706]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.395 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[186706]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:31:38.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[180297]: osdmap e280: 8 total, 8 up, 6 in 2023-06-10T21:31:38.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[180297]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:38 smithi064 ceph-mon[180297]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:38.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:38.499 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:38.528 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:31:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:38 smithi152 ceph-mon[156819]: osdmap e280: 8 total, 8 up, 6 in 2023-06-10T21:31:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:38 smithi152 ceph-mon[156819]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:38 smithi152 ceph-mon[156819]: failed to encode map e280 with expected crc 2023-06-10T21:31:38.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:31:38.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:38.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:38.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:39.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:39.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 26555 26539 103.619 58.375 0.0113808 0.00963974 2023-06-10T21:31:39.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:39.245 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:39.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[180297]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[180297]: pgmap v988: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 11 B/s, 0 objects/s recovering 2023-06-10T21:31:39.304 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[180297]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.304 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[186706]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.305 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:39.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:39.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:39.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:39.499 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:39.544 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:39.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:39.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[186706]: pgmap v988: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 11 B/s, 0 objects/s recovering 2023-06-10T21:31:39.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:39 smithi064 ceph-mon[186706]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:39.681 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:39 smithi152 ceph-mon[156819]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.682 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:39 smithi152 ceph-mon[156819]: pgmap v988: 89 pgs: 1 peering, 88 active+clean; 1.3 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s; 11 B/s, 0 objects/s recovering 2023-06-10T21:31:39.682 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:39 smithi152 ceph-mon[156819]: failed to encode map e280 with expected crc 2023-06-10T21:31:39.684 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:39.733 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:39.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:39.840 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:39.868 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:39.915 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:39.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:40.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:40.099 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:40.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 27827 27811 102.2 79.5 0.00520637 0.0097767 2023-06-10T21:31:40.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:40 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:31:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:40 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:31:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:40 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:31:40.759 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:40.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:40.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:40.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:41.047 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:41.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:41.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:41.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 29318 29302 101.699 93.1875 0.008804 0.00982686 2023-06-10T21:31:41.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:41.228 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:41.302 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:41.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:41.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:41.460 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:41.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:41.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:41.580 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:41 smithi064 ceph-mon[180297]: pgmap v989: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:41.583 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:41 smithi064 ceph-mon[186706]: pgmap v989: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:41.681 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:41.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:41 smithi152 ceph-mon[156819]: pgmap v989: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 10 B/s, 0 objects/s recovering 2023-06-10T21:31:41.709 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:41.758 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:41.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:41.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:41.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:41.892 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:41.937 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:41.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:42.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:42.072 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:42.115 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:42.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:42.121 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 30218 30202 99.3075 56.25 0.134935 0.0100463 2023-06-10T21:31:42.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:42.274 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:42.651 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:42.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:42.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:42.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:42.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:42.861 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:42.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:42.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:43.055 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:43.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:31:43.124052+0000 min lat: 0.00116152 max lat: 0.294538 avg lat: 0.0102226 2023-06-10T21:31:43.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:31:43.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 31318 31302 97.7599 68.75 0.00855772 0.0102226 2023-06-10T21:31:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:42] "GET /metrics HTTP/1.1" 200 223956 "" "Prometheus/2.33.4" 2023-06-10T21:31:43.403 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:43.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:43.516 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:43.543 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:43.590 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:43.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:43.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:43.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:43 smithi064 ceph-mon[180297]: pgmap v990: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 8 B/s, 0 objects/s recovering 2023-06-10T21:31:43.616 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:43 smithi064 ceph-mon[186706]: pgmap v990: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 8 B/s, 0 objects/s recovering 2023-06-10T21:31:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:43 smithi152 ceph-mon[156819]: pgmap v990: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s; 8 B/s, 0 objects/s recovering 2023-06-10T21:31:43.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:43.724 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:43.779 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:43.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:43.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:43.870 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:31:43.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:43.911 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:43.929 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:31:43.956 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:43.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:31:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:43.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:44.031 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:31:44.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:44.069 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:31:44.074 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:31:44.074 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:31:44.074 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-06-10T21:31:44.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:31:44.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:44.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:31:44.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 32477 32461 96.5543 72.4375 0.0276134 0.0103467 2023-06-10T21:31:44.140 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:44.141 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:31:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:44.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:44.182 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:31:44.211 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.212 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.213 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:31:44.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:44.329 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:44.453 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:44.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:44.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:44.595 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:44.636 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:44.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2880916827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:44.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:44.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2880916827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2880916827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:44.741 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:44.770 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:44.810 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:44.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:44.917 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:44.943 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:44.984 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:44.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:45.091 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:45.118 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:45.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 33296 33280 94.4928 51.1875 0.00179634 0.0105424 2023-06-10T21:31:45.159 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:45.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:45.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:45.293 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:45.341 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:45.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:45.423 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:31:45.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:45.476 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:45.523 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:45.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:45.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:45.727 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:45 smithi152 ceph-mon[156819]: pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-06-10T21:31:45.728 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:45.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:45 smithi064 ceph-mon[180297]: pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-06-10T21:31:45.729 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:45.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:45 smithi064 ceph-mon[186706]: pgmap v991: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 89 MiB/s wr, 1.42k op/s; 7 B/s, 0 objects/s recovering 2023-06-10T21:31:45.729 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:31:45.756 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:45.801 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:45.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:45.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:45.939 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:45.981 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:45.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:46.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:46.117 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:46.126 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 34298 34282 93.1075 62.625 0.0171613 0.0107334 2023-06-10T21:31:46.164 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:46.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:46.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:46.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[180297]: osdmap e281: 8 total, 8 up, 6 in 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[180297]: failed to encode map e281 with expected crc 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[180297]: failed to encode map e281 with expected crc 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[186706]: osdmap e281: 8 total, 8 up, 6 in 2023-06-10T21:31:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[186706]: failed to encode map e281 with expected crc 2023-06-10T21:31:46.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:46 smithi064 ceph-mon[186706]: failed to encode map e281 with expected crc 2023-06-10T21:31:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:46 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4231020245' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:31:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:46 smithi152 ceph-mon[156819]: osdmap e281: 8 total, 8 up, 6 in 2023-06-10T21:31:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:46 smithi152 ceph-mon[156819]: failed to encode map e281 with expected crc 2023-06-10T21:31:46.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:46 smithi152 ceph-mon[156819]: failed to encode map e281 with expected crc 2023-06-10T21:31:47.134 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 35204 35188 91.5842 56.625 0.0100801 0.0109128 2023-06-10T21:31:47.252 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:47.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:47.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:47.432 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:47.457 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:47.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:47 smithi064 ceph-mon[180297]: pgmap v993: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:31:47.579 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:47 smithi064 ceph-mon[186706]: pgmap v993: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:31:47.580 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:47.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:47.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:47.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:47.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:47 smithi152 ceph-mon[156819]: pgmap v993: 89 pgs: 89 active+clean; 1.9 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:31:47.715 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:47.760 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:47.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:47.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:47.944 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:47.969 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:31:47.969 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:31:48.127 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 36318 36302 90.7059 69.625 0.0153927 0.0110183 2023-06-10T21:31:48.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:48.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:48.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:31:49.128 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 37636 37620 90.3854 82.375 0.0140119 0.0110561 2023-06-10T21:31:49.201 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:49.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:49.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:49.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:31:49.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:31:49.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:49.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:49.438 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:49.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:49.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:49.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:49 smithi064 ceph-mon[186706]: pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-06-10T21:31:49.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:49.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:49 smithi064 ceph-mon[180297]: pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-06-10T21:31:49.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:49.638 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:49 smithi152 ceph-mon[156819]: pgmap v994: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-06-10T21:31:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:31:50.130 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 38196 38180 88.3348 35 0.0104075 0.011314 2023-06-10T21:31:50.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:31:50.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:50.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:50.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:50.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:50.766 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:50.788 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:50.835 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:50.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:50.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:31:50.972 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:51.020 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:31:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:51.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:31:51.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:51.128 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 39278 39262 87.5952 67.625 0.0154565 0.0114116 2023-06-10T21:31:51.152 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:51.197 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:51.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:51.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:51.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:51.381 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:51.390 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:51.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:51.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:51 smithi064 ceph-mon[180297]: pgmap v995: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-06-10T21:31:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:51.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:51 smithi064 ceph-mon[186706]: pgmap v995: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-06-10T21:31:51.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:51.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:51 smithi152 ceph-mon[156819]: pgmap v995: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-06-10T21:31:51.737 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:51.846 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:31:51.848 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:31:51.998 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:52.128 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 40206 40190 86.5749 58 0.00265715 0.011543 2023-06-10T21:31:53.128 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 40984 40968 85.3103 48.625 0.00855055 0.0117169 2023-06-10T21:31:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:31:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:31:52] "GET /metrics HTTP/1.1" 200 224002 "" "Prometheus/2.33.4" 2023-06-10T21:31:53.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:53.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:53.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:53.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:53.336 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:53.380 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:53.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:53.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:53.515 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:53.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:53.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:53.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:53 smithi064 ceph-mon[180297]: pgmap v996: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:31:53.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:53 smithi064 ceph-mon[180297]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:53.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:53 smithi064 ceph-mon[186706]: pgmap v996: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:31:53.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:53 smithi064 ceph-mon[186706]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:53.670 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:53 smithi152 ceph-mon[156819]: pgmap v996: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:31:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:53 smithi152 ceph-mon[156819]: from='client.56077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:31:53.764 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:54.131 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 41900 41884 84.3958 57.25 0.0160541 0.0118427 2023-06-10T21:31:54.176 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:54.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:54.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:54.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:54.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:54.959 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":996,"stamp":"2023-06-10T21:31:51.752022+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2513639310,"num_objects":38370,"num_object_clones":0,"num_object_copies":115110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38370,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":38607,"num_write_kb":2456590,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":1186972,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38541,"ondisk_log_size":38541,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9897164,"kb_used_data":7515548,"kb_used_omap":30,"kb_used_meta":2381537,"kb_avail":552573748,"statfs":{"total":575970213888,"available":565835517952,"internally_reserved":0,"allocated":7695921152,"data_stored":7691106174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2438693926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":689897472,"num_objects":10527,"num_object_clones":0,"num_object_copies":31581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10527,"num_write_kb":673728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002749"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":273,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.439956+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:31:37.439956+0000","last_peered":"2023-06-10T21:31:37.439956+0000","last_clean":"2023-06-10T21:31:37.439956+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:31:37.439956+0000","last_undegraded":"2023-06-10T21:31:37.439956+0000","last_fullsized":"2023-06-10T21:31:37.439956+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":214,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442401+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:31:37.442401+0000","last_peered":"2023-06-10T21:31:37.442401+0000","last_clean":"2023-06-10T21:31:37.442401+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:31:37.442401+0000","last_undegraded":"2023-06-10T21:31:37.442401+0000","last_fullsized":"2023-06-10T21:31:37.442401+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":189,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442105+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:31:37.442105+0000","last_peered":"2023-06-10T21:31:37.442105+0000","last_clean":"2023-06-10T21:31:37.442105+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:31:37.442105+0000","last_undegraded":"2023-06-10T21:31:37.442105+0000","last_fullsized":"2023-06-10T21:31:37.442105+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":127,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407680+0000","last_change":"2023-06-10T21:31:18.288163+0000","last_active":"2023-06-10T21:31:37.407680+0000","last_peered":"2023-06-10T21:31:37.407680+0000","last_clean":"2023-06-10T21:31:37.407680+0000","last_became_active":"2023-06-10T21:31:18.287670+0000","last_became_peered":"2023-06-10T21:31:18.287670+0000","last_unstale":"2023-06-10T21:31:37.407680+0000","last_undegraded":"2023-06-10T21:31:37.407680+0000","last_fullsized":"2023-06-10T21:31:37.407680+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:39:53.547174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":254,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441970+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:31:37.441970+0000","last_peered":"2023-06-10T21:31:37.441970+0000","last_clean":"2023-06-10T21:31:37.441970+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:31:37.441970+0000","last_undegraded":"2023-06-10T21:31:37.441970+0000","last_fullsized":"2023-06-10T21:31:37.441970+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":246,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406992+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:31:37.406992+0000","last_peered":"2023-06-10T21:31:37.406992+0000","last_clean":"2023-06-10T21:31:37.406992+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:31:37.406992+0000","last_undegraded":"2023-06-10T21:31:37.406992+0000","last_fullsized":"2023-06-10T21:31:37.406992+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407084+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:31:37.407084+0000","last_peered":"2023-06-10T21:31:37.407084+0000","last_clean":"2023-06-10T21:31:37.407084+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:31:37.407084+0000","last_undegraded":"2023-06-10T21:31:37.407084+0000","last_fullsized":"2023-06-10T21:31:37.407084+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441623+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:31:37.441623+0000","last_peered":"2023-06-10T21:31:37.441623+0000","last_clean":"2023-06-10T21:31:37.441623+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:31:37.441623+0000","last_undegraded":"2023-06-10T21:31:37.441623+0000","last_fullsized":"2023-06-10T21:31:37.441623+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441874+0000","last_change":"2023-06-10T21:31:18.128658+0000","last_active":"2023-06-10T21:31:37.441874+0000","last_peered":"2023-06-10T21:31:37.441874+0000","last_clean":"2023-06-10T21:31:37.441874+0000","last_became_active":"2023-06-10T21:31:18.126694+0000","last_became_peered":"2023-06-10T21:31:18.126694+0000","last_unstale":"2023-06-10T21:31:37.441874+0000","last_undegraded":"2023-06-10T21:31:37.441874+0000","last_fullsized":"2023-06-10T21:31:37.441874+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:28.729299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406619+0000","last_change":"2023-06-10T21:31:34.294486+0000","last_active":"2023-06-10T21:31:37.406619+0000","last_peered":"2023-06-10T21:31:37.406619+0000","last_clean":"2023-06-10T21:31:37.406619+0000","last_became_active":"2023-06-10T21:31:34.294130+0000","last_became_peered":"2023-06-10T21:31:34.294130+0000","last_unstale":"2023-06-10T21:31:37.406619+0000","last_undegraded":"2023-06-10T21:31:37.406619+0000","last_fullsized":"2023-06-10T21:31:37.406619+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":37,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442266+0000","last_change":"2023-06-10T21:31:18.367898+0000","last_active":"2023-06-10T21:31:37.442266+0000","last_peered":"2023-06-10T21:31:37.442266+0000","last_clean":"2023-06-10T21:31:37.442266+0000","last_became_active":"2023-06-10T21:31:18.133546+0000","last_became_peered":"2023-06-10T21:31:18.133546+0000","last_unstale":"2023-06-10T21:31:37.442266+0000","last_undegraded":"2023-06-10T21:31:37.442266+0000","last_fullsized":"2023-06-10T21:31:37.442266+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:35:55.635994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.440137+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:31:37.440137+0000","last_peered":"2023-06-10T21:31:37.440137+0000","last_clean":"2023-06-10T21:31:37.440137+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:31:37.440137+0000","last_undegraded":"2023-06-10T21:31:37.440137+0000","last_fullsized":"2023-06-10T21:31:37.440137+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":151,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406824+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:31:37.406824+0000","last_peered":"2023-06-10T21:31:37.406824+0000","last_clean":"2023-06-10T21:31:37.406824+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:31:37.406824+0000","last_undegraded":"2023-06-10T21:31:37.406824+0000","last_fullsized":"2023-06-10T21:31:37.406824+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406737+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:31:37.406737+0000","last_peered":"2023-06-10T21:31:37.406737+0000","last_clean":"2023-06-10T21:31:37.406737+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:31:37.406737+0000","last_undegraded":"2023-06-10T21:31:37.406737+0000","last_fullsized":"2023-06-10T21:31:37.406737+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":280,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406785+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:31:37.406785+0000","last_peered":"2023-06-10T21:31:37.406785+0000","last_clean":"2023-06-10T21:31:37.406785+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:31:37.406785+0000","last_undegraded":"2023-06-10T21:31:37.406785+0000","last_fullsized":"2023-06-10T21:31:37.406785+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":214,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441272+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:31:37.441272+0000","last_peered":"2023-06-10T21:31:37.441272+0000","last_clean":"2023-06-10T21:31:37.441272+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:31:37.441272+0000","last_undegraded":"2023-06-10T21:31:37.441272+0000","last_fullsized":"2023-06-10T21:31:37.441272+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406406+0000","last_change":"2023-06-10T21:31:18.132332+0000","last_active":"2023-06-10T21:31:37.406406+0000","last_peered":"2023-06-10T21:31:37.406406+0000","last_clean":"2023-06-10T21:31:37.406406+0000","last_became_active":"2023-06-10T21:31:18.132038+0000","last_became_peered":"2023-06-10T21:31:18.132038+0000","last_unstale":"2023-06-10T21:31:37.406406+0000","last_undegraded":"2023-06-10T21:31:37.406406+0000","last_fullsized":"2023-06-10T21:31:37.406406+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:55:26.439553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":209,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406892+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:31:37.406892+0000","last_peered":"2023-06-10T21:31:37.406892+0000","last_clean":"2023-06-10T21:31:37.406892+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:31:37.406892+0000","last_undegraded":"2023-06-10T21:31:37.406892+0000","last_fullsized":"2023-06-10T21:31:37.406892+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":257,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406684+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:31:37.406684+0000","last_peered":"2023-06-10T21:31:37.406684+0000","last_clean":"2023-06-10T21:31:37.406684+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:31:37.406684+0000","last_undegraded":"2023-06-10T21:31:37.406684+0000","last_fullsized":"2023-06-10T21:31:37.406684+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":127,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407599+0000","last_change":"2023-06-10T21:31:18.134134+0000","last_active":"2023-06-10T21:31:37.407599+0000","last_peered":"2023-06-10T21:31:37.407599+0000","last_clean":"2023-06-10T21:31:37.407599+0000","last_became_active":"2023-06-10T21:31:18.133769+0000","last_became_peered":"2023-06-10T21:31:18.133769+0000","last_unstale":"2023-06-10T21:31:37.407599+0000","last_undegraded":"2023-06-10T21:31:37.407599+0000","last_fullsized":"2023-06-10T21:31:37.407599+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:52:22.223142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":255,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406958+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:31:37.406958+0000","last_peered":"2023-06-10T21:31:37.406958+0000","last_clean":"2023-06-10T21:31:37.406958+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:31:37.406958+0000","last_undegraded":"2023-06-10T21:31:37.406958+0000","last_fullsized":"2023-06-10T21:31:37.406958+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":223,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442932+0000","last_change":"2023-06-10T21:31:18.126468+0000","last_active":"2023-06-10T21:31:37.442932+0000","last_peered":"2023-06-10T21:31:37.442932+0000","last_clean":"2023-06-10T21:31:37.442932+0000","last_became_active":"2023-06-10T21:31:18.125912+0000","last_became_peered":"2023-06-10T21:31:18.125912+0000","last_unstale":"2023-06-10T21:31:37.442932+0000","last_undegraded":"2023-06-10T21:31:37.442932+0000","last_fullsized":"2023-06-10T21:31:37.442932+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:45.523966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"281'2271","reported_seq":2292,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:47.820611+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:31:47.820611+0000","last_peered":"2023-06-10T21:31:47.820611+0000","last_clean":"2023-06-10T21:31:47.820611+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:31:47.820611+0000","last_undegraded":"2023-06-10T21:31:47.820611+0000","last_fullsized":"2023-06-10T21:31:47.820611+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2271,"log_dups_size":0,"ondisk_log_size":2271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148832256,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2271,"num_write_kb":145344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":178,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407507+0000","last_change":"2023-06-10T21:31:18.289130+0000","last_active":"2023-06-10T21:31:37.407507+0000","last_peered":"2023-06-10T21:31:37.407507+0000","last_clean":"2023-06-10T21:31:37.407507+0000","last_became_active":"2023-06-10T21:31:18.130901+0000","last_became_peered":"2023-06-10T21:31:18.130901+0000","last_unstale":"2023-06-10T21:31:37.407507+0000","last_undegraded":"2023-06-10T21:31:37.407507+0000","last_fullsized":"2023-06-10T21:31:37.407507+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:59:31.143101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441762+0000","last_change":"2023-06-10T21:31:18.128590+0000","last_active":"2023-06-10T21:31:37.441762+0000","last_peered":"2023-06-10T21:31:37.441762+0000","last_clean":"2023-06-10T21:31:37.441762+0000","last_became_active":"2023-06-10T21:31:18.126288+0000","last_became_peered":"2023-06-10T21:31:18.126288+0000","last_unstale":"2023-06-10T21:31:37.441762+0000","last_undegraded":"2023-06-10T21:31:37.441762+0000","last_fullsized":"2023-06-10T21:31:37.441762+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:58:32.179949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.6","version":"281'2285","reported_seq":2306,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:47.796294+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:31:47.796294+0000","last_peered":"2023-06-10T21:31:47.796294+0000","last_clean":"2023-06-10T21:31:47.796294+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:31:47.796294+0000","last_undegraded":"2023-06-10T21:31:47.796294+0000","last_fullsized":"2023-06-10T21:31:47.796294+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2285,"log_dups_size":0,"ondisk_log_size":2285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149749760,"num_objects":2285,"num_object_clones":0,"num_object_copies":6855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2285,"num_write_kb":146240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":23,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407465+0000","last_change":"2023-06-10T21:31:18.369455+0000","last_active":"2023-06-10T21:31:37.407465+0000","last_peered":"2023-06-10T21:31:37.407465+0000","last_clean":"2023-06-10T21:31:37.407465+0000","last_became_active":"2023-06-10T21:31:18.287763+0000","last_became_peered":"2023-06-10T21:31:18.287763+0000","last_unstale":"2023-06-10T21:31:37.407465+0000","last_undegraded":"2023-06-10T21:31:37.407465+0000","last_fullsized":"2023-06-10T21:31:37.407465+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:05.073879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442172+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:31:37.442172+0000","last_peered":"2023-06-10T21:31:37.442172+0000","last_clean":"2023-06-10T21:31:37.442172+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:31:37.442172+0000","last_undegraded":"2023-06-10T21:31:37.442172+0000","last_fullsized":"2023-06-10T21:31:37.442172+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"281'2512","reported_seq":2532,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.270392+0000","last_change":"2023-06-10T21:31:18.288013+0000","last_active":"2023-06-10T21:31:51.270392+0000","last_peered":"2023-06-10T21:31:51.270392+0000","last_clean":"2023-06-10T21:31:51.270392+0000","last_became_active":"2023-06-10T21:31:18.287612+0000","last_became_peered":"2023-06-10T21:31:18.287612+0000","last_unstale":"2023-06-10T21:31:51.270392+0000","last_undegraded":"2023-06-10T21:31:51.270392+0000","last_fullsized":"2023-06-10T21:31:51.270392+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2512,"log_dups_size":0,"ondisk_log_size":2512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:35:59.943523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164626432,"num_objects":2512,"num_object_clones":0,"num_object_copies":7536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2512,"num_write_kb":160768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":288,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406812+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:31:37.406812+0000","last_peered":"2023-06-10T21:31:37.406812+0000","last_clean":"2023-06-10T21:31:37.406812+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:31:37.406812+0000","last_undegraded":"2023-06-10T21:31:37.406812+0000","last_fullsized":"2023-06-10T21:31:37.406812+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":132,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406153+0000","last_change":"2023-06-10T21:31:18.287706+0000","last_active":"2023-06-10T21:31:37.406153+0000","last_peered":"2023-06-10T21:31:37.406153+0000","last_clean":"2023-06-10T21:31:37.406153+0000","last_became_active":"2023-06-10T21:31:18.287340+0000","last_became_peered":"2023-06-10T21:31:18.287340+0000","last_unstale":"2023-06-10T21:31:37.406153+0000","last_undegraded":"2023-06-10T21:31:37.406153+0000","last_fullsized":"2023-06-10T21:31:37.406153+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:07.486334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.4","version":"281'2317","reported_seq":2337,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.565204+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:31:49.565204+0000","last_peered":"2023-06-10T21:31:49.565204+0000","last_clean":"2023-06-10T21:31:49.565204+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:31:49.565204+0000","last_undegraded":"2023-06-10T21:31:49.565204+0000","last_fullsized":"2023-06-10T21:31:49.565204+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2317,"log_dups_size":0,"ondisk_log_size":2317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2317,"num_write_kb":148288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441434+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:31:37.441434+0000","last_peered":"2023-06-10T21:31:37.441434+0000","last_clean":"2023-06-10T21:31:37.441434+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:31:37.441434+0000","last_undegraded":"2023-06-10T21:31:37.441434+0000","last_fullsized":"2023-06-10T21:31:37.441434+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406331+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:31:37.406331+0000","last_peered":"2023-06-10T21:31:37.406331+0000","last_clean":"2023-06-10T21:31:37.406331+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:31:37.406331+0000","last_undegraded":"2023-06-10T21:31:37.406331+0000","last_fullsized":"2023-06-10T21:31:37.406331+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"281'2379","reported_seq":2399,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.280068+0000","last_change":"2023-06-10T21:31:18.120668+0000","last_active":"2023-06-10T21:31:49.280068+0000","last_peered":"2023-06-10T21:31:49.280068+0000","last_clean":"2023-06-10T21:31:49.280068+0000","last_became_active":"2023-06-10T21:31:18.120327+0000","last_became_peered":"2023-06-10T21:31:18.120327+0000","last_unstale":"2023-06-10T21:31:49.280068+0000","last_undegraded":"2023-06-10T21:31:49.280068+0000","last_fullsized":"2023-06-10T21:31:49.280068+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2379,"log_dups_size":0,"ondisk_log_size":2379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:51:56.937156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2379,"num_write_kb":152256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":231,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441424+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:31:37.441424+0000","last_peered":"2023-06-10T21:31:37.441424+0000","last_clean":"2023-06-10T21:31:37.441424+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:31:37.441424+0000","last_undegraded":"2023-06-10T21:31:37.441424+0000","last_fullsized":"2023-06-10T21:31:37.441424+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441509+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:31:37.441509+0000","last_peered":"2023-06-10T21:31:37.441509+0000","last_clean":"2023-06-10T21:31:37.441509+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:31:37.441509+0000","last_undegraded":"2023-06-10T21:31:37.441509+0000","last_fullsized":"2023-06-10T21:31:37.441509+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"281'2409","reported_seq":2430,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.274308+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:31:49.274308+0000","last_peered":"2023-06-10T21:31:49.274308+0000","last_clean":"2023-06-10T21:31:49.274308+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:31:49.274308+0000","last_undegraded":"2023-06-10T21:31:49.274308+0000","last_fullsized":"2023-06-10T21:31:49.274308+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2409,"log_dups_size":0,"ondisk_log_size":2409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157876224,"num_objects":2409,"num_object_clones":0,"num_object_copies":7227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2409,"num_write_kb":154176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":54,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.443685+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:31:37.443685+0000","last_peered":"2023-06-10T21:31:37.443685+0000","last_clean":"2023-06-10T21:31:37.443685+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:31:37.443685+0000","last_undegraded":"2023-06-10T21:31:37.443685+0000","last_fullsized":"2023-06-10T21:31:37.443685+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":271,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406643+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:31:37.406643+0000","last_peered":"2023-06-10T21:31:37.406643+0000","last_clean":"2023-06-10T21:31:37.406643+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:31:37.406643+0000","last_undegraded":"2023-06-10T21:31:37.406643+0000","last_fullsized":"2023-06-10T21:31:37.406643+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"281'2432","reported_seq":2453,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.309385+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:31:51.309385+0000","last_peered":"2023-06-10T21:31:51.309385+0000","last_clean":"2023-06-10T21:31:51.309385+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:31:51.309385+0000","last_undegraded":"2023-06-10T21:31:51.309385+0000","last_fullsized":"2023-06-10T21:31:51.309385+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2432,"log_dups_size":0,"ondisk_log_size":2432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383552,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2432,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":257,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.439124+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:31:37.439124+0000","last_peered":"2023-06-10T21:31:37.439124+0000","last_clean":"2023-06-10T21:31:37.439124+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:31:37.439124+0000","last_undegraded":"2023-06-10T21:31:37.439124+0000","last_fullsized":"2023-06-10T21:31:37.439124+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441960+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:31:37.441960+0000","last_peered":"2023-06-10T21:31:37.441960+0000","last_clean":"2023-06-10T21:31:37.441960+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:31:37.441960+0000","last_undegraded":"2023-06-10T21:31:37.441960+0000","last_fullsized":"2023-06-10T21:31:37.441960+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"281'2484","reported_seq":2505,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.294395+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:31:51.294395+0000","last_peered":"2023-06-10T21:31:51.294395+0000","last_clean":"2023-06-10T21:31:51.294395+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:31:51.294395+0000","last_undegraded":"2023-06-10T21:31:51.294395+0000","last_fullsized":"2023-06-10T21:31:51.294395+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2484,"log_dups_size":0,"ondisk_log_size":2484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162791424,"num_objects":2484,"num_object_clones":0,"num_object_copies":7452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2484,"num_write_kb":158976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":225,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441884+0000","last_change":"2023-06-10T21:31:18.119779+0000","last_active":"2023-06-10T21:31:37.441884+0000","last_peered":"2023-06-10T21:31:37.441884+0000","last_clean":"2023-06-10T21:31:37.441884+0000","last_became_active":"2023-06-10T21:31:18.119393+0000","last_became_peered":"2023-06-10T21:31:18.119393+0000","last_unstale":"2023-06-10T21:31:37.441884+0000","last_undegraded":"2023-06-10T21:31:37.441884+0000","last_fullsized":"2023-06-10T21:31:37.441884+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:04:05.878463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.443348+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:31:37.443348+0000","last_peered":"2023-06-10T21:31:37.443348+0000","last_clean":"2023-06-10T21:31:37.443348+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:31:37.443348+0000","last_undegraded":"2023-06-10T21:31:37.443348+0000","last_fullsized":"2023-06-10T21:31:37.443348+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"281'2375","reported_seq":2396,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.281666+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:31:49.281666+0000","last_peered":"2023-06-10T21:31:49.281666+0000","last_clean":"2023-06-10T21:31:49.281666+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:31:49.281666+0000","last_undegraded":"2023-06-10T21:31:49.281666+0000","last_fullsized":"2023-06-10T21:31:49.281666+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2375,"log_dups_size":0,"ondisk_log_size":2375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155648000,"num_objects":2375,"num_object_clones":0,"num_object_copies":7125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2375,"num_write_kb":152000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406096+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:31:37.406096+0000","last_peered":"2023-06-10T21:31:37.406096+0000","last_clean":"2023-06-10T21:31:37.406096+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:31:37.406096+0000","last_undegraded":"2023-06-10T21:31:37.406096+0000","last_fullsized":"2023-06-10T21:31:37.406096+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":274,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442481+0000","last_change":"2023-06-10T21:31:18.126672+0000","last_active":"2023-06-10T21:31:37.442481+0000","last_peered":"2023-06-10T21:31:37.442481+0000","last_clean":"2023-06-10T21:31:37.442481+0000","last_became_active":"2023-06-10T21:31:18.126223+0000","last_became_peered":"2023-06-10T21:31:18.126223+0000","last_unstale":"2023-06-10T21:31:37.442481+0000","last_undegraded":"2023-06-10T21:31:37.442481+0000","last_fullsized":"2023-06-10T21:31:37.442481+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:56:43.102343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406228+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:31:37.406228+0000","last_peered":"2023-06-10T21:31:37.406228+0000","last_clean":"2023-06-10T21:31:37.406228+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:31:37.406228+0000","last_undegraded":"2023-06-10T21:31:37.406228+0000","last_fullsized":"2023-06-10T21:31:37.406228+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"281'2452","reported_seq":2473,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.280233+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:31:51.280233+0000","last_peered":"2023-06-10T21:31:51.280233+0000","last_clean":"2023-06-10T21:31:51.280233+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:31:51.280233+0000","last_undegraded":"2023-06-10T21:31:51.280233+0000","last_fullsized":"2023-06-10T21:31:51.280233+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2452,"log_dups_size":0,"ondisk_log_size":2452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160694272,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2452,"num_write_kb":156928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":149,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406308+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:31:37.406308+0000","last_peered":"2023-06-10T21:31:37.406308+0000","last_clean":"2023-06-10T21:31:37.406308+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:31:37.406308+0000","last_undegraded":"2023-06-10T21:31:37.406308+0000","last_fullsized":"2023-06-10T21:31:37.406308+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":223,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406494+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:31:37.406494+0000","last_peered":"2023-06-10T21:31:37.406494+0000","last_clean":"2023-06-10T21:31:37.406494+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:31:37.406494+0000","last_undegraded":"2023-06-10T21:31:37.406494+0000","last_fullsized":"2023-06-10T21:31:37.406494+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":113,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:45.427513+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:31:45.427513+0000","last_peered":"2023-06-10T21:31:45.427513+0000","last_clean":"2023-06-10T21:31:45.427513+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:31:45.427513+0000","last_undegraded":"2023-06-10T21:31:45.427513+0000","last_fullsized":"2023-06-10T21:31:45.427513+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"281'2494","reported_seq":2515,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.309421+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:31:51.309421+0000","last_peered":"2023-06-10T21:31:51.309421+0000","last_clean":"2023-06-10T21:31:51.309421+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:31:51.309421+0000","last_undegraded":"2023-06-10T21:31:51.309421+0000","last_fullsized":"2023-06-10T21:31:51.309421+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2494,"log_dups_size":0,"ondisk_log_size":2494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163446784,"num_objects":2494,"num_object_clones":0,"num_object_copies":7482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2494,"num_write_kb":159616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":124,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:45.429006+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:31:45.429006+0000","last_peered":"2023-06-10T21:31:45.429006+0000","last_clean":"2023-06-10T21:31:45.429006+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:31:45.429006+0000","last_undegraded":"2023-06-10T21:31:45.429006+0000","last_fullsized":"2023-06-10T21:31:45.429006+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406016+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:31:37.406016+0000","last_peered":"2023-06-10T21:31:37.406016+0000","last_clean":"2023-06-10T21:31:37.406016+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:31:37.406016+0000","last_undegraded":"2023-06-10T21:31:37.406016+0000","last_fullsized":"2023-06-10T21:31:37.406016+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":163,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407162+0000","last_change":"2023-06-10T21:31:18.134234+0000","last_active":"2023-06-10T21:31:37.407162+0000","last_peered":"2023-06-10T21:31:37.407162+0000","last_clean":"2023-06-10T21:31:37.407162+0000","last_became_active":"2023-06-10T21:31:18.133805+0000","last_became_peered":"2023-06-10T21:31:18.133805+0000","last_unstale":"2023-06-10T21:31:37.407162+0000","last_undegraded":"2023-06-10T21:31:37.407162+0000","last_fullsized":"2023-06-10T21:31:37.407162+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:32:40.924800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"281'2442","reported_seq":2461,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.280011+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:31:51.280011+0000","last_peered":"2023-06-10T21:31:51.280011+0000","last_clean":"2023-06-10T21:31:51.280011+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:31:51.280011+0000","last_undegraded":"2023-06-10T21:31:51.280011+0000","last_fullsized":"2023-06-10T21:31:51.280011+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2442,"log_dups_size":0,"ondisk_log_size":2442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160038912,"num_objects":2442,"num_object_clones":0,"num_object_copies":7326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2442,"num_write_kb":156288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":294,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406342+0000","last_change":"2023-06-10T21:31:18.130817+0000","last_active":"2023-06-10T21:31:37.406342+0000","last_peered":"2023-06-10T21:31:37.406342+0000","last_clean":"2023-06-10T21:31:37.406342+0000","last_became_active":"2023-06-10T21:31:18.130515+0000","last_became_peered":"2023-06-10T21:31:18.130515+0000","last_unstale":"2023-06-10T21:31:37.406342+0000","last_undegraded":"2023-06-10T21:31:37.406342+0000","last_fullsized":"2023-06-10T21:31:37.406342+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:17:11.299968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":435,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406625+0000","last_change":"2023-06-10T21:31:21.158169+0000","last_active":"2023-06-10T21:31:37.406625+0000","last_peered":"2023-06-10T21:31:37.406625+0000","last_clean":"2023-06-10T21:31:37.406625+0000","last_became_active":"2023-06-10T21:31:21.157730+0000","last_became_peered":"2023-06-10T21:31:21.157730+0000","last_unstale":"2023-06-10T21:31:37.406625+0000","last_undegraded":"2023-06-10T21:31:37.406625+0000","last_fullsized":"2023-06-10T21:31:37.406625+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:50:55.992599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":217,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.443842+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:31:37.443842+0000","last_peered":"2023-06-10T21:31:37.443842+0000","last_clean":"2023-06-10T21:31:37.443842+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:31:37.443842+0000","last_undegraded":"2023-06-10T21:31:37.443842+0000","last_fullsized":"2023-06-10T21:31:37.443842+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":95,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406859+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:31:37.406859+0000","last_peered":"2023-06-10T21:31:37.406859+0000","last_clean":"2023-06-10T21:31:37.406859+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:31:37.406859+0000","last_undegraded":"2023-06-10T21:31:37.406859+0000","last_fullsized":"2023-06-10T21:31:37.406859+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"281'2435","reported_seq":2456,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:51.255630+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:31:51.255630+0000","last_peered":"2023-06-10T21:31:51.255630+0000","last_clean":"2023-06-10T21:31:51.255630+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:31:51.255630+0000","last_undegraded":"2023-06-10T21:31:51.255630+0000","last_fullsized":"2023-06-10T21:31:51.255630+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2435,"log_dups_size":0,"ondisk_log_size":2435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159580160,"num_objects":2435,"num_object_clones":0,"num_object_copies":7305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2435,"num_write_kb":155840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":300,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406565+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:31:37.406565+0000","last_peered":"2023-06-10T21:31:37.406565+0000","last_clean":"2023-06-10T21:31:37.406565+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:31:37.406565+0000","last_undegraded":"2023-06-10T21:31:37.406565+0000","last_fullsized":"2023-06-10T21:31:37.406565+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":118,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407663+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:31:37.407663+0000","last_peered":"2023-06-10T21:31:37.407663+0000","last_clean":"2023-06-10T21:31:37.407663+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:31:37.407663+0000","last_undegraded":"2023-06-10T21:31:37.407663+0000","last_fullsized":"2023-06-10T21:31:37.407663+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407552+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:31:37.407552+0000","last_peered":"2023-06-10T21:31:37.407552+0000","last_clean":"2023-06-10T21:31:37.407552+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:31:37.407552+0000","last_undegraded":"2023-06-10T21:31:37.407552+0000","last_fullsized":"2023-06-10T21:31:37.407552+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"281'2338","reported_seq":2359,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.269014+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:31:49.269014+0000","last_peered":"2023-06-10T21:31:49.269014+0000","last_clean":"2023-06-10T21:31:49.269014+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:31:49.269014+0000","last_undegraded":"2023-06-10T21:31:49.269014+0000","last_fullsized":"2023-06-10T21:31:49.269014+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2338,"log_dups_size":0,"ondisk_log_size":2338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153223168,"num_objects":2338,"num_object_clones":0,"num_object_copies":7014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2338,"num_write_kb":149632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":285,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406801+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:31:37.406801+0000","last_peered":"2023-06-10T21:31:37.406801+0000","last_clean":"2023-06-10T21:31:37.406801+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:31:37.406801+0000","last_undegraded":"2023-06-10T21:31:37.406801+0000","last_fullsized":"2023-06-10T21:31:37.406801+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:45.427595+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:31:45.427595+0000","last_peered":"2023-06-10T21:31:45.427595+0000","last_clean":"2023-06-10T21:31:45.427595+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:31:45.427595+0000","last_undegraded":"2023-06-10T21:31:45.427595+0000","last_fullsized":"2023-06-10T21:31:45.427595+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":99,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406640+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:31:37.406640+0000","last_peered":"2023-06-10T21:31:37.406640+0000","last_clean":"2023-06-10T21:31:37.406640+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:31:37.406640+0000","last_undegraded":"2023-06-10T21:31:37.406640+0000","last_fullsized":"2023-06-10T21:31:37.406640+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"281'2375","reported_seq":2394,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.643508+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:31:49.643508+0000","last_peered":"2023-06-10T21:31:49.643508+0000","last_clean":"2023-06-10T21:31:49.643508+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:31:49.643508+0000","last_undegraded":"2023-06-10T21:31:49.643508+0000","last_fullsized":"2023-06-10T21:31:49.643508+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2375,"log_dups_size":0,"ondisk_log_size":2375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155648000,"num_objects":2375,"num_object_clones":0,"num_object_copies":7125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2375,"num_write_kb":152000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":244,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.443032+0000","last_change":"2023-06-10T21:31:18.371780+0000","last_active":"2023-06-10T21:31:37.443032+0000","last_peered":"2023-06-10T21:31:37.443032+0000","last_clean":"2023-06-10T21:31:37.443032+0000","last_became_active":"2023-06-10T21:31:18.288097+0000","last_became_peered":"2023-06-10T21:31:18.288097+0000","last_unstale":"2023-06-10T21:31:37.443032+0000","last_undegraded":"2023-06-10T21:31:37.443032+0000","last_fullsized":"2023-06-10T21:31:37.443032+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:26:44.394037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":54,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441789+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:31:37.441789+0000","last_peered":"2023-06-10T21:31:37.441789+0000","last_clean":"2023-06-10T21:31:37.441789+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:31:37.441789+0000","last_undegraded":"2023-06-10T21:31:37.441789+0000","last_fullsized":"2023-06-10T21:31:37.441789+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441795+0000","last_change":"2023-06-10T21:31:18.128783+0000","last_active":"2023-06-10T21:31:37.441795+0000","last_peered":"2023-06-10T21:31:37.441795+0000","last_clean":"2023-06-10T21:31:37.441795+0000","last_became_active":"2023-06-10T21:31:18.126664+0000","last_became_peered":"2023-06-10T21:31:18.126664+0000","last_unstale":"2023-06-10T21:31:37.441795+0000","last_undegraded":"2023-06-10T21:31:37.441795+0000","last_fullsized":"2023-06-10T21:31:37.441795+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:05.679250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"281'2346","reported_seq":2366,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-06-10T21:31:49.624364+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:31:49.624364+0000","last_peered":"2023-06-10T21:31:49.624364+0000","last_clean":"2023-06-10T21:31:49.624364+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:31:49.624364+0000","last_undegraded":"2023-06-10T21:31:49.624364+0000","last_fullsized":"2023-06-10T21:31:49.624364+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":2346,"log_dups_size":0,"ondisk_log_size":2346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153747456,"num_objects":2346,"num_object_clones":0,"num_object_copies":7038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2346,"num_write_kb":150144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":252,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406701+0000","last_change":"2023-06-10T21:31:18.120470+0000","last_active":"2023-06-10T21:31:37.406701+0000","last_peered":"2023-06-10T21:31:37.406701+0000","last_clean":"2023-06-10T21:31:37.406701+0000","last_became_active":"2023-06-10T21:31:18.120135+0000","last_became_peered":"2023-06-10T21:31:18.120135+0000","last_unstale":"2023-06-10T21:31:37.406701+0000","last_undegraded":"2023-06-10T21:31:37.406701+0000","last_fullsized":"2023-06-10T21:31:37.406701+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:38:33.846268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":107,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407641+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:31:37.407641+0000","last_peered":"2023-06-10T21:31:37.407641+0000","last_clean":"2023-06-10T21:31:37.407641+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:31:37.407641+0000","last_undegraded":"2023-06-10T21:31:37.407641+0000","last_fullsized":"2023-06-10T21:31:37.407641+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":226,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407510+0000","last_change":"2023-06-10T21:31:18.130117+0000","last_active":"2023-06-10T21:31:37.407510+0000","last_peered":"2023-06-10T21:31:37.407510+0000","last_clean":"2023-06-10T21:31:37.407510+0000","last_became_active":"2023-06-10T21:31:18.129804+0000","last_became_peered":"2023-06-10T21:31:18.129804+0000","last_unstale":"2023-06-10T21:31:37.407510+0000","last_undegraded":"2023-06-10T21:31:37.407510+0000","last_fullsized":"2023-06-10T21:31:37.407510+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:16:11.560639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406729+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:31:37.406729+0000","last_peered":"2023-06-10T21:31:37.406729+0000","last_clean":"2023-06-10T21:31:37.406729+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:31:37.406729+0000","last_undegraded":"2023-06-10T21:31:37.406729+0000","last_fullsized":"2023-06-10T21:31:37.406729+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":121,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.439270+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:31:37.439270+0000","last_peered":"2023-06-10T21:31:37.439270+0000","last_clean":"2023-06-10T21:31:37.439270+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:31:37.439270+0000","last_undegraded":"2023-06-10T21:31:37.439270+0000","last_fullsized":"2023-06-10T21:31:37.439270+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":23,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.406977+0000","last_change":"2023-06-10T21:31:18.131286+0000","last_active":"2023-06-10T21:31:37.406977+0000","last_peered":"2023-06-10T21:31:37.406977+0000","last_clean":"2023-06-10T21:31:37.406977+0000","last_became_active":"2023-06-10T21:31:18.123071+0000","last_became_peered":"2023-06-10T21:31:18.123071+0000","last_unstale":"2023-06-10T21:31:37.406977+0000","last_undegraded":"2023-06-10T21:31:37.406977+0000","last_fullsized":"2023-06-10T21:31:37.406977+0000","mapping_epoch":272,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:19:41.707381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441251+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:31:37.441251+0000","last_peered":"2023-06-10T21:31:37.441251+0000","last_clean":"2023-06-10T21:31:37.441251+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:31:37.441251+0000","last_undegraded":"2023-06-10T21:31:37.441251+0000","last_fullsized":"2023-06-10T21:31:37.441251+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":58,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.442074+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:31:37.442074+0000","last_peered":"2023-06-10T21:31:37.442074+0000","last_clean":"2023-06-10T21:31:37.442074+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:31:37.442074+0000","last_undegraded":"2023-06-10T21:31:37.442074+0000","last_fullsized":"2023-06-10T21:31:37.442074+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":218,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.443762+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:31:37.443762+0000","last_peered":"2023-06-10T21:31:37.443762+0000","last_clean":"2023-06-10T21:31:37.443762+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:31:37.443762+0000","last_undegraded":"2023-06-10T21:31:37.443762+0000","last_fullsized":"2023-06-10T21:31:37.443762+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":263,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441597+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:31:37.441597+0000","last_peered":"2023-06-10T21:31:37.441597+0000","last_clean":"2023-06-10T21:31:37.441597+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:31:37.441597+0000","last_undegraded":"2023-06-10T21:31:37.441597+0000","last_fullsized":"2023-06-10T21:31:37.441597+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":267,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.441879+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:31:37.441879+0000","last_peered":"2023-06-10T21:31:37.441879+0000","last_clean":"2023-06-10T21:31:37.441879+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:31:37.441879+0000","last_undegraded":"2023-06-10T21:31:37.441879+0000","last_fullsized":"2023-06-10T21:31:37.441879+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":168,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.407249+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:31:37.407249+0000","last_peered":"2023-06-10T21:31:37.407249+0000","last_clean":"2023-06-10T21:31:37.407249+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:31:37.407249+0000","last_undegraded":"2023-06-10T21:31:37.407249+0000","last_fullsized":"2023-06-10T21:31:37.407249+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-06-10T21:31:37.405736+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:31:37.405736+0000","last_peered":"2023-06-10T21:31:37.405736+0000","last_clean":"2023-06-10T21:31:37.405736+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:31:37.405736+0000","last_undegraded":"2023-06-10T21:31:37.405736+0000","last_fullsized":"2023-06-10T21:31:37.405736+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2513043456,"num_objects":38346,"num_object_clones":0,"num_object_copies":115038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":38346,"num_write_kb":2454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7526744064,"data_stored":7526744064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":38346,"ondisk_log_size":38346,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":6236,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430617,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138604,"kb_used_data":603492,"kb_used_omap":0,"kb_used_meta":535104,"kb_avail":92606548,"statfs":{"total":95995035648,"available":94829105152,"internally_reserved":0,"allocated":617975808,"data_stored":617429368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":547946496},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:38 2023","interfaces":[{"interface":"back","average":{"1min":12.305999999999999,"5min":8.5950000000000006,"15min":9.3369999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":184.83699999999999,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.316},{"interface":"front","average":{"1min":11.945,"5min":8.6839999999999993,"15min":9.5939999999999994},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.223},"max":{"1min":185.58000000000001,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.42599999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:31:10 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":2.2810000000000001,"15min":2.819},"min":{"1min":0.307,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.464,"5min":93.093999999999994,"15min":130.19399999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.101,"5min":1.9059999999999999,"15min":3.1320000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.456,"5min":93.119,"15min":212.20599999999999},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:31:18 2023","interfaces":[{"interface":"back","average":{"1min":8.8350000000000009,"5min":4.3170000000000002,"15min":2.948},"min":{"1min":0.39200000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":138.36099999999999,"5min":138.36099999999999,"15min":160.99299999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":9.3689999999999998,"5min":4.1980000000000004,"15min":3.5920000000000001},"min":{"1min":0.46100000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":137.708,"5min":137.708,"15min":212.13399999999999},"last":0.435}]},{"osd":3,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":2.383,"15min":2.4609999999999999},"min":{"1min":0.38500000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":10.550000000000001,"5min":10.550000000000001,"15min":10.550000000000001},"last":1.028},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.7769999999999999,"15min":1.8140000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.6480000000000001,"5min":4.242,"15min":4.242},"last":0.95099999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.65900000000000003,"15min":0.89400000000000002},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.45200000000000001,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.308},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.749,"15min":1.3129999999999999},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.48599999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.34499999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.97799999999999998,"15min":2.145},"min":{"1min":0.28100000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6280000000000001,"5min":12.048,"15min":211.99799999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.67000000000000004,"15min":1.946},"min":{"1min":0.32200000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.56599999999999995,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.97899999999999998,"15min":1.294},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.8849999999999998,"5min":13.073,"15min":13.073},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.626,"15min":1.093},"min":{"1min":0.371,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.897,"5min":3.8239999999999998,"15min":10.874000000000001},"last":0.36899999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626845,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566288,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2536232,"kb_used_data":2041696,"kb_used_omap":4,"kb_used_meta":494523,"kb_avail":91208920,"statfs":{"total":95995035648,"available":93397934080,"internally_reserved":0,"allocated":2090696704,"data_stored":2089801265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":506392462},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:39 2023","interfaces":[{"interface":"back","average":{"1min":8.8599999999999994,"5min":11.287000000000001,"15min":12.914999999999999},"min":{"1min":0.25600000000000001,"5min":0.221,"15min":0.221},"max":{"1min":154.31399999999999,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":8.5229999999999997,"5min":11.237,"15min":12.625999999999999},"min":{"1min":0.29699999999999999,"5min":0.221,"15min":0.221},"max":{"1min":154.27199999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:31:29 2023","interfaces":[{"interface":"back","average":{"1min":6.8239999999999998,"5min":6.6760000000000002,"15min":3.2599999999999998},"min":{"1min":0.27400000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":104.334,"5min":107.857,"15min":107.857},"last":54.017000000000003},{"interface":"front","average":{"1min":6.6639999999999997,"5min":7.0490000000000004,"15min":3.0499999999999998},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":104.98699999999999,"5min":107.958,"15min":107.958},"last":54.140999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":2.1059999999999999,"15min":2.0190000000000001},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.9879999999999995,"5min":11.445,"15min":11.445},"last":62.942},{"interface":"front","average":{"1min":5.1050000000000004,"5min":3.5659999999999998,"15min":3.3100000000000001},"min":{"1min":0.24299999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":23.93,"5min":23.93,"15min":23.93},"last":67.475999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":1.579,"15min":1.7},"min":{"1min":0.40000000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.036,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.343,"5min":1.175,"15min":1.478},"min":{"1min":0.35499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.8620000000000001,"5min":6.8949999999999996,"15min":14.460000000000001},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.6299999999999999,"15min":4.25},"min":{"1min":0.375,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.497},{"interface":"front","average":{"1min":1.879,"5min":1.593,"15min":4.0579999999999998},"min":{"1min":0.40200000000000002,"5min":0.33900000000000002,"15min":0.29699999999999999},"max":{"1min":10.116,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":1.8360000000000001,"15min":2.1389999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":8.8580000000000005,"5min":9.5690000000000008,"15min":14.858000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.8979999999999999,"5min":1.6950000000000001,"15min":1.9039999999999999},"min":{"1min":0.46200000000000002,"5min":0.39000000000000001,"15min":0.32900000000000001},"max":{"1min":10.066000000000001,"5min":11.476000000000001,"15min":13.48},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:31:08 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.762,"15min":2.6619999999999999},"min":{"1min":0.44800000000000001,"5min":0.34599999999999997,"15min":0.32600000000000001},"max":{"1min":10.092000000000001,"5min":16.113,"15min":162.399},"last":0.439},{"interface":"front","average":{"1min":1.552,"5min":1.5009999999999999,"15min":2.3250000000000002},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.9329999999999998,"5min":6.9459999999999997,"15min":162.27199999999999},"last":0.50900000000000001}]}]},{"osd":0,"up_from":244,"seq":1047972020264,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1346876,"kb_used_data":1139252,"kb_used_omap":10,"kb_used_meta":207605,"kb_avail":92398276,"statfs":{"total":95995035648,"available":94615834624,"internally_reserved":0,"allocated":1166594048,"data_stored":1165619140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":212587572},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":4.0839999999999996,"15min":4.3010000000000002},"min":{"1min":0.20799999999999999,"5min":0.191,"15min":0.191},"max":{"1min":17.251999999999999,"5min":19.806999999999999,"15min":19.806999999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.625,"5min":4.9240000000000004,"15min":5.6820000000000004},"min":{"1min":0.219,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":17.510000000000002,"5min":29.841000000000001,"15min":29.841000000000001},"last":1.71}]},{"osd":2,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":2.1080000000000001,"5min":6.4980000000000002,"15min":6.1340000000000003},"min":{"1min":0.23599999999999999,"5min":0.222,"15min":0.222},"max":{"1min":10.401,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.792,"5min":6.3479999999999999,"15min":5.5199999999999996},"min":{"1min":0.28999999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.361000000000001,"5min":208.34299999999999,"15min":208.34299999999999},"last":1.0680000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:31:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":5.827,"15min":6.133},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":27.952999999999999,"5min":27.952999999999999,"15min":27.952999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":4.8230000000000004,"5min":6.7910000000000004,"15min":7.1189999999999998},"min":{"1min":0.32600000000000001,"5min":0.31,"15min":0.31},"max":{"1min":17.184000000000001,"5min":32.822000000000003,"15min":32.822000000000003},"last":1.165}]},{"osd":4,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":3.9060000000000001,"15min":4.7000000000000002},"min":{"1min":0.379,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.6219999999999999,"5min":29.709,"15min":29.709},"last":1.7769999999999999},{"interface":"front","average":{"1min":1.3999999999999999,"5min":3.4409999999999998,"15min":3.9969999999999999},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.0289999999999999,"5min":13.872,"15min":13.872},"last":0.879}]},{"osd":5,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":3.5150000000000001,"15min":4.0949999999999998},"min":{"1min":0.34300000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.1189999999999998,"5min":14.105,"15min":14.105},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.79200000000000004,"5min":3.0750000000000002,"15min":3.4340000000000002},"min":{"1min":0.38900000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.2799999999999998,"5min":19.846,"15min":19.846},"last":0.92400000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":4.0579999999999998,"15min":4.8970000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.7050000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":1.8480000000000001},{"interface":"front","average":{"1min":1.4419999999999999,"5min":3.5499999999999998,"15min":4.1319999999999997},"min":{"1min":0.45500000000000002,"5min":0.434,"15min":0.434},"max":{"1min":7.0839999999999996,"5min":13.82,"15min":13.82},"last":0.93700000000000006}]},{"osd":7,"last update":"Sat Jun 10 21:31:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":3.3650000000000002,"15min":3.847},"min":{"1min":0.38100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.3100000000000001,"5min":19.866,"15min":19.866},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.83799999999999997,"5min":4.0620000000000003,"15min":4.8970000000000002},"min":{"1min":0.44,"5min":0.378,"15min":0.378},"max":{"1min":3.0609999999999999,"5min":29.77,"15min":29.77},"last":1.8149999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075676,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918496,"kb_used_data":1509080,"kb_used_omap":7,"kb_used_meta":409400,"kb_avail":91826656,"statfs":{"total":95995035648,"available":94030495744,"internally_reserved":0,"allocated":1545297920,"data_stored":1544410557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":419225681},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:38 2023","interfaces":[{"interface":"back","average":{"1min":4.3150000000000004,"5min":4.6920000000000002,"15min":4.7000000000000002},"min":{"1min":0.224,"5min":0.222,"15min":0.222},"max":{"1min":66.784000000000006,"5min":66.784000000000006,"15min":66.784000000000006},"last":0.35099999999999998},{"interface":"front","average":{"1min":4.5810000000000004,"5min":5.4569999999999999,"15min":5.4130000000000003},"min":{"1min":0.26700000000000002,"5min":0.252,"15min":0.252},"max":{"1min":67.251999999999995,"5min":67.251999999999995,"15min":67.251999999999995},"last":1.02}]},{"osd":1,"last update":"Sat Jun 10 21:31:23 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.3490000000000002,"15min":1.7490000000000001},"min":{"1min":0.27600000000000002,"5min":0.19,"15min":0.19},"max":{"1min":6.7140000000000004,"5min":11.6,"15min":29.486000000000001},"last":1.4179999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.0099999999999998,"15min":1.575},"min":{"1min":0.28000000000000003,"5min":0.214,"15min":0.214},"max":{"1min":17.812000000000001,"5min":17.812000000000001,"15min":28.013999999999999},"last":0.373}]},{"osd":3,"last update":"Sat Jun 10 21:30:57 2023","interfaces":[{"interface":"back","average":{"1min":2.9870000000000001,"5min":4.79,"15min":5.0899999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":17.417999999999999,"5min":17.417999999999999,"15min":17.417999999999999},"last":0.503},{"interface":"front","average":{"1min":2.8359999999999999,"5min":5.6020000000000003,"15min":6.0629999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.6620000000000008,"5min":16.411000000000001,"15min":16.411000000000001},"last":1.0620000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:31:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":5.4169999999999998,"15min":2.6440000000000001},"min":{"1min":0.36499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.5510000000000002,"5min":224.00999999999999,"15min":224.00999999999999},"last":1.145},{"interface":"front","average":{"1min":0.70599999999999996,"5min":5.726,"15min":2.7829999999999999},"min":{"1min":0.42099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.6970000000000001,"5min":223.28399999999999,"15min":223.28399999999999},"last":1.448}]},{"osd":5,"last update":"Sat Jun 10 21:31:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.55,"15min":1.3360000000000001},"min":{"1min":0.42199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.6719999999999999,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.7410000000000001,"15min":1.556},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.6020000000000003,"5min":16.125,"15min":16.125},"last":1.177}]},{"osd":6,"last update":"Sat Jun 10 21:31:23 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.9650000000000001,"15min":1.613},"min":{"1min":0.435,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":4.5170000000000003,"5min":16.183,"15min":16.183},"last":1.2150000000000001},{"interface":"front","average":{"1min":0.746,"5min":2.2069999999999999,"15min":1.784},"min":{"1min":0.46800000000000003,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":1.7310000000000001,"5min":11.029,"15min":17.244},"last":1.4710000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:31:23 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":2.9809999999999999,"15min":1.7310000000000001},"min":{"1min":0.498,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.6399999999999999,"5min":113.913,"15min":113.913},"last":0.443},{"interface":"front","average":{"1min":1.0269999999999999,"5min":3.2360000000000002,"15min":2.056},"min":{"1min":0.433,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":4.5780000000000003,"5min":113.937,"15min":113.937},"last":1.1100000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824036,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1771500,"kb_used_data":1617572,"kb_used_omap":7,"kb_used_meta":153912,"kb_avail":91973652,"statfs":{"total":95995035648,"available":94181019648,"internally_reserved":0,"allocated":1656393728,"data_stored":1655434332,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":157605907},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":4.9409999999999998,"15min":5.3460000000000001},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":19.916,"5min":56.828000000000003,"15min":56.828000000000003},"last":0.504},{"interface":"front","average":{"1min":1.881,"5min":4.6420000000000003,"15min":5.1020000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":9.1159999999999997,"5min":57.600000000000001,"15min":57.600000000000001},"last":0.61399999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.9910000000000001,"5min":3.7090000000000001,"15min":3.6619999999999999},"min":{"1min":0.248,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":19.859000000000002,"5min":19.859000000000002,"15min":19.859000000000002},"last":6.1799999999999997},{"interface":"front","average":{"1min":1.302,"5min":3.048,"15min":3.339},"min":{"1min":0.31900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.1219999999999999,"5min":8.2579999999999991,"15min":8.2579999999999991},"last":0.378}]},{"osd":2,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.036,"5min":10.744,"15min":12.196},"min":{"1min":0.29299999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.218999999999999,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.70699999999999996},{"interface":"front","average":{"1min":4.758,"5min":12.426,"15min":13.704000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.832000000000001,"5min":80.566999999999993,"15min":80.566999999999993},"last":6.1349999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.397,"15min":1.381},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.2330000000000005,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.484,"5min":2.8879999999999999,"15min":2.7879999999999998},"min":{"1min":0.372,"5min":0.311,"15min":0.311},"max":{"1min":19.728000000000002,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.91900000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.282,"15min":1.3560000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.2029999999999998,"5min":6.2480000000000002,"15min":6.2480000000000002},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.4079999999999999,"5min":1.425,"15min":1.4279999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":9.2599999999999998,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":0.68000000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5190000000000001,"5min":2.915,"15min":2.8140000000000001},"min":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":19.805,"5min":19.805,"15min":19.805},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.349,"15min":1.409},"min":{"1min":0.443,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.2490000000000001,"5min":6.3449999999999998,"15min":6.3449999999999998},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:31:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5129999999999999,"5min":2.9249999999999998,"15min":2.827},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":19.777000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.3260000000000001,"15min":1.3839999999999999},"min":{"1min":0.38500000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.2770000000000001,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.41399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019301,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1185456,"kb_used_data":604456,"kb_used_omap":0,"kb_used_meta":580992,"kb_avail":92559696,"statfs":{"total":95995035648,"available":94781128704,"internally_reserved":0,"allocated":618962944,"data_stored":618411512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":594935808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:31:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":6.8719999999999999,"15min":9.4090000000000007},"min":{"1min":0.25700000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.2519999999999998,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.76600000000000001,"5min":7.3179999999999996,"15min":9.0660000000000007},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.5659999999999998,"5min":144.303,"15min":144.303},"last":2.8479999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:31:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.5820000000000001,"15min":2.9460000000000002},"min":{"1min":0.32200000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":5.0609999999999999,"5min":15.063000000000001,"15min":182.035},"last":3.641},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.8839999999999999,"15min":2.8149999999999999},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5570000000000004,"5min":15.052,"15min":181.73099999999999},"last":3.2429999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":2.1429999999999998,"15min":1.452},"min":{"1min":0.29499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.7119999999999997,"5min":22.614000000000001,"15min":22.614000000000001},"last":7.101},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.8839999999999999,"15min":1.4399999999999999},"min":{"1min":0.36699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.4039999999999999,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":2.262,"15min":2.3479999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8529999999999998,"5min":9.2379999999999995,"15min":9.2379999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.2519999999999998,"5min":2.085,"15min":2.0569999999999999},"min":{"1min":0.35099999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.3710000000000004,"5min":6.3710000000000004,"15min":6.3710000000000004},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.72399999999999998,"15min":1.109},"min":{"1min":0.28399999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56599999999999995,"5min":6.742,"15min":13.941000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.85899999999999999,"15min":1.1759999999999999},"min":{"1min":0.29899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.63100000000000001,"5min":6.0819999999999999,"15min":11.904999999999999},"last":0.376}]},{"osd":6,"last update":"Sat Jun 10 21:30:49 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.82299999999999995,"15min":1.179},"min":{"1min":0.34100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.749,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":0.434},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.92300000000000004,"15min":1.2569999999999999},"min":{"1min":0.34000000000000002,"5min":0.32700000000000001,"15min":0.30399999999999999},"max":{"1min":1.1990000000000001,"5min":8.3109999999999999,"15min":13.845000000000001},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.84599999999999997,"15min":1.393},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.65400000000000003,"5min":6.798,"15min":16.265000000000001},"last":0.5},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.91600000000000004,"15min":1.254},"min":{"1min":0.36799999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.80400000000000005,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":0.36199999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823073,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1186136064,"data_stored":1186136064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2097545216,"data_stored":2097545216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1547960320,"data_stored":1547960320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1457586176,"data_stored":1457586176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":619249664,"data_stored":619249664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":618266624,"data_stored":618266624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:31:54.969 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:31:54.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set device_health_metrics pg_num 21 2023-06-10T21:31:55.132 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 42741 42725 83.4013 52.5625 0.00996711 0.0119847 2023-06-10T21:31:55.432 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:55.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:55.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:55 smithi064 ceph-mon[180297]: pgmap v997: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-10T21:31:55.535 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:55 smithi064 ceph-mon[186706]: pgmap v997: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-10T21:31:55.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:55.638 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:55.858 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:55.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:55.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:55 smithi152 ceph-mon[156819]: pgmap v997: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-10T21:31:55.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:31:56.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:56.053 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:31:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:56.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:31:56.132 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 43543 43527 82.3931 50.125 0.00464959 0.0120799 2023-06-10T21:31:56.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:56.189 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:56.258 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:56.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:56.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:56.440 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:56.663 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:56.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:56.771 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:56.797 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:56.843 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:31:56.856 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:56.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:56.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:56.990 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:57.035 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:57.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:57.106 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[180297]: pgmap v998: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2023-06-10T21:31:57.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1164683547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.107 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[186706]: pgmap v998: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2023-06-10T21:31:57.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1164683547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.133 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 44495 44479 81.7179 59.5 0.00811802 0.0122296 2023-06-10T21:31:57.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:31:57.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:56 smithi152 ceph-mon[156819]: pgmap v998: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 59 MiB/s wr, 938 op/s 2023-06-10T21:31:57.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1164683547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.171 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:31:57.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:57.217 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:31:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:57.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:31:57.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:31:57.408 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:57.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:31:57.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:31:57.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:31:57.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:31:57.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:31:57.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:31:57.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:31:57.455 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:31:57.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:31:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:57.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:31:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.533 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.534 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.534 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.534 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:31:57.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:31:57.695 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:57.733 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:31:57.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:57.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:31:57.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:31:57.918 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:57.947 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:57.948 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: failed to encode map e282 with expected crc 2023-06-10T21:31:57.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:57.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: osdmap e282: 8 total, 8 up, 6 in 2023-06-10T21:31:57.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: failed to encode map e282 with expected crc 2023-06-10T21:31:57.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/667581625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:58.133 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 45311 45295 80.8405 51 0.0156533 0.0123626 2023-06-10T21:31:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: failed to encode map e282 with expected crc 2023-06-10T21:31:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: osdmap e282: 8 total, 8 up, 6 in 2023-06-10T21:31:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: failed to encode map e282 with expected crc 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/667581625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: failed to encode map e282 with expected crc 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: osdmap e282: 8 total, 8 up, 6 in 2023-06-10T21:31:58.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: failed to encode map e282 with expected crc 2023-06-10T21:31:58.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/667581625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:31:58.463 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:31:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:58.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:31:58.527 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:31:58.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:31:58.644 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:58.816 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:31:58.816 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:31:58.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.816 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:45975 11.8 11:1ab4284f:::benchmark_data_smithi064_385725_object45974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-06-10T21:31:58.816 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.797073+0000", 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011590428999999999, 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011625931000000001, 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:31:58.817 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45975 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797073+0000", 2023-06-10T21:31:58.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797073+0000", 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797071+0000", 2023-06-10T21:31:58.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:58.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797097+0000", 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5732999999999999e-05 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797110+0000", 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2622999999999999e-05 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797118+0000", 2023-06-10T21:31:58.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2190000000000004e-06 2023-06-10T21:31:58.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797147+0000", 2023-06-10T21:31:58.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9359000000000001e-05 2023-06-10T21:31:58.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797301+0000", 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153566 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:31:58.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797471+0000", 2023-06-10T21:31:58.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016997299999999999 2023-06-10T21:31:58.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:31:58.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.802583+0000", 2023-06-10T21:31:58.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051124070000000002 2023-06-10T21:31:58.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:58.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808202+0000", 2023-06-10T21:31:58.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056191239999999996 2023-06-10T21:31:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.833 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:45983 11.a 11:5eb156b1:::benchmark_data_smithi064_385725_object45982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-06-10T21:31:58.833 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.797175+0000", 2023-06-10T21:31:58.833 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011487638999999999, 2023-06-10T21:31:58.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011603501, 2023-06-10T21:31:58.834 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.834 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:31:58.834 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:31:58.835 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:31:58.835 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:31:58.835 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45983 2023-06-10T21:31:58.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.836 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797175+0000", 2023-06-10T21:31:58.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797175+0000", 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797175+0000", 2023-06-10T21:31:58.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:58.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797202+0000", 2023-06-10T21:31:58.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7314999999999999e-05 2023-06-10T21:31:58.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797210+0000", 2023-06-10T21:31:58.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0600000000000008e-06 2023-06-10T21:31:58.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797216+0000", 2023-06-10T21:31:58.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0889999999999996e-06 2023-06-10T21:31:58.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799387+0000", 2023-06-10T21:31:58.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002170951 2023-06-10T21:31:58.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799538+0000", 2023-06-10T21:31:58.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015113600000000001 2023-06-10T21:31:58.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:31:58.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799718+0000", 2023-06-10T21:31:58.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017919400000000001 2023-06-10T21:31:58.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.855 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.856 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:45984 11.b 11:d8f6228b:::benchmark_data_smithi064_385725_object45983:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-06-10T21:31:58.856 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.797230+0000", 2023-06-10T21:31:58.856 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011433064999999999, 2023-06-10T21:31:58.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011637385, 2023-06-10T21:31:58.857 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.857 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:31:58.857 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:31:58.857 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:31:58.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:31:58.858 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45984 2023-06-10T21:31:58.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.858 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797230+0000", 2023-06-10T21:31:58.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797230+0000", 2023-06-10T21:31:58.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797230+0000", 2023-06-10T21:31:58.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:58.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797254+0000", 2023-06-10T21:31:58.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4097e-05 2023-06-10T21:31:58.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797261+0000", 2023-06-10T21:31:58.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8730000000000007e-06 2023-06-10T21:31:58.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797268+0000", 2023-06-10T21:31:58.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1909999999999997e-06 2023-06-10T21:31:58.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797306+0000", 2023-06-10T21:31:58.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8195999999999998e-05 2023-06-10T21:31:58.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797435+0000", 2023-06-10T21:31:58.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012873999999999999 2023-06-10T21:31:58.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:31:58.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.797561+0000", 2023-06-10T21:31:58.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000126029 2023-06-10T21:31:58.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:58.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807993+0000", 2023-06-10T21:31:58.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010432074 2023-06-10T21:31:58.870 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.870 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.871 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:45986 11.8 11:107e8580:::benchmark_data_smithi064_385725_object45985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-06-10T21:31:58.872 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.799400+0000", 2023-06-10T21:31:58.872 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0092633330000000003, 2023-06-10T21:31:58.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095175290000000003, 2023-06-10T21:31:58.873 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.873 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:31:58.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:31:58.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:31:58.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:31:58.874 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45986 2023-06-10T21:31:58.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.875 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799400+0000", 2023-06-10T21:31:58.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799400+0000", 2023-06-10T21:31:58.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799399+0000", 2023-06-10T21:31:58.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:31:58.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799425+0000", 2023-06-10T21:31:58.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6420000000000001e-05 2023-06-10T21:31:58.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799438+0000", 2023-06-10T21:31:58.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2951e-05 2023-06-10T21:31:58.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799446+0000", 2023-06-10T21:31:58.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.089e-06 2023-06-10T21:31:58.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799492+0000", 2023-06-10T21:31:58.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5623e-05 2023-06-10T21:31:58.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799616+0000", 2023-06-10T21:31:58.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124244 2023-06-10T21:31:58.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:31:58.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.799738+0000", 2023-06-10T21:31:58.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012228200000000001 2023-06-10T21:31:58.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:31:58.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.802591+0000", 2023-06-10T21:31:58.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028527460000000002 2023-06-10T21:31:58.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:31:58.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808236+0000", 2023-06-10T21:31:58.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056452309999999997 2023-06-10T21:31:58.892 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.892 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.893 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.894 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.36159.0:45983 11.a e282/269)", 2023-06-10T21:31:58.894 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.807892+0000", 2023-06-10T21:31:58.894 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00077091400000000004, 2023-06-10T21:31:58.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010713820000000001, 2023-06-10T21:31:58.895 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.895 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:31:58.895 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807892+0000", 2023-06-10T21:31:58.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807892+0000", 2023-06-10T21:31:58.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807891+0000", 2023-06-10T21:31:58.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:31:58.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807899+0000", 2023-06-10T21:31:58.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9510000000000006e-06 2023-06-10T21:31:58.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807919+0000", 2023-06-10T21:31:58.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9587000000000001e-05 2023-06-10T21:31:58.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.807927+0000", 2023-06-10T21:31:58.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8689999999999993e-06 2023-06-10T21:31:58.905 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.905 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.905 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.906 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:45979 11.3 e282/272 11:c200fe80:::benchmark_data_smithi064_385725_object45978:head v 282'2897, mlcod=282'2895)", 2023-06-10T21:31:58.906 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.808035+0000", 2023-06-10T21:31:58.907 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00062788899999999996, 2023-06-10T21:31:58.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095660099999999996, 2023-06-10T21:31:58.907 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.908 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:31:58.909 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808035+0000", 2023-06-10T21:31:58.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808035+0000", 2023-06-10T21:31:58.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808035+0000", 2023-06-10T21:31:58.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:58.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808060+0000", 2023-06-10T21:31:58.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.514e-05 2023-06-10T21:31:58.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808067+0000", 2023-06-10T21:31:58.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7889999999999997e-06 2023-06-10T21:31:58.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808072+0000", 2023-06-10T21:31:58.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8199999999999996e-06 2023-06-10T21:31:58.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808108+0000", 2023-06-10T21:31:58.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6112999999999998e-05 2023-06-10T21:31:58.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808126+0000", 2023-06-10T21:31:58.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8839e-05 2023-06-10T21:31:58.921 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.921 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.922 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.922 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:45981 11.3 e282/272)", 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.808102+0000", 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00056106999999999999, 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00092084399999999998, 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:31:58.923 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808102+0000", 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808102+0000", 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808102+0000", 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808125+0000", 2023-06-10T21:31:58.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2830999999999999e-05 2023-06-10T21:31:58.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808131+0000", 2023-06-10T21:31:58.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0000000000000002e-06 2023-06-10T21:31:58.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808135+0000", 2023-06-10T21:31:58.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.352e-06 2023-06-10T21:31:58.929 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.929 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.930 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.930 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:45989 11.9 11:98842442:::benchmark_data_smithi064_385725_object45988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e282)", 2023-06-10T21:31:58.931 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:58.808177+0000", 2023-06-10T21:31:58.931 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00048651800000000002, 2023-06-10T21:31:58.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00088624899999999998, 2023-06-10T21:31:58.931 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:31:58.932 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:31:58.932 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:31:58.933 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:31:58.933 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:31:58.936 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 45989 2023-06-10T21:31:58.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.937 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:31:58.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:31:58.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808177+0000", 2023-06-10T21:31:58.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:31:58.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808177+0000", 2023-06-10T21:31:58.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:31:58.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:31:58.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808175+0000", 2023-06-10T21:31:58.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:31:58.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:31:58.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808199+0000", 2023-06-10T21:31:58.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3589999999999999e-05 2023-06-10T21:31:58.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:31:58.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808208+0000", 2023-06-10T21:31:58.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4429999999999992e-06 2023-06-10T21:31:58.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:31:58.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808215+0000", 2023-06-10T21:31:58.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5250000000000002e-06 2023-06-10T21:31:58.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:31:58.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808389+0000", 2023-06-10T21:31:58.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017432799999999999 2023-06-10T21:31:58.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:31:58.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808504+0000", 2023-06-10T21:31:58.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114885 2023-06-10T21:31:58.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:31:58.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:31:58.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:31:58.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:58.808638+0000", 2023-06-10T21:31:58.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013433600000000001 2023-06-10T21:31:58.950 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.950 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:31:58.950 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.950 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:31:58.951 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:31:58.951 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 8 2023-06-10T21:31:58.951 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:31:59.133 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 46374 46358 80.4405 66.4375 0.0121935 0.0124248 2023-06-10T21:31:59.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:31:58 smithi064 ceph-mon[186706]: pgmap v1000: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 937 op/s 2023-06-10T21:31:59.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:31:58 smithi064 ceph-mon[180297]: pgmap v1000: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 937 op/s 2023-06-10T21:31:59.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:31:58 smithi152 ceph-mon[156819]: pgmap v1000: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 59 MiB/s wr, 937 op/s 2023-06-10T21:31:59.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:31:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:59.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:31:59.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:59.815 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:31:59.861 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:31:59.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:31:59.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:31:59.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:31:59.994 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:00.042 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:00.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:00.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:00.134 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 47276 47260 79.7901 56.375 0.00359187 0.0125113 2023-06-10T21:32:00.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:00.176 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:00.218 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:00.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:00.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:00.369 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:00.414 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:00.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:00.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:00.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:00.556 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:32:00.606 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:00.829 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:32:00.874 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:00.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:00.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:01.086 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:01.134 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 48147 48131 79.123 54.4375 0.0179245 0.0126341 2023-06-10T21:32:01.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:00 smithi064 ceph-mon[186706]: pgmap v1001: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s 2023-06-10T21:32:01.164 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:00 smithi064 ceph-mon[180297]: pgmap v1001: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s 2023-06-10T21:32:01.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:00 smithi152 ceph-mon[156819]: pgmap v1001: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 57 MiB/s wr, 904 op/s 2023-06-10T21:32:02.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:02.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:02.133 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 48971 48955 78.4149 51.5 0.0043541 0.012745 2023-06-10T21:32:02.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:02.289 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:02.510 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:02.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:02.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:02.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:02.692 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:02.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:02.804 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:02.859 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:32:02.897 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:02.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:03.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:03.033 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:03.081 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:03.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:03.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:03.138 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:32:03.136542+0000 min lat: 0.00116152 max lat: 0.294538 avg lat: 0.0128385 2023-06-10T21:32:03.139 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:32:03.139 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 49868 49852 77.846 56.0625 0.023655 0.0128385 2023-06-10T21:32:03.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:02 smithi064 ceph-mon[180297]: pgmap v1002: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-06-10T21:32:03.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:02 smithi064 ceph-mon[186706]: pgmap v1002: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-06-10T21:32:03.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:02] "GET /metrics HTTP/1.1" 200 224002 "" "Prometheus/2.33.4" 2023-06-10T21:32:03.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:03.197 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:02 smithi152 ceph-mon[156819]: pgmap v1002: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-06-10T21:32:03.223 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:03.271 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:03.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:03.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:03.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:03.404 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:03.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:03.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:03.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-10T21:32:03.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:03.584 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:03.630 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:03.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:03.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:03.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:03.766 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:03.811 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:03.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:03.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:03.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3697618746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-10T21:32:03.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:04.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:04.139 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 50767 50751 77.3167 56.1875 0.0122915 0.0129285 2023-06-10T21:32:04.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:04.164 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3697618746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-10T21:32:04.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:04.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3697618746' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-10T21:32:05.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 51862 51846 77.1053 68.4375 0.00862141 0.0129635 2023-06-10T21:32:05.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:04 smithi064 ceph-mon[180297]: pgmap v1003: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-06-10T21:32:05.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:04 smithi064 ceph-mon[186706]: pgmap v1003: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-06-10T21:32:05.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:04 smithi152 ceph-mon[156819]: pgmap v1003: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 877 op/s 2023-06-10T21:32:06.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 52593 52577 76.3742 45.6875 0.0116384 0.0130866 2023-06-10T21:32:06.487 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:06.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:06.600 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:06.627 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:06.674 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:06.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:06.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-10T21:32:06.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:06.882 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:07.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:06 smithi064 ceph-mon[180297]: pgmap v1004: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:32:07.011 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:06 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:32:07.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:06 smithi064 ceph-mon[186706]: pgmap v1004: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:32:07.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:06 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:32:07.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:07.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:07.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 53532 53516 75.9715 58.6875 0.0100437 0.0131568 2023-06-10T21:32:07.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:06 smithi152 ceph-mon[156819]: pgmap v1004: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:32:07.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:06 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:32:07.228 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:07.279 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:07.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:07.379 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:07.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:07.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:07.615 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:07.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:07.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:07.671 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:32:07.671 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:32:07.672 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:32:07.672 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:32:07.672 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:32:07.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:07.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:07.968 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:32:08.140 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 54328 54312 75.389 49.75 0.0174918 0.0132585 2023-06-10T21:32:08.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:08.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:08.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:09.142 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 55015 54999 74.6825 42.9375 0.0124687 0.0133843 2023-06-10T21:32:09.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 52 MiB/s wr, 828 op/s 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: osdmap e283: 8 total, 8 up, 6 in 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:08 smithi152 ceph-mon[156819]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:09.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 52 MiB/s wr, 828 op/s 2023-06-10T21:32:09.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:09.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:32:09.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:09.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: osdmap e283: 8 total, 8 up, 6 in 2023-06-10T21:32:09.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[186706]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: pgmap v1005: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 52 MiB/s wr, 828 op/s 2023-06-10T21:32:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:09.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:32:09.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/958940241' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:09.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: osdmap e283: 8 total, 8 up, 6 in 2023-06-10T21:32:09.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:08 smithi064 ceph-mon[180297]: failed to encode map e283 with expected crc 2023-06-10T21:32:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:09.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:09.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:09.550 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:09.595 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:09.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:09.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:09.726 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:09.876 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:09.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:09.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:10.067 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:10.144 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 55777 55761 74.1018 47.625 0.0134007 0.0134891 2023-06-10T21:32:10.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:32:10.854 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:10.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:10.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[180297]: pgmap v1007: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-10T21:32:10.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:10.958 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:10.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:11.044 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:11.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 56586 56570 73.6113 50.5625 0.0260146 0.0135771 2023-06-10T21:32:11.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:11.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[186706]: pgmap v1007: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-10T21:32:11.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:11.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:11.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:11.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:11.429 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:10 smithi152 ceph-mon[156819]: pgmap v1007: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-10T21:32:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:11.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:11.639 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:11.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:11.738 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:32:11.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 1 1 2023-06-10T21:32:11.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:11.755 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:32:11.789 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:32:11.818 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:11.847 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:32:11.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:11.906 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:32:11.957 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:32:12.001 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:32:12.002 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:12.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:12.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:12.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:32:12.086 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:32:12.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:12.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.120 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.120 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.120 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.120 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.120 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:32:12.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 57339 57323 73.0697 47.0625 0.0210885 0.0136804 2023-06-10T21:32:12.148 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:12.193 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:12.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:12.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:12.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2208282183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:12.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2208282183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:12.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:12.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2208282183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:13.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:32:13.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 58242 58226 72.7372 56.4375 0.027032 0.0137425 2023-06-10T21:32:13.162 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:12] "GET /metrics HTTP/1.1" 200 223994 "" "Prometheus/2.33.4" 2023-06-10T21:32:13.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:12 smithi064 ceph-mon[180297]: pgmap v1008: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s 2023-06-10T21:32:13.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:12 smithi064 ceph-mon[186706]: pgmap v1008: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s 2023-06-10T21:32:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:12 smithi152 ceph-mon[156819]: pgmap v1008: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s 2023-06-10T21:32:14.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 58754 58738 71.9387 32 0.182863 0.0138646 2023-06-10T21:32:14.543 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:14.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:14.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:14.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:14.735 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:14.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:14.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:15.044 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:15.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:15.124 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:14 smithi064 ceph-mon[180297]: pgmap v1009: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-06-10T21:32:15.126 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:14 smithi064 ceph-mon[186706]: pgmap v1009: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-06-10T21:32:15.128 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:15.139 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:15.152 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 59559 59543 71.5229 50.3125 0.0202882 0.0139753 2023-06-10T21:32:15.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:15.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:15.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:15.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:14 smithi152 ceph-mon[156819]: pgmap v1009: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-06-10T21:32:15.455 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:15.501 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:15.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:15.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:15.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:16.127 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:16.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:16.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 60351 60335 71.1075 49.5 0.0304648 0.0140567 2023-06-10T21:32:16.164 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:32:16.164 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:32:16.233 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:16.318 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:16.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:32:17.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 61095 61079 70.652 46.5 0.00785445 0.0141212 2023-06-10T21:32:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:16 smithi064 ceph-mon[180297]: pgmap v1010: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-06-10T21:32:17.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:16 smithi064 ceph-mon[186706]: pgmap v1010: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-06-10T21:32:17.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:16 smithi152 ceph-mon[156819]: pgmap v1010: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 51 MiB/s wr, 816 op/s 2023-06-10T21:32:18.145 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 61883 61867 70.2629 49.25 0.0028991 0.0142183 2023-06-10T21:32:18.696 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:18.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:18.804 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:18.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:19.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:32:19.146 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 62521 62505 69.7204 39.875 0.0172944 0.0143362 2023-06-10T21:32:19.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:19.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:19.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[180297]: pgmap v1011: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 782 op/s 2023-06-10T21:32:19.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:19.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:19.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[186706]: pgmap v1011: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 782 op/s 2023-06-10T21:32:19.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:19.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:19.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:19.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:19 smithi152 ceph-mon[156819]: pgmap v1011: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 782 op/s 2023-06-10T21:32:19.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:19.422 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:19.424 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:19.486 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:19.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:19.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:19.650 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:19.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:19.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:19.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:20.008 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:20.107 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:20.153 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 63140 63124 69.172 38.6875 0.0328584 0.0144493 2023-06-10T21:32:20.526 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:20.529 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:20.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:20.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:20.703 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:21.153 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 63966 63950 68.8673 51.625 0.025512 0.0145143 2023-06-10T21:32:21.200 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:32:21.385 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-06-10T21:32:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:21 smithi152 ceph-mon[156819]: pgmap v1012: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 792 op/s 2023-06-10T21:32:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4105253029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:32:21.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[180297]: pgmap v1012: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 792 op/s 2023-06-10T21:32:21.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4105253029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:32:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[186706]: pgmap v1012: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 50 MiB/s wr, 792 op/s 2023-06-10T21:32:21.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4105253029' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:32:21.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:32:22.053 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:22.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:22.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:22.151 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 64588 64572 68.3591 38.875 0.0145734 0.0146229 2023-06-10T21:32:22.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:22.266 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:22.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:32:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:32:22.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: osdmap e284: 8 total, 8 up, 6 in 2023-06-10T21:32:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[180297]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:22.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:32:22.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:32:22.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: osdmap e284: 8 total, 8 up, 6 in 2023-06-10T21:32:22.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:22 smithi064 ceph-mon[186706]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:22.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:32:22.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3777808673' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:32:22.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: osdmap e284: 8 total, 8 up, 6 in 2023-06-10T21:32:22.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.451 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:22 smithi152 ceph-mon[156819]: failed to encode map e284 with expected crc 2023-06-10T21:32:22.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:22] "GET /metrics HTTP/1.1" 200 223974 "" "Prometheus/2.33.4" 2023-06-10T21:32:22.972 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:22.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:23.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:23.156 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:32:23.150466+0000 min lat: 0.00116152 max lat: 0.345297 avg lat: 0.0147157 2023-06-10T21:32:23.157 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:32:23.157 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 65264 65248 67.9231 42.25 0.032418 0.0147157 2023-06-10T21:32:23.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:23.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:23 smithi064 ceph-mon[186706]: pgmap v1014: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:32:23.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:23 smithi064 ceph-mon[180297]: pgmap v1014: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:32:23.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:23 smithi152 ceph-mon[156819]: pgmap v1014: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-10T21:32:24.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:24.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:24.152 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 65953 65937 67.5157 43.0625 0.00286084 0.0147909 2023-06-10T21:32:24.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:24.188 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:24.236 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:24.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:24.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:24.373 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:24.422 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:24.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:24.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:24.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:24.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:24.700 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:32:24.737 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:32:24.787 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:32:24.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:32:24.865 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:32:24.903 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:32:24.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:32:24.994 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:32:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.998 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.999 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.999 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.999 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:32:24.999 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:32:25.153 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 66731 66715 67.2106 48.625 0.0183127 0.0148718 2023-06-10T21:32:25.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:32:25.198 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:32:25.311 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:25 smithi064 ceph-mon[180297]: pgmap v1015: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 731 op/s 2023-06-10T21:32:25.311 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2672559514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:25.318 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:25 smithi064 ceph-mon[186706]: pgmap v1015: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 731 op/s 2023-06-10T21:32:25.319 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2672559514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:25 smithi152 ceph-mon[156819]: pgmap v1015: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 46 MiB/s wr, 731 op/s 2023-06-10T21:32:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2672559514' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:26.156 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 67217 67201 66.6224 30.375 0.0162244 0.0150034 2023-06-10T21:32:26.498 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:26.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:26.611 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:26.640 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:26.683 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:26.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:26.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:26.819 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:26.867 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:26.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:26.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:26.982 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:26.999 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:27.043 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:27.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:27.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:27.166 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 67938 67922 66.2758 45.0625 0.0543439 0.0150807 2023-06-10T21:32:27.197 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67942 11.0 e284/269 11:07df3d82:::benchmark_data_smithi064_385725_object67941:head v 284'4223, mlcod=284'4221)", 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.174171+0000", 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.018528395, 2023-06-10T21:32:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018557974000000001, 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174171+0000", 2023-06-10T21:32:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174171+0000", 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174170+0000", 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174191+0000", 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0837000000000001e-05 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174201+0000", 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9869999999999994e-06 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.174207+0000", 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.37e-06 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.176300+0000", 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020927720000000001 2023-06-10T21:32:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.176323+0000", 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2631000000000001e-05 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67953 11.f e284/269 11:f589f793:::benchmark_data_smithi064_385725_object67952:head v 284'4235, mlcod=284'4233)", 2023-06-10T21:32:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.176988+0000", 2023-06-10T21:32:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.015711467, 2023-06-10T21:32:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015819374000000001, 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.176988+0000", 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.176988+0000", 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.176987+0000", 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.177027+0000", 2023-06-10T21:32:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0405999999999999e-05 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.177040+0000", 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3040000000000001e-05 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.177048+0000", 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.177116+0000", 2023-06-10T21:32:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7756000000000007e-05 2023-06-10T21:32:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.177136+0000", 2023-06-10T21:32:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9755e-05 2023-06-10T21:32:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67946 11.6 e284/269 11:640cb1f8:::benchmark_data_smithi064_385725_object67945:head v 284'4255, mlcod=284'4253)", 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.180571+0000", 2023-06-10T21:32:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012128420000000001, 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012311102000000001, 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180571+0000", 2023-06-10T21:32:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180571+0000", 2023-06-10T21:32:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180570+0000", 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180599+0000", 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9261999999999999e-05 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180617+0000", 2023-06-10T21:32:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7368999999999999e-05 2023-06-10T21:32:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180625+0000", 2023-06-10T21:32:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6780000000000006e-06 2023-06-10T21:32:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180672+0000", 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7071999999999999e-05 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180693+0000", 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0387e-05 2023-06-10T21:32:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67947 11.6 e284/269 11:61ef1212:::benchmark_data_smithi064_385725_object67946:head v 284'4256, mlcod=284'4253)", 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.180642+0000", 2023-06-10T21:32:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.01205736, 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012340235, 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180642+0000", 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180642+0000", 2023-06-10T21:32:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180642+0000", 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180664+0000", 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2425e-05 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180671+0000", 2023-06-10T21:32:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9299999999999997e-06 2023-06-10T21:32:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.180676+0000", 2023-06-10T21:32:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3340000000000001e-06 2023-06-10T21:32:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.182975+0000", 2023-06-10T21:32:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002298507 2023-06-10T21:32:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.183000+0000", 2023-06-10T21:32:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4777e-05 2023-06-10T21:32:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67952 11.5 e284/272 11:a6bbc66f:::benchmark_data_smithi064_385725_object67951:head v 284'4393, mlcod=284'4390)", 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.181059+0000", 2023-06-10T21:32:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011639956, 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011993246000000001, 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181059+0000", 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181059+0000", 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181059+0000", 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181083+0000", 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4003999999999999e-05 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181094+0000", 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1122e-05 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181101+0000", 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0219999999999999e-06 2023-06-10T21:32:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181150+0000", 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9360999999999997e-05 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181172+0000", 2023-06-10T21:32:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1976000000000001e-05 2023-06-10T21:32:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67949 11.0 e284/269 11:06f1dad0:::benchmark_data_smithi064_385725_object67948:head v 284'4224, mlcod=284'4221)", 2023-06-10T21:32:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.181114+0000", 2023-06-10T21:32:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011585763000000001, 2023-06-10T21:32:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012007840000000001, 2023-06-10T21:32:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181114+0000", 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181114+0000", 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181113+0000", 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181136+0000", 2023-06-10T21:32:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3269e-05 2023-06-10T21:32:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181144+0000", 2023-06-10T21:32:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2320000000000004e-06 2023-06-10T21:32:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181149+0000", 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9790000000000001e-06 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.182833+0000", 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016843489999999999 2023-06-10T21:32:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.182853+0000", 2023-06-10T21:32:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0214999999999999e-05 2023-06-10T21:32:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67954 11.5 e284/272 11:a967015a:::benchmark_data_smithi064_385725_object67953:head v 284'4394, mlcod=284'4390)", 2023-06-10T21:32:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.181166+0000", 2023-06-10T21:32:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.011533277999999999, 2023-06-10T21:32:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012089348, 2023-06-10T21:32:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181166+0000", 2023-06-10T21:32:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181166+0000", 2023-06-10T21:32:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181166+0000", 2023-06-10T21:32:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181187+0000", 2023-06-10T21:32:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0899000000000001e-05 2023-06-10T21:32:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181194+0000", 2023-06-10T21:32:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3320000000000003e-06 2023-06-10T21:32:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.181199+0000", 2023-06-10T21:32:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1930000000000001e-06 2023-06-10T21:32:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.185492+0000", 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042924149999999999 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.185512+0000", 2023-06-10T21:32:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0577999999999998e-05 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67945 11.d e284/269 11:b33ff134:::benchmark_data_smithi064_385725_object67944:head v 284'4235, mlcod=284'4233)", 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.186429+0000", 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0062699339999999996, 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006887799, 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186429+0000", 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186429+0000", 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186428+0000", 2023-06-10T21:32:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186458+0000", 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9484000000000001e-05 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186470+0000", 2023-06-10T21:32:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2636999999999999e-05 2023-06-10T21:32:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186479+0000", 2023-06-10T21:32:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2439999999999993e-06 2023-06-10T21:32:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186744+0000", 2023-06-10T21:32:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000265844 2023-06-10T21:32:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186765+0000", 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0605999999999999e-05 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67948 11.c e284/269 11:37b396e8:::benchmark_data_smithi064_385725_object67947:head v 284'4238, mlcod=284'4236)", 2023-06-10T21:32:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.186491+0000", 2023-06-10T21:32:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0062078120000000001, 2023-06-10T21:32:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068624810000000001, 2023-06-10T21:32:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186491+0000", 2023-06-10T21:32:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186491+0000", 2023-06-10T21:32:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186491+0000", 2023-06-10T21:32:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186515+0000", 2023-06-10T21:32:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3651000000000001e-05 2023-06-10T21:32:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186521+0000", 2023-06-10T21:32:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5030000000000002e-06 2023-06-10T21:32:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.186526+0000", 2023-06-10T21:32:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.741e-06 2023-06-10T21:32:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.187056+0000", 2023-06-10T21:32:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00053041500000000001 2023-06-10T21:32:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.187078+0000", 2023-06-10T21:32:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1781999999999999e-05 2023-06-10T21:32:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67955 11.9 e284/269 11:9112e5bb:::benchmark_data_smithi064_385725_object67954:head v 284'4254, mlcod=284'4252)", 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.188310+0000", 2023-06-10T21:32:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0043892469999999998, 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050994279999999996, 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188310+0000", 2023-06-10T21:32:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188310+0000", 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188309+0000", 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188333+0000", 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3394999999999999e-05 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188344+0000", 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1632e-05 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188352+0000", 2023-06-10T21:32:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3359999999999997e-06 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.189056+0000", 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000704311 2023-06-10T21:32:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.189074+0000", 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8621999999999999e-05 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67956 11.5 e284/272 11:ac18e62e:::benchmark_data_smithi064_385725_object67955:head v 284'4395, mlcod=284'4390)", 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.188371+0000", 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0043282199999999998, 2023-06-10T21:32:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050814700000000003, 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188371+0000", 2023-06-10T21:32:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188371+0000", 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188371+0000", 2023-06-10T21:32:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188391+0000", 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0196000000000001e-05 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188398+0000", 2023-06-10T21:32:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9519999999999997e-06 2023-06-10T21:32:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188403+0000", 2023-06-10T21:32:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8790000000000002e-06 2023-06-10T21:32:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188457+0000", 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4412e-05 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.188476+0000", 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8383e-05 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:67957 11.7 e284/269 11:e7975291:::benchmark_data_smithi064_385725_object67956:head v 284'4243, mlcod=284'4241)", 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:27.190818+0000", 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001880842, 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026675539999999999, 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190818+0000", 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190818+0000", 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190818+0000", 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190841+0000", 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3339000000000001e-05 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190852+0000", 2023-06-10T21:32:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0829000000000001e-05 2023-06-10T21:32:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190859+0000", 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8889999999999996e-06 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190902+0000", 2023-06-10T21:32:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3031000000000002e-05 2023-06-10T21:32:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:27.190922+0000", 2023-06-10T21:32:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9956e-05 2023-06-10T21:32:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:32:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 12 2023-06-10T21:32:27.330 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:32:27.330 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:27.331 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:27.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:27 smithi152 ceph-mon[156819]: pgmap v1016: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-06-10T21:32:27.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:27 smithi064 ceph-mon[186706]: pgmap v1016: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-06-10T21:32:27.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:27 smithi064 ceph-mon[180297]: pgmap v1016: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 45 MiB/s wr, 724 op/s 2023-06-10T21:32:28.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 68726 68710 66.0115 49.25 0.0337148 0.0151425 2023-06-10T21:32:28.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:32:29.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 69285 69269 65.541 34.9375 0.00656874 0.0152509 2023-06-10T21:32:29.215 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:29.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:29.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:29.445 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[180297]: pgmap v1017: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s 2023-06-10T21:32:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[186706]: pgmap v1017: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s 2023-06-10T21:32:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:29.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:29 smithi152 ceph-mon[156819]: pgmap v1017: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 44 MiB/s wr, 702 op/s 2023-06-10T21:32:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:30.013 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:32:30.179 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 69880 69864 65.115 37.1875 0.0226197 0.0153528 2023-06-10T21:32:30.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:30.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:30.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:30.384 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:30.429 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:30.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:30.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:30.580 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:30.626 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:30.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:30.733 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:30.763 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:30.808 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:30.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:30.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:31.005 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:31.172 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 70598 70582 64.8175 44.875 0.0221721 0.0154221 2023-06-10T21:32:31.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[180297]: pgmap v1018: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s 2023-06-10T21:32:31.282 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:32:31.283 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:32:31.283 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:31.378 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:31.595 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:31.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:31.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[186706]: pgmap v1018: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s 2023-06-10T21:32:31.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:32:31.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:32:31.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:31.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:31 smithi152 ceph-mon[156819]: pgmap v1018: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 42 MiB/s wr, 675 op/s 2023-06-10T21:32:31.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:32:31.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:32:31.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:31.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:31.734 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:31.783 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:31.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:31.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:31.919 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:31.964 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:31.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:32.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:32.125 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:32.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 71304 71288 64.515 44.125 0.0217482 0.0154912 2023-06-10T21:32:32.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:32.357 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:32.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:32.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:32.568 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:32.775 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:32.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:32.884 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:32] "GET /metrics HTTP/1.1" 200 223974 "" "Prometheus/2.33.4" 2023-06-10T21:32:32.894 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:33.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:33.059 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:33.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:33.162 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:33.175 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 71884 71868 64.1115 36.25 0.00324666 0.0155584 2023-06-10T21:32:33.192 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:33.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:33.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:33.319 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:32:33.342 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:33.344 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:32:33.368 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:33.414 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:33.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:33.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:33.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[180297]: pgmap v1019: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 49 MiB/s wr, 777 op/s 2023-06-10T21:32:33.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:32:33.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:33.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:33.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:33.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[186706]: pgmap v1019: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 49 MiB/s wr, 777 op/s 2023-06-10T21:32:33.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:32:33.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:33.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:33.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:33.553 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:33 smithi152 ceph-mon[156819]: pgmap v1019: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 49 MiB/s wr, 777 op/s 2023-06-10T21:32:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:32:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:32:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:34.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 72387 72371 63.6499 31.4375 0.0112744 0.0157044 2023-06-10T21:32:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:32:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: osdmap e285: 8 total, 8 up, 6 in 2023-06-10T21:32:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: failed to encode map e285 with expected crc 2023-06-10T21:32:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[186706]: failed to encode map e285 with expected crc 2023-06-10T21:32:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:32:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: osdmap e285: 8 total, 8 up, 6 in 2023-06-10T21:32:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: failed to encode map e285 with expected crc 2023-06-10T21:32:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:34 smithi064 ceph-mon[180297]: failed to encode map e285 with expected crc 2023-06-10T21:32:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2642023621' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:32:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2541301158' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: osdmap e285: 8 total, 8 up, 6 in 2023-06-10T21:32:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: failed to encode map e285 with expected crc 2023-06-10T21:32:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:34 smithi152 ceph-mon[156819]: failed to encode map e285 with expected crc 2023-06-10T21:32:35.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 73157 73141 63.4344 48.125 0.00632987 0.0157452 2023-06-10T21:32:35.615 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:35.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:35.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:35 smithi064 ceph-mon[180297]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s 2023-06-10T21:32:35.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:35 smithi064 ceph-mon[186706]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s 2023-06-10T21:32:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:35 smithi152 ceph-mon[156819]: pgmap v1021: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 42 MiB/s wr, 672 op/s 2023-06-10T21:32:35.724 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:35.751 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:35.797 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:35.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:35.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:36.005 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:36.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 73804 73788 63.119 40.4375 0.0169946 0.0158374 2023-06-10T21:32:36.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:36.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:36.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:36.370 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 4 1 2023-06-10T21:32:36.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:36.583 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:36.876 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:36.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:36.952 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:32:36.993 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:32:36.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:37.025 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:37.032 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:32:37.071 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:37.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:37.084 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:32:37.115 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:32:37.153 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:32:37.180 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:37.182 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 74452 74436 62.8119 40.5 0.0140173 0.0159132 2023-06-10T21:32:37.184 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:32:37.208 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:37.223 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:32:37.231 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:32:37.232 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:32:37.241 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.242 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.243 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.243 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:32:37.252 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:37.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:37.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:37.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:37.394 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:37.436 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:37.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:37.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:37.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:37 smithi064 ceph-mon[180297]: pgmap v1022: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 659 op/s 2023-06-10T21:32:37.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3091463244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:37.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:37 smithi064 ceph-mon[186706]: pgmap v1022: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 659 op/s 2023-06-10T21:32:37.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3091463244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:37.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:37 smithi152 ceph-mon[156819]: pgmap v1022: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 659 op/s 2023-06-10T21:32:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3091463244' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:37.761 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:37.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:37.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:37.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:37.952 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:38.096 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:38.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:38.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:38.180 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 75213 75197 62.6087 47.5625 0.0165532 0.0159634 2023-06-10T21:32:38.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:38.230 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:38.273 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:38.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:38.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:32:38.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:38.405 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:38.451 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:38.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:38.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:38.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:38.585 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:39.180 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 75802 75786 62.2694 36.8125 0.0197801 0.016053 2023-06-10T21:32:39.230 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:39.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:39.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:39.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:32:39.320 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:39.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:39.482 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:39 smithi064 ceph-mon[180297]: pgmap v1023: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 654 op/s 2023-06-10T21:32:39.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:39 smithi064 ceph-mon[186706]: pgmap v1023: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 654 op/s 2023-06-10T21:32:39.483 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:39.669 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:39.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:39.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:39.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:39 smithi152 ceph-mon[156819]: pgmap v1023: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 41 MiB/s wr, 654 op/s 2023-06-10T21:32:39.781 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:39.891 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:39.937 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:39.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:40.007 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:40.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:40.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:40.180 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 76454 76438 61.9901 40.75 0.0126801 0.0161248 2023-06-10T21:32:41.076 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:41.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:41.180 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 77002 76986 61.6347 34.25 0.0217146 0.0162177 2023-06-10T21:32:41.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:41.201 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:41.274 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:41.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:32:41.406 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:41.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:41.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:41.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:41.539 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:41.582 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:41.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:41.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:41 smithi064 ceph-mon[186706]: pgmap v1024: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 655 op/s 2023-06-10T21:32:41.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:41 smithi064 ceph-mon[180297]: pgmap v1024: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 655 op/s 2023-06-10T21:32:41.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:41 smithi152 ceph-mon[156819]: pgmap v1024: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 655 op/s 2023-06-10T21:32:41.719 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:41.764 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:41.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:41.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:41.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:41.904 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:41.951 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:41.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:41.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:42.058 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:42.087 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:42.134 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:42.181 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 77666 77650 61.38 41.5 0.00943581 0.0162861 2023-06-10T21:32:42.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:42.307 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:43.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:43.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:43.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:43.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:42] "GET /metrics HTTP/1.1" 200 224011 "" "Prometheus/2.33.4" 2023-06-10T21:32:43.181 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:32:43.179040+0000 min lat: 0.00116152 max lat: 0.376337 avg lat: 0.016356 2023-06-10T21:32:43.182 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:32:43.182 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 78310 78294 61.116 40.25 0.029642 0.016356 2023-06-10T21:32:43.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:43.286 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:43.330 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:43.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:43.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:43.465 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:43.510 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:43.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:43.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:43.645 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:43 smithi064 ceph-mon[180297]: pgmap v1025: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 650 op/s 2023-06-10T21:32:43.646 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:43 smithi064 ceph-mon[186706]: pgmap v1025: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 650 op/s 2023-06-10T21:32:43.648 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:43 smithi152 ceph-mon[156819]: pgmap v1025: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 41 MiB/s wr, 650 op/s 2023-06-10T21:32:44.021 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:44.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:44.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:44.161 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:44.188 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 78903 78887 60.8145 37.0625 0.0246235 0.0164364 2023-06-10T21:32:44.206 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:44.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:44.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:44.339 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:44.390 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:44.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:44.439 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-06-10T21:32:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2126079196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:44.526 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:44.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2126079196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2126079196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:32:44.570 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:44.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:44.582 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:44.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:44.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:45.187 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 79421 79405 60.4679 32.375 0.00962139 0.0164938 2023-06-10T21:32:45.304 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:32:45.305 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:32:45.315 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:45.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:45.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:45.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:45.443 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:32:45.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[186706]: pgmap v1026: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s 2023-06-10T21:32:45.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[186706]: failed to encode map e286 with expected crc 2023-06-10T21:32:45.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:32:45.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[186706]: osdmap e286: 8 total, 8 up, 6 in 2023-06-10T21:32:45.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:32:45.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[180297]: pgmap v1026: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s 2023-06-10T21:32:45.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[180297]: failed to encode map e286 with expected crc 2023-06-10T21:32:45.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:32:45.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[180297]: osdmap e286: 8 total, 8 up, 6 in 2023-06-10T21:32:45.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:32:45.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:45.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:45 smithi152 ceph-mon[156819]: pgmap v1026: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 620 op/s 2023-06-10T21:32:45.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:45 smithi152 ceph-mon[156819]: failed to encode map e286 with expected crc 2023-06-10T21:32:45.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:32:45.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:45 smithi152 ceph-mon[156819]: osdmap e286: 8 total, 8 up, 6 in 2023-06-10T21:32:45.706 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:32:45.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:32:46.189 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 79997 79981 60.1733 36 0.075928 0.0165973 2023-06-10T21:32:46.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:46.702 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.703 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:32:46.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: osdmap e287: 8 total, 8 up, 6 in 2023-06-10T21:32:46.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.704 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.705 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:46 smithi152 ceph-mon[156819]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.747 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: osdmap e287: 8 total, 8 up, 6 in 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[186706]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.750 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1830321903' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: osdmap e287: 8 total, 8 up, 6 in 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.751 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:46 smithi064 ceph-mon[180297]: failed to encode map e287 with expected crc 2023-06-10T21:32:46.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:46.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:46.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:46.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:46.912 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:46.953 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:46.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:47.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:47.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:47.143 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:47.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:47.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:47.189 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 80569 80553 59.8828 35.75 0.0278173 0.0166937 2023-06-10T21:32:47.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:47.392 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:47.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:32:47.617 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:47 smithi064 ceph-mon[180297]: pgmap v1029: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 623 op/s 2023-06-10T21:32:47.618 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:47 smithi064 ceph-mon[186706]: pgmap v1029: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 623 op/s 2023-06-10T21:32:47.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:47.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:47.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:47.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:47 smithi152 ceph-mon[156819]: pgmap v1029: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 39 MiB/s wr, 623 op/s 2023-06-10T21:32:47.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:32:47.874 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:48.029 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:32:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:48.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:32:48.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:48.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:48.189 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 81137 81121 59.5961 35.5 0.00339621 0.0167488 2023-06-10T21:32:48.615 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:48.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:48.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:48.727 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:48.728 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:32:48.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:48.822 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:48.839 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:48.979 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:48.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:49.021 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:49.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:49.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:49.187 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 81779 81763 59.3698 40.125 0.0186972 0.0168364 2023-06-10T21:32:49.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:32:49.496 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:49.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:49.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:49.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:49.718 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:49 smithi064 ceph-mon[180297]: pgmap v1030: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 34 MiB/s wr, 551 op/s 2023-06-10T21:32:49.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:49 smithi064 ceph-mon[186706]: pgmap v1030: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 34 MiB/s wr, 551 op/s 2023-06-10T21:32:49.786 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:49.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:49 smithi152 ceph-mon[156819]: pgmap v1030: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 34 MiB/s wr, 551 op/s 2023-06-10T21:32:49.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:32:50.027 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:32:50.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:50.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:32:50.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:32:50.140 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:32:50.144 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:50.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:32:50.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:50.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 82371 82355 59.1097 37 0.012804 0.0169125 2023-06-10T21:32:50.212 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:32:50.221 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:50.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:50.246 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:32:50.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.265 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.266 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.266 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.266 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.266 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:32:50.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:50.358 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:50.404 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:50.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:50.511 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:50.538 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:50.586 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1658690967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:50.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1658690967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:50.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:50.718 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1658690967' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:32:50.720 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:50.765 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:50.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:50.900 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:50.951 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:50.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:51.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:51.158 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:51.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 83004 82988 58.8877 39.5625 0.0189953 0.0169742 2023-06-10T21:32:51.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:51 smithi064 ceph-mon[180297]: pgmap v1031: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 36 MiB/s wr, 576 op/s 2023-06-10T21:32:51.915 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:51 smithi064 ceph-mon[186706]: pgmap v1031: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 36 MiB/s wr, 576 op/s 2023-06-10T21:32:51.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:51 smithi152 ceph-mon[156819]: pgmap v1031: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 36 MiB/s wr, 576 op/s 2023-06-10T21:32:52.193 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 83662 83646 58.6882 41.125 0.00404065 0.0170313 2023-06-10T21:32:52.526 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:52.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:52.635 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:52.662 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:52.707 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:52.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:52.814 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:52.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:32:52.841 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:52.887 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:32:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:52.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:32:52.912 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:32:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:32:52] "GET /metrics HTTP/1.1" 200 224000 "" "Prometheus/2.33.4" 2023-06-10T21:32:52.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:53.094 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:53.193 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0118 2023-06-10T21:32:53.193 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 84270 2023-06-10T21:32:53.194 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:32:53.194 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:32:53.194 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 58.5132 2023-06-10T21:32:53.195 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 26.7394 2023-06-10T21:32:53.195 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 151.625 2023-06-10T21:32:53.195 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 30.375 2023-06-10T21:32:53.195 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 936 2023-06-10T21:32:53.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 427.831 2023-06-10T21:32:53.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 2426 2023-06-10T21:32:53.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 486 2023-06-10T21:32:53.197 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0170858 2023-06-10T21:32:53.197 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0208847 2023-06-10T21:32:53.197 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 0.376337 2023-06-10T21:32:53.198 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00116152 2023-06-10T21:32:53.198 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:32:53.403 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:53.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:53.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:53.422 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:32:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:32:53.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:84377 11.3 11:c231d7a5:::benchmark_data_smithi064_385725_object104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-06-10T21:32:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:53.385714+0000", 2023-06-10T21:32:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0065660340000000001, 2023-06-10T21:32:53.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006600631, 2023-06-10T21:32:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:32:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:32:53.424 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:32:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:32:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 84377 2023-06-10T21:32:53.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:53.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385714+0000", 2023-06-10T21:32:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385714+0000", 2023-06-10T21:32:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385713+0000", 2023-06-10T21:32:53.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385720+0000", 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.55e-06 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385733+0000", 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3443e-05 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:53.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385742+0000", 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2410000000000001e-06 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385795+0000", 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2908999999999998e-05 2023-06-10T21:32:53.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385947+0000", 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015196200000000001 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:32:53.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.386079+0000", 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013187799999999999 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.392175+0000", 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060959170000000002 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.434 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:84380 11.3 11:cc7a6cc4:::benchmark_data_smithi064_385725_object107:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:53.385757+0000", 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0065232850000000002, 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066740089999999998, 2023-06-10T21:32:53.435 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 84380 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.436 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385757+0000", 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385757+0000", 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385757+0000", 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:32:53.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385760+0000", 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0489999999999999e-06 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:53.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385766+0000", 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2349999999999996e-06 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.385772+0000", 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2779999999999999e-06 2023-06-10T21:32:53.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388771+0000", 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029992040000000001 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388906+0000", 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013450300000000001 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389544+0000", 2023-06-10T21:32:53.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00063785000000000003 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:84385 11.c 11:39aefc80:::benchmark_data_smithi064_385725_object112:head [delete] snapc 0=[] ondisk+write+known_if_redirected e287)", 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:53.388907+0000", 2023-06-10T21:32:53.443 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0033738729999999999, 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036086949999999999, 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:32:53.444 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 84385 2023-06-10T21:32:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:53.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388907+0000", 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388907+0000", 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388906+0000", 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:53.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388912+0000", 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2020000000000001e-06 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388922+0000", 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0317e-05 2023-06-10T21:32:53.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388930+0000", 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2500000000000006e-06 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:53.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.388984+0000", 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3743999999999999e-05 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389098+0000", 2023-06-10T21:32:53.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011362399999999999 2023-06-10T21:32:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,4", 2023-06-10T21:32:53.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389205+0000", 2023-06-10T21:32:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107699 2023-06-10T21:32:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:32:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.392330+0000", 2023-06-10T21:32:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031245790000000002 2023-06-10T21:32:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.453 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:84383 11.f e287/269 11:f029b7fb:::benchmark_data_smithi064_385725_object110:head v 287'5206, mlcod=287'5204)", 2023-06-10T21:32:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:53.389090+0000", 2023-06-10T21:32:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00319083, 2023-06-10T21:32:53.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035145350000000001, 2023-06-10T21:32:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:53.456 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:53.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389090+0000", 2023-06-10T21:32:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389090+0000", 2023-06-10T21:32:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:53.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389089+0000", 2023-06-10T21:32:53.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:53.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:53.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389097+0000", 2023-06-10T21:32:53.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1869999999999997e-06 2023-06-10T21:32:53.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:53.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389107+0000", 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8830000000000001e-06 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389114+0000", 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3629999999999996e-06 2023-06-10T21:32:53.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389329+0000", 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021435399999999999 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.389351+0000", 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2016000000000001e-05 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.466 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:84388 11.f e287/269 11:f6c25664:::benchmark_data_smithi064_385725_object115:head v 287'5207, mlcod=287'5204)", 2023-06-10T21:32:53.466 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:53.391911+0000", 2023-06-10T21:32:53.466 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.000369436, 2023-06-10T21:32:53.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00074382799999999998, 2023-06-10T21:32:53.467 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:32:53.467 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:32:53.467 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:32:53.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:32:53.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391911+0000", 2023-06-10T21:32:53.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:32:53.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391911+0000", 2023-06-10T21:32:53.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:32:53.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:32:53.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391910+0000", 2023-06-10T21:32:53.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:32:53.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:32:53.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391918+0000", 2023-06-10T21:32:53.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6510000000000001e-06 2023-06-10T21:32:53.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:32:53.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391929+0000", 2023-06-10T21:32:53.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1331e-05 2023-06-10T21:32:53.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:32:53.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.391936+0000", 2023-06-10T21:32:53.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2570000000000001e-06 2023-06-10T21:32:53.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:32:53.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.392174+0000", 2023-06-10T21:32:53.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000238161 2023-06-10T21:32:53.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:32:53.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:32:53.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:32:53.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:53.392193+0000", 2023-06-10T21:32:53.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8944999999999999e-05 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 5 2023-06-10T21:32:53.478 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:32:53.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:32:53.543 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:53.589 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:32:53.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:53.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:32:53.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:53 smithi064 ceph-mon[180297]: pgmap v1032: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 698 op/s 2023-06-10T21:32:53.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:53 smithi064 ceph-mon[186706]: pgmap v1032: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 698 op/s 2023-06-10T21:32:53.700 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:32:53.725 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:53 smithi152 ceph-mon[156819]: pgmap v1032: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 698 op/s 2023-06-10T21:32:53.727 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:53.770 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:32:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:53.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:32:53.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:32:53.953 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:54.527 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:32:54.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:32:54.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1164984447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:54.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:54.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1164984447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:54.822 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:54.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1164984447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:32:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[180297]: pgmap v1033: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s 2023-06-10T21:32:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:32:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[180297]: osdmap e288: 8 total, 8 up, 6 in 2023-06-10T21:32:55.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[180297]: failed to encode map e288 with expected crc 2023-06-10T21:32:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[186706]: pgmap v1033: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s 2023-06-10T21:32:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:32:55.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[186706]: osdmap e288: 8 total, 8 up, 6 in 2023-06-10T21:32:55.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:55 smithi064 ceph-mon[186706]: failed to encode map e288 with expected crc 2023-06-10T21:32:55.746 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:32:55.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:55.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:32:55.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:55.853 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:32:55.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:55.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 0 1 2023-06-10T21:32:55.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:55 smithi152 ceph-mon[156819]: pgmap v1033: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 35 MiB/s wr, 553 op/s 2023-06-10T21:32:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:55 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:32:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:55 smithi152 ceph-mon[156819]: osdmap e288: 8 total, 8 up, 6 in 2023-06-10T21:32:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:55 smithi152 ceph-mon[156819]: failed to encode map e288 with expected crc 2023-06-10T21:32:56.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:56.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:56.132 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:56.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:32:56.199 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:56.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:32:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:56.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:32:56.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:56.385 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:56.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[186706]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[186706]: osdmap e289: 8 total, 8 up, 6 in 2023-06-10T21:32:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[186706]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[186706]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[186706]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[180297]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[180297]: osdmap e289: 8 total, 8 up, 6 in 2023-06-10T21:32:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[180297]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[180297]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:56 smithi064 ceph-mon[180297]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.750 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:56.812 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:32:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:56.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:32:56.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:56.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:56 smithi152 ceph-mon[156819]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:56 smithi152 ceph-mon[156819]: osdmap e289: 8 total, 8 up, 6 in 2023-06-10T21:32:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:56 smithi152 ceph-mon[156819]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:56 smithi152 ceph-mon[156819]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:56 smithi152 ceph-mon[156819]: failed to encode map e289 with expected crc 2023-06-10T21:32:56.950 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:57.061 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:32:57.202 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:57.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:57.315 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:32:57.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:57.569 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:32:57.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: pgmap v1036: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 127 B/s rd, 34 MiB/s wr, 591 op/s 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: osdmap e290: 8 total, 8 up, 6 in 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4152544363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: pgmap v1036: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 127 B/s rd, 34 MiB/s wr, 591 op/s 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: osdmap e290: 8 total, 8 up, 6 in 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4152544363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:57.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:57 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:57.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: pgmap v1036: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 127 B/s rd, 34 MiB/s wr, 591 op/s 2023-06-10T21:32:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: osdmap e290: 8 total, 8 up, 6 in 2023-06-10T21:32:57.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: failed to encode map e290 with expected crc 2023-06-10T21:32:57.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4152544363' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:57.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:57 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:32:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: failed to encode map e290 with expected crc 2023-06-10T21:32:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: osdmap e291: 8 total, 8 up, 6 in 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[180297]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: failed to encode map e290 with expected crc 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: osdmap e291: 8 total, 8 up, 6 in 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:58 smithi064 ceph-mon[186706]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: failed to encode map e290 with expected crc 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: osdmap e291: 8 total, 8 up, 6 in 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: failed to encode map e291 with expected crc 2023-06-10T21:32:58.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:58 smithi152 ceph-mon[156819]: failed to encode map e291 with expected crc 2023-06-10T21:32:59.463 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:32:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:32:59.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:32:59.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:32:59.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:32:59.879 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[186706]: pgmap v1039: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 255 B/s rd, 22 MiB/s wr, 536 op/s 2023-06-10T21:32:59.880 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[186706]: osdmap e292: 8 total, 8 up, 6 in 2023-06-10T21:32:59.880 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[186706]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.880 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[186706]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.880 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[186706]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.880 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[180297]: pgmap v1039: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 255 B/s rd, 22 MiB/s wr, 536 op/s 2023-06-10T21:32:59.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[180297]: osdmap e292: 8 total, 8 up, 6 in 2023-06-10T21:32:59.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[180297]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[180297]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.881 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:32:59 smithi064 ceph-mon[180297]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:59 smithi152 ceph-mon[156819]: pgmap v1039: 89 pgs: 8 peering, 81 active+clean; 5.1 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 255 B/s rd, 22 MiB/s wr, 536 op/s 2023-06-10T21:32:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:59 smithi152 ceph-mon[156819]: osdmap e292: 8 total, 8 up, 6 in 2023-06-10T21:32:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:59 smithi152 ceph-mon[156819]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:59 smithi152 ceph-mon[156819]: failed to encode map e292 with expected crc 2023-06-10T21:32:59.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:32:59 smithi152 ceph-mon[156819]: failed to encode map e292 with expected crc 2023-06-10T21:33:00.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:00.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:00.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:00.510 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:00.559 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:00.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:00.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:00.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:00.864 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:00 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 5516/247803 objects degraded (2.226%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:00.865 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:00 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 5516/247803 objects degraded (2.226%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:00.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:00 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 5516/247803 objects degraded (2.226%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:00.951 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:00.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:00.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:01.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:01.098 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:01.148 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:01.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:01.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:01.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:01.838 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:01.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:01 smithi064 ceph-mon[180297]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.8 MiB/s wr, 433 op/s; 5516/247803 objects degraded (2.226%); 5020/247803 objects misplaced (2.026%); 1003 KiB/s, 16 objects/s recovering 2023-06-10T21:33:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:01 smithi064 ceph-mon[186706]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.8 MiB/s wr, 433 op/s; 5516/247803 objects degraded (2.226%); 5020/247803 objects misplaced (2.026%); 1003 KiB/s, 16 objects/s recovering 2023-06-10T21:33:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:01 smithi152 ceph-mon[156819]: pgmap v1041: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 7.8 MiB/s wr, 433 op/s; 5516/247803 objects degraded (2.226%); 5020/247803 objects misplaced (2.026%); 1003 KiB/s, 16 objects/s recovering 2023-06-10T21:33:01.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:02.023 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:02.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:02.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:02.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:33:02.481 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:33:02.481 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:33:02.481 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:33:02.482 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:33:02.482 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:33:02.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:02.553 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:33:02.581 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:02.629 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:33:02.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4010925308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:02.689 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:33:02.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:33:02.782 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:33:02.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:33:02.867 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:33:02.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:33:02.915 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.915 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.916 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.916 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.916 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.916 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.916 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.917 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:33:02.944 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:02] "GET /metrics HTTP/1.1" 200 224000 "" "Prometheus/2.33.4" 2023-06-10T21:33:02.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4010925308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:02.946 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4010925308' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:03.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:03.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:03.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:03.516 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[186706]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 5.6 MiB/s wr, 468 op/s; 5516/244935 objects degraded (2.252%); 5020/244935 objects misplaced (2.050%); 715 KiB/s, 11 objects/s recovering 2023-06-10T21:33:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:03.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:03.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[180297]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 5.6 MiB/s wr, 468 op/s; 5516/244935 objects degraded (2.252%); 5020/244935 objects misplaced (2.050%); 715 KiB/s, 11 objects/s recovering 2023-06-10T21:33:03.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:03.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:03.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:03 smithi152 ceph-mon[156819]: pgmap v1042: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 5.6 MiB/s wr, 468 op/s; 5516/244935 objects degraded (2.252%); 5020/244935 objects misplaced (2.050%); 715 KiB/s, 11 objects/s recovering 2023-06-10T21:33:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:03.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:04.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:33:04.513 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:04.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:04.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:04.653 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:04.698 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:04.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:04.806 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:04.833 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:04.873 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:04.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:04.981 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:05.011 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:05.060 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:05.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:05.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:05.167 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:05.247 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:05.413 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:05.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:05.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:05.492 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:05.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:05.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:05.561 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:05.602 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:05.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:05.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:05.707 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:05.711 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-06-10T21:33:05.734 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:05.780 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:05.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:05.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:05.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:05 smithi152 ceph-mon[156819]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 829 KiB/s wr, 355 op/s; 5516/243207 objects degraded (2.268%); 5020/243207 objects misplaced (2.064%); 591 KiB/s, 9 objects/s recovering 2023-06-10T21:33:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2616349378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:05 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[180297]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 829 KiB/s wr, 355 op/s; 5516/243207 objects degraded (2.268%); 5020/243207 objects misplaced (2.064%); 591 KiB/s, 9 objects/s recovering 2023-06-10T21:33:05.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2616349378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.961 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[186706]: pgmap v1043: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 829 KiB/s wr, 355 op/s; 5516/243207 objects degraded (2.268%); 5020/243207 objects misplaced (2.064%); 591 KiB/s, 9 objects/s recovering 2023-06-10T21:33:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2616349378' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.961 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:05 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:33:05.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:07.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:33:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: osdmap e293: 8 total, 8 up, 6 in 2023-06-10T21:33:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5516/243207 objects degraded (2.268%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:07.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:33:07.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: osdmap e293: 8 total, 8 up, 6 in 2023-06-10T21:33:07.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5516/243207 objects degraded (2.268%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:07.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:33:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: osdmap e293: 8 total, 8 up, 6 in 2023-06-10T21:33:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: failed to encode map e293 with expected crc 2023-06-10T21:33:07.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5516/243207 objects degraded (2.268%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:07.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:33:08.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:08.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:08.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:08.138 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:07 smithi064 ceph-mon[180297]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 752 KiB/s wr, 488 op/s; 5572/239211 objects degraded (2.329%); 4837/239211 objects misplaced (2.022%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:08.139 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:07 smithi064 ceph-mon[186706]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 752 KiB/s wr, 488 op/s; 5572/239211 objects degraded (2.329%); 4837/239211 objects misplaced (2.022%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:08.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:07 smithi152 ceph-mon[156819]: pgmap v1045: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 752 KiB/s wr, 488 op/s; 5572/239211 objects degraded (2.329%); 4837/239211 objects misplaced (2.022%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:08.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:08.267 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:08.314 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:08.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:08.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:08.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:08.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:33:08.560 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:08.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:08.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:08.960 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:09.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:09.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:09.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:09.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:09.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:09.540 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:09.546 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:09.599 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:09.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:09.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:09.706 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:09.734 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:09.786 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:09.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:09.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:09.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:09.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:09 smithi152 ceph-mon[156819]: pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 222 B/s wr, 498 op/s; 5572/236046 objects degraded (2.361%); 4837/236046 objects misplaced (2.049%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:33:09.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:09.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:09 smithi064 ceph-mon[180297]: pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 222 B/s wr, 498 op/s; 5572/236046 objects degraded (2.361%); 4837/236046 objects misplaced (2.049%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:33:09.992 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:09.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:09 smithi064 ceph-mon[186706]: pgmap v1046: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 222 B/s wr, 498 op/s; 5572/236046 objects degraded (2.361%); 4837/236046 objects misplaced (2.049%); 1.3 MiB/s, 21 objects/s recovering 2023-06-10T21:33:09.993 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:10.015 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:10.104 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:10.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:10.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:10.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:10.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:10.490 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:10.672 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:10.673 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:11.648 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:11.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:11.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:11.826 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:11.856 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:33:11.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:11.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:11.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:12.004 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[180297]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 5619/231084 objects degraded (2.432%); 4667/231084 objects misplaced (2.020%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:12.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5619/231084 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:12.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3451424179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.005 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[186706]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 5619/231084 objects degraded (2.432%); 4667/231084 objects misplaced (2.020%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:12.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5619/231084 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:12.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3451424179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.006 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.095 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:12.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:11 smithi152 ceph-mon[156819]: pgmap v1047: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 556 op/s; 5619/231084 objects degraded (2.432%); 4667/231084 objects misplaced (2.020%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:12.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5619/231084 objects degraded (2.432%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:12.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3451424179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:33:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: osdmap e294: 8 total, 8 up, 6 in 2023-06-10T21:33:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:33:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: osdmap e294: 8 total, 8 up, 6 in 2023-06-10T21:33:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:12.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:12 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.117 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:13.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:13.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:12] "GET /metrics HTTP/1.1" 200 224072 "" "Prometheus/2.33.4" 2023-06-10T21:33:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:33:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: osdmap e294: 8 total, 8 up, 6 in 2023-06-10T21:33:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:12 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:13.254 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:13.302 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:13.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:13.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:13.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:33:13.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:13.762 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:13 smithi064 ceph-mon[180297]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 5619/228024 objects degraded (2.464%); 4667/228024 objects misplaced (2.047%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:13.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:13 smithi064 ceph-mon[180297]: failed to encode map e294 with expected crc 2023-06-10T21:33:13.892 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:13.893 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:13.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:13 smithi152 ceph-mon[156819]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 5619/228024 objects degraded (2.464%); 4667/228024 objects misplaced (2.047%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:13 smithi152 ceph-mon[156819]: failed to encode map e294 with expected crc 2023-06-10T21:33:14.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:14.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:13 smithi064 ceph-mon[186706]: pgmap v1048: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 562 op/s; 5619/228024 objects degraded (2.464%); 4667/228024 objects misplaced (2.047%); 1.6 MiB/s, 24 objects/s recovering 2023-06-10T21:33:14.109 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:13 smithi064 ceph-mon[186706]: failed to encode map e294 with expected crc 2023-06-10T21:33:14.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:14.343 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:14.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:14.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:14.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:14.686 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:14.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:14.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:14.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:14.888 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:14 smithi064 ceph-mon[180297]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 528 op/s; 5619/226353 objects degraded (2.482%); 4667/226353 objects misplaced (2.062%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:33:14.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:14 smithi064 ceph-mon[186706]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 528 op/s; 5619/226353 objects degraded (2.482%); 4667/226353 objects misplaced (2.062%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:33:14.901 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:15.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:14 smithi152 ceph-mon[156819]: pgmap v1050: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.6 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 528 op/s; 5619/226353 objects degraded (2.482%); 4667/226353 objects misplaced (2.062%); 971 KiB/s, 15 objects/s recovering 2023-06-10T21:33:15.345 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:15.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:15.443 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:33:15.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:15.481 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:33:15.524 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:33:15.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:15.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:33:15.573 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:33:15.592 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:33:15.611 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:33:15.614 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:15.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:15.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:15.677 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:33:15.708 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.709 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.710 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:33:15.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:15.843 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:33:15.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3845876430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:15.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3845876430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:16.072 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3845876430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:16.149 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:16.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:16.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:16.347 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:16.574 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:16.574 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:33:16.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:96458 11.d e294/269 11:b9fb5247:::benchmark_data_smithi064_385725_object12185:head v 294'5989, mlcod=294'5985)", 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:16.569278+0000", 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0026940850000000001, 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027307410000000001, 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:33:16.575 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569278+0000", 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:16.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569278+0000", 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569277+0000", 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4919999999999996e-06 2023-06-10T21:33:16.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569284+0000", 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5366000000000002e-05 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569299+0000", 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3969999999999999e-06 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:16.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569306+0000", 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.028e-05 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569356+0000", 2023-06-10T21:33:16.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9595000000000001e-05 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.569376+0000", 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8146000000000005e-05 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:16.581 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:96463 11.d e294/269 11:b8cfd015:::benchmark_data_smithi064_385725_object12190:head v 294'5990, mlcod=294'5987)", 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:16.571810+0000", 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.000161652, 2023-06-10T21:33:16.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027092700000000002, 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571810+0000", 2023-06-10T21:33:16.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571810+0000", 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571809+0000", 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4179999999999996e-06 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571815+0000", 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8419999999999994e-06 2023-06-10T21:33:16.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571824+0000", 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1980000000000003e-06 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:16.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571828+0000", 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9851000000000002e-05 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571868+0000", 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0307000000000001e-05 2023-06-10T21:33:16.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:16.571888+0000", 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8087e-05 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:16.588 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:16.589 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:33:16.589 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 2 2023-06-10T21:33:16.589 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:16.881 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:33:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:16 smithi152 ceph-mon[156819]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 5657/222255 objects degraded (2.545%); 4487/222255 objects misplaced (2.019%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5619/226353 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3038159132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:16.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:17.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[180297]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 5657/222255 objects degraded (2.545%); 4487/222255 objects misplaced (2.019%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:17.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5619/226353 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:17.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3038159132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:17.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[186706]: pgmap v1051: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 564 op/s; 5657/222255 objects degraded (2.545%); 4487/222255 objects misplaced (2.019%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:17.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5619/226353 objects degraded (2.482%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3038159132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:17.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:33:17.322 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:17.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:17.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:17.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:33:17.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:17.503 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:17.789 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:17.940 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:17.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:17.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:18.057 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:18.125 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:18.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:33:18.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[180297]: osdmap e295: 8 total, 8 up, 6 in 2023-06-10T21:33:18.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[180297]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.162 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[180297]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:33:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[186706]: osdmap e295: 8 total, 8 up, 6 in 2023-06-10T21:33:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[186706]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:17 smithi064 ceph-mon[186706]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:33:18.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:17 smithi152 ceph-mon[156819]: osdmap e295: 8 total, 8 up, 6 in 2023-06-10T21:33:18.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:17 smithi152 ceph-mon[156819]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:17 smithi152 ceph-mon[156819]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:18.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:18.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:18.491 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:33:18.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:33:18.548 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:18.623 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:18.688 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:18.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:18.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:18.812 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:18.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[180297]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[180297]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[180297]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 5657/219120 objects degraded (2.582%); 4487/219120 objects misplaced (2.048%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:33:18.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:18.885 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:18.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[186706]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[186706]: failed to encode map e295 with expected crc 2023-06-10T21:33:18.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[186706]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 5657/219120 objects degraded (2.582%); 4487/219120 objects misplaced (2.048%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:33:18.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:18.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:18.902 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:19.083 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:19.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:18 smithi152 ceph-mon[156819]: failed to encode map e295 with expected crc 2023-06-10T21:33:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:18 smithi152 ceph-mon[156819]: failed to encode map e295 with expected crc 2023-06-10T21:33:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:18 smithi152 ceph-mon[156819]: pgmap v1053: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 5657/219120 objects degraded (2.582%); 4487/219120 objects misplaced (2.048%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:33:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:19.201 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:19.237 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:19.293 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:19.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:19.400 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:19.500 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:19.500 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:33:19.500 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:33:19.500 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:33:19.501 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:19.519 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:19.709 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:19.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:19.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:19.847 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:19.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:19.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:19.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:20.088 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:20.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:20.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:20.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:20.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:20.281 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:20.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:33:20.326 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:20.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:20.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:33:20.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:20.568 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:20.707 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:20.836 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:20.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:20.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:20.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:21.087 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:20 smithi064 ceph-mon[186706]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 597 op/s; 5704/213669 objects degraded (2.670%); 4319/213669 objects misplaced (2.021%); 2.1 MiB/s, 32 objects/s recovering 2023-06-10T21:33:21.087 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:20 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5704/213669 objects degraded (2.670%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:21.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:20 smithi064 ceph-mon[180297]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 597 op/s; 5704/213669 objects degraded (2.670%); 4319/213669 objects misplaced (2.021%); 2.1 MiB/s, 32 objects/s recovering 2023-06-10T21:33:21.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:20 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5704/213669 objects degraded (2.670%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:21.113 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:21.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:20 smithi152 ceph-mon[156819]: pgmap v1054: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 597 op/s; 5704/213669 objects degraded (2.670%); 4319/213669 objects misplaced (2.021%); 2.1 MiB/s, 32 objects/s recovering 2023-06-10T21:33:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:20 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5704/213669 objects degraded (2.670%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:21.425 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:21.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:21.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:21.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:21.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:21.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:21.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:21.943 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:22.017 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:22.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:22.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:22.218 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:22.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:22.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:22.327 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:22.327 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:33:22.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:33:22.327 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:33:22.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166807 9.4 9:2480dc05:::benchmark_data_smithi064_247471_object57819:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 577.00019615600002, 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040999767, 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:22.328 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166807 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322455+0000", 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322453+0000", 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9229999999999996e-06 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322460+0000", 2023-06-10T21:33:22.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9490000000000008e-06 2023-06-10T21:33:22.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322470+0000", 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2830000000000004e-06 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322478+0000", 2023-06-10T21:33:22.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003841133 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326319+0000", 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9380000000000004e-06 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326329+0000", 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0051517590000001 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331480+0000", 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1434000000000006e-05 2023-06-10T21:33:22.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331572+0000", 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031794995 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363367+0000", 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8282999999999995e-05 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363455+0000", 2023-06-10T21:33:22.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040999767 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.338 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166814 9.4 9:21a2952b:::benchmark_data_smithi064_247471_object57826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 577.00015983699996, 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0414135449999999, 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.339 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166814 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322492+0000", 2023-06-10T21:33:22.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322491+0000", 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.658e-06 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322495+0000", 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9670000000000004e-06 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322502+0000", 2023-06-10T21:33:22.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.096e-06 2023-06-10T21:33:22.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322507+0000", 2023-06-10T21:33:22.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038619230000000002 2023-06-10T21:33:22.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326369+0000", 2023-06-10T21:33:22.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7579999999999998e-06 2023-06-10T21:33:22.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326377+0000", 2023-06-10T21:33:22.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0052419239999999 2023-06-10T21:33:22.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331619+0000", 2023-06-10T21:33:22.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6457000000000002e-05 2023-06-10T21:33:22.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331716+0000", 2023-06-10T21:33:22.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3530999999999996e-05 2023-06-10T21:33:22.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.331809+0000", 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.023985312000000002 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355795+0000", 2023-06-10T21:33:22.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079406909999999997 2023-06-10T21:33:22.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363735+0000", 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1652999999999997e-05 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363767+0000", 2023-06-10T21:33:22.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5206000000000001e-05 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363812+0000", 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2964e-05 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363905+0000", 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0414135449999999 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166820 9.4 9:2fd87fc4:::benchmark_data_smithi064_247471_object57832:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.997969807, 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.039453033, 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.359 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166820 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324682+0000", 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324680+0000", 2023-06-10T21:33:22.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4710000000000004e-06 2023-06-10T21:33:22.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324687+0000", 2023-06-10T21:33:22.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6809999999999993e-06 2023-06-10T21:33:22.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324696+0000", 2023-06-10T21:33:22.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9989999999999999e-06 2023-06-10T21:33:22.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.324701+0000", 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017058240000000001 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326407+0000", 2023-06-10T21:33:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9110000000000004e-06 2023-06-10T21:33:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.326414+0000", 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.008438868 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.334853+0000", 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9625999999999998e-05 2023-06-10T21:33:22.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.334953+0000", 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8136999999999999e-05 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.335051+0000", 2023-06-10T21:33:22.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020754058999999998 2023-06-10T21:33:22.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355805+0000", 2023-06-10T21:33:22.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081369440000000001 2023-06-10T21:33:22.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363942+0000", 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4404000000000002e-05 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364006+0000", 2023-06-10T21:33:22.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1557999999999998e-05 2023-06-10T21:33:22.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364048+0000", 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6891000000000005e-05 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364135+0000", 2023-06-10T21:33:22.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.039453033 2023-06-10T21:33:22.375 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.375 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.375 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.376 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166852 9.4 9:2c2b0a22:::benchmark_data_smithi064_247471_object57864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.376 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.94036640399997, 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98200426799999996, 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.377 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166852 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382285+0000", 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382283+0000", 2023-06-10T21:33:22.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5140000000000007e-06 2023-06-10T21:33:22.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382293+0000", 2023-06-10T21:33:22.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5099999999999999e-05 2023-06-10T21:33:22.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382308+0000", 2023-06-10T21:33:22.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4240000000000004e-06 2023-06-10T21:33:22.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382315+0000", 2023-06-10T21:33:22.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1654000000000001e-05 2023-06-10T21:33:22.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382397+0000", 2023-06-10T21:33:22.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1015999999999999e-05 2023-06-10T21:33:22.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.382408+0000", 2023-06-10T21:33:22.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95501303800000004 2023-06-10T21:33:22.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337421+0000", 2023-06-10T21:33:22.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0987000000000003e-05 2023-06-10T21:33:22.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337492+0000", 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0113999999999998e-05 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.337582+0000", 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018232515000000001 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355815+0000", 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0081570289999999997 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363972+0000", 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019824 2023-06-10T21:33:22.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364170+0000", 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6148e-05 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364206+0000", 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3392000000000004e-05 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364289+0000", 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98200426799999996 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.394 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166875 9.4 9:2e6a97c5:::benchmark_data_smithi064_247471_object57887:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.78617579299998, 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82800355000000003, 2023-06-10T21:33:22.395 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166875 2023-06-10T21:33:22.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.397 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:33:22.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536476+0000", 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536473+0000", 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4477999999999999e-05 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536488+0000", 2023-06-10T21:33:22.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7987000000000002e-05 2023-06-10T21:33:22.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536516+0000", 2023-06-10T21:33:22.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9879999999999999e-06 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536526+0000", 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4812e-05 2023-06-10T21:33:22.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536610+0000", 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2381999999999999e-05 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.536623+0000", 2023-06-10T21:33:22.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.803186336 2023-06-10T21:33:22.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339809+0000", 2023-06-10T21:33:22.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143893 2023-06-10T21:33:22.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339953+0000", 2023-06-10T21:33:22.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000146976 2023-06-10T21:33:22.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.340100+0000", 2023-06-10T21:33:22.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015724806000000001 2023-06-10T21:33:22.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355825+0000", 2023-06-10T21:33:22.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084980560000000004 2023-06-10T21:33:22.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364323+0000", 2023-06-10T21:33:22.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8036999999999999e-05 2023-06-10T21:33:22.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364351+0000", 2023-06-10T21:33:22.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7922e-05 2023-06-10T21:33:22.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364389+0000", 2023-06-10T21:33:22.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0325e-05 2023-06-10T21:33:22.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364479+0000", 2023-06-10T21:33:22.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82800355000000003 2023-06-10T21:33:22.414 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.415 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.415 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166881 9.4 9:2065f492:::benchmark_data_smithi064_247471_object57893:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.77151220200005, 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81353793299999999, 2023-06-10T21:33:22.416 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166881 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.417 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:33:22.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551139+0000", 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551137+0000", 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4239999999999993e-06 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551146+0000", 2023-06-10T21:33:22.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6155e-05 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551162+0000", 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8860000000000004e-06 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551170+0000", 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0584000000000001e-05 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551231+0000", 2023-06-10T21:33:22.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0097e-05 2023-06-10T21:33:22.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.551241+0000", 2023-06-10T21:33:22.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79565823499999999 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346899+0000", 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010809700000000001 2023-06-10T21:33:22.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.347007+0000", 2023-06-10T21:33:22.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2709999999999999e-05 2023-06-10T21:33:22.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.347090+0000", 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016244689 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363334+0000", 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001185237 2023-06-10T21:33:22.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.428 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364520+0000", 2023-06-10T21:33:22.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0880000000000002e-05 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364550+0000", 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8509e-05 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364589+0000", 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8195000000000005e-05 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364677+0000", 2023-06-10T21:33:22.430 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81353793299999999 2023-06-10T21:33:22.431 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.431 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.431 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.432 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166885 9.4 9:2f426375:::benchmark_data_smithi064_247471_object57897:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.432 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:33:22.432 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.76664820500002, 2023-06-10T21:33:22.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80886061200000003, 2023-06-10T21:33:22.432 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.433 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.433 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.433 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.433 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.434 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166885 2023-06-10T21:33:22.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.434 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556003+0000", 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556001+0000", 2023-06-10T21:33:22.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.028e-05 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556012+0000", 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1024999999999999e-05 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556033+0000", 2023-06-10T21:33:22.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0660000000000004e-06 2023-06-10T21:33:22.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556041+0000", 2023-06-10T21:33:22.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0553e-05 2023-06-10T21:33:22.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556111+0000", 2023-06-10T21:33:22.441 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2969999999999999e-05 2023-06-10T21:33:22.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.441 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.556124+0000", 2023-06-10T21:33:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79310488000000001 2023-06-10T21:33:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.442 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349229+0000", 2023-06-10T21:33:22.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110064 2023-06-10T21:33:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349339+0000", 2023-06-10T21:33:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130178 2023-06-10T21:33:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349469+0000", 2023-06-10T21:33:22.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013871082 2023-06-10T21:33:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363340+0000", 2023-06-10T21:33:22.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001378514 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364719+0000", 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7698000000000001e-05 2023-06-10T21:33:22.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364747+0000", 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5630999999999998e-05 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364782+0000", 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1612000000000004e-05 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364864+0000", 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80886061200000003 2023-06-10T21:33:22.450 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166889 9.4 9:208e17d2:::benchmark_data_smithi064_247471_object57901:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.76164148800001, 2023-06-10T21:33:22.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81055013600000003, 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166889 2023-06-10T21:33:22.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561010+0000", 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561008+0000", 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9509999999999995e-06 2023-06-10T21:33:22.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561017+0000", 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6741000000000001e-05 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561034+0000", 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5610000000000004e-06 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561041+0000", 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8444999999999999e-05 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561109+0000", 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2872e-05 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.561122+0000", 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79092483199999997 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352047+0000", 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111666 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352159+0000", 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130674 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.352289+0000", 2023-06-10T21:33:22.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011159099 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363448+0000", 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014472140000000001 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364896+0000", 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064686250000000004 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371364+0000", 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4372e-05 2023-06-10T21:33:22.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371429+0000", 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131564 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371560+0000", 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81055013600000003 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166904 9.4 9:2d035138:::benchmark_data_smithi064_247471_object57916:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.73462757300001, 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78375932999999998, 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.464 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166904 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588024+0000", 2023-06-10T21:33:22.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588022+0000", 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0492999999999999e-05 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588032+0000", 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0548e-05 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588053+0000", 2023-06-10T21:33:22.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8210000000000002e-06 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588061+0000", 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2191000000000001e-05 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588123+0000", 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1460000000000002e-06 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.588132+0000", 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76775603000000003 2023-06-10T21:33:22.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.355888+0000", 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014207900000000001 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.356030+0000", 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016228099999999999 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.356193+0000", 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072615379999999997 2023-06-10T21:33:22.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.363454+0000", 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014683350000000001 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364922+0000", 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067034099999999999 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371626+0000", 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8415000000000002e-05 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371684+0000", 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8995999999999999e-05 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371783+0000", 2023-06-10T21:33:22.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78375932999999998 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166915 9.4 9:2e5c8b28:::benchmark_data_smithi064_247471_object57927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:22.476 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 576.69124691800005, 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74064215499999997, 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.477 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166915 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631405+0000", 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631403+0000", 2023-06-10T21:33:22.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9249999999999995e-06 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631410+0000", 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4515000000000001e-05 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631425+0000", 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8109999999999999e-06 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631431+0000", 2023-06-10T21:33:22.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8529000000000002e-05 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631479+0000", 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4639999999999995e-06 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.631489+0000", 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.72782496200000002 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359314+0000", 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112879 2023-06-10T21:33:22.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359427+0000", 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013150099999999999 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.359558+0000", 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051280220000000003 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.364686+0000", 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071560779999999997 2023-06-10T21:33:22.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371842+0000", 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4267000000000001e-05 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371896+0000", 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4815999999999999e-05 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.371941+0000", 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010541500000000001 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.372047+0000", 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.74064215499999997 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:209425 9.4 9:2f96b396:::benchmark_data_smithi064_247471_object100437:head [delete] snapc 0=[] ondisk+write+known_if_redirected e182)", 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 501.87022769800001, 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.237427307, 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.489 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 209425 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452424+0000", 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452422+0000", 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1479999999999995e-06 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452431+0000", 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8559e-05 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452449+0000", 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9000000000000006e-06 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452457+0000", 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0895e-05 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452508+0000", 2023-06-10T21:33:22.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105846 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452614+0000", 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121283 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.452735+0000", 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044428790000000003 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.457178+0000", 2023-06-10T21:33:22.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114129 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.457292+0000", 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.232459255 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689752+0000", 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5896999999999997e-05 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689808+0000", 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3510999999999997e-05 2023-06-10T21:33:22.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:00.689851+0000", 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.237427307 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(10.f 227/220 [PushOp(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head, version: 200'380, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head@200'380, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 362.80849993099997, 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001, 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.500 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514151+0000", 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5096999999999998e-05 2023-06-10T21:33:22.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514206+0000", 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5574e-05 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514222+0000", 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0900000000000005e-06 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514230+0000", 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036878129999999999 2023-06-10T21:33:22.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517918+0000", 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0910000000000003e-06 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517922+0000", 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.231370187 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.749292+0000", 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.506 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:96995 10.f e227/220 10:f95a89c6:::benchmark_data_smithi064_312713_object6239:head v 227'5921, mlcod=200'373)", 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 362.80418503800001, 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001, 2023-06-10T21:33:22.507 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1389999999999998e-06 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518473+0000", 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0293e-05 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518483+0000", 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6609999999999997e-06 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518489+0000", 2023-06-10T21:33:22.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23332091099999999 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751810+0000", 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9099999999999999e-05 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751839+0000", 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027593894000000001 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779433+0000", 2023-06-10T21:33:22.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1985999999999999e-05 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779455+0000", 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.514 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:97001 10.f e227/220 10:fd7a027e:::benchmark_data_smithi064_312713_object6245:head v 227'5922, mlcod=200'373)", 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 362.800155539, 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002, 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.515 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522495+0000", 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5910000000000001e-06 2023-06-10T21:33:22.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522502+0000", 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22114999699999999 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743652+0000", 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4776999999999999e-05 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743667+0000", 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013856040999999999 2023-06-10T21:33:22.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757523+0000", 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8875999999999999e-05 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757551+0000", 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021906202999999999 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779458+0000", 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9106e-05 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779477+0000", 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111479 10.6 10:610f9587:::benchmark_data_smithi064_312713_object20723:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 341.01029868099999, 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999, 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.523 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.524 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.524 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111479 2023-06-10T21:33:22.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.524 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312351+0000", 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1720000000000004e-06 2023-06-10T21:33:22.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312358+0000", 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312370+0000", 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4580000000000004e-06 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312378+0000", 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024553309999999998 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314833+0000", 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314842+0000", 2023-06-10T21:33:22.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0343133339999999 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.349155+0000", 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001791864 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350947+0000", 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.531 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111480 10.d 10:bb138473:::benchmark_data_smithi064_312713_object20724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 341.01026537799999, 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999, 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.532 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111480 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:33:22.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.802e-06 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312388+0000", 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9600000000000002e-06 2023-06-10T21:33:22.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312392+0000", 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7259999999999998e-06 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312395+0000", 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7097000000000001e-05 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312432+0000", 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5759999999999996e-06 2023-06-10T21:33:22.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312438+0000", 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.019375897 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331814+0000", 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6889000000000003e-05 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331881+0000", 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.539 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111483 10.d 10:b3a132fc:::benchmark_data_smithi064_312713_object20727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 341.00133893999998, 2023-06-10T21:33:22.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817, 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111483 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.541 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321311+0000", 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1680000000000003e-06 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321316+0000", 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2384999999999999e-05 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321329+0000", 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9739999999999999e-06 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321335+0000", 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069082699999999998 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322026+0000", 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2101e-05 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322038+0000", 2023-06-10T21:33:22.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.009867013 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331905+0000", 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2904000000000005e-05 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331978+0000", 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039655499999999999 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 6", 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.332374+0000", 2023-06-10T21:33:22.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011245184 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343619+0000", 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7795999999999998e-05 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343707+0000", 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0745000000000003e-05 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343758+0000", 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9522e-05 2023-06-10T21:33:22.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343857+0000", 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111488 10.4 10.41ecfd74 (undecoded) ondisk+write+known_if_redirected e232)", 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 340.99091847800003, 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999, 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.552 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111488 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:22.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331731+0000", 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7428e-05 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4331999999999999e-05 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331763+0000", 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5490000000000003e-06 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331769+0000", 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9897e-05 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331819+0000", 2023-06-10T21:33:22.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0000000000000002e-06 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331825+0000", 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0023298279999999 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334155+0000", 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0281e-05 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334185+0000", 2023-06-10T21:33:22.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111491 10.6 10:69475ff3:::benchmark_data_smithi064_312713_object20735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:33:22.560 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 340.972706227, 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693, 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111491 2023-06-10T21:33:22.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349943+0000", 2023-06-10T21:33:22.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5360000000000004e-06 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349951+0000", 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4100999999999999e-05 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349965+0000", 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2029999999999997e-06 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349971+0000", 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4274000000000002e-05 2023-06-10T21:33:22.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350035+0000", 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9600000000000006e-06 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350044+0000", 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000890976 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350935+0000", 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051541900000000003 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351451+0000", 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029125 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:33:22.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351742+0000", 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098297039999999999 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.361571+0000", 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025176170000000002 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364089+0000", 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5660000000000001e-05 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364125+0000", 2023-06-10T21:33:22.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4188000000000001e-05 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364209+0000", 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.571 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111494 10.6 10:60c1325c:::benchmark_data_smithi064_312713_object20738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 340.956478635, 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013, 2023-06-10T21:33:22.572 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111494 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:22.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:33:22.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366171+0000", 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9200000000000001e-06 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366177+0000", 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366190+0000", 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9599999999999997e-06 2023-06-10T21:33:22.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366196+0000", 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6909000000000001e-05 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366243+0000", 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4109999999999996e-06 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366251+0000", 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.989107077 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355358+0000", 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102029 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355460+0000", 2023-06-10T21:33:22.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2402000000000001e-05 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355553+0000", 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086816740000000003 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364234+0000", 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003252605 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367487+0000", 2023-06-10T21:33:22.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8502999999999999e-05 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367546+0000", 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7332999999999998e-05 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:22.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367643+0000", 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:22.583 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:22.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:22.617 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:22.668 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:22.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:22.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:22.799 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:22.849 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:22.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:22.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:22.958 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:22.959 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:33:22.982 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:23.131 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:23.133 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:22 smithi152 ceph-mon[156819]: pgmap v1055: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5704/210519 objects degraded (2.709%); 4319/210519 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:23.133 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1728800721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.133 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:23.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:22] "GET /metrics HTTP/1.1" 200 224073 "" "Prometheus/2.33.4" 2023-06-10T21:33:23.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[186706]: pgmap v1055: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5704/210519 objects degraded (2.709%); 4319/210519 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:23.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1728800721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[180297]: pgmap v1055: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5704/210519 objects degraded (2.709%); 4319/210519 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:23.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1728800721' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:23.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:33:23.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:23.302 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:23.599 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:24.190 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:24.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:23 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:24.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:23 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:24.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:23 smithi152 ceph-mon[156819]: osdmap e296: 8 total, 8 up, 6 in 2023-06-10T21:33:24.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:23 smithi152 ceph-mon[156819]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:23 smithi152 ceph-mon[156819]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:24.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:24.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[186706]: osdmap e296: 8 total, 8 up, 6 in 2023-06-10T21:33:24.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[186706]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.196 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[186706]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:24.197 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:24.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[180297]: osdmap e296: 8 total, 8 up, 6 in 2023-06-10T21:33:24.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[180297]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.198 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:23 smithi064 ceph-mon[180297]: failed to encode map e296 with expected crc 2023-06-10T21:33:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:24.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:24.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:24.330 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:24.373 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:24.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:24.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:24.504 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:24.811 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:24.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:24.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:24.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:25.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:25.040 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:24 smithi152 ceph-mon[156819]: pgmap v1057: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 549 op/s; 5704/209073 objects degraded (2.728%); 4319/209073 objects misplaced (2.066%); 968 KiB/s, 15 objects/s recovering 2023-06-10T21:33:25.155 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:25.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:25.253 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:25.253 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:33:25.253 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:33:25.253 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:25.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:25.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:24 smithi064 ceph-mon[186706]: pgmap v1057: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 549 op/s; 5704/209073 objects degraded (2.728%); 4319/209073 objects misplaced (2.066%); 968 KiB/s, 15 objects/s recovering 2023-06-10T21:33:25.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:24 smithi064 ceph-mon[180297]: pgmap v1057: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 549 op/s; 5704/209073 objects degraded (2.728%); 4319/209073 objects misplaced (2.066%); 968 KiB/s, 15 objects/s recovering 2023-06-10T21:33:25.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:25.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:25.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:25.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:25.417 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:33:25.418 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:33:25.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:25.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:25.517 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:25.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:25.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:25.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:25.993 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:26.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:33:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:26.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:26.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:26.132 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:26.186 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:26.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:26.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:26.293 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:26.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:33:26.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:25 smithi064 ceph-mon[180297]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:26.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:25 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5704/209073 objects degraded (2.728%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:26.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:25 smithi064 ceph-mon[186706]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:26.380 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:25 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5704/209073 objects degraded (2.728%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:26.397 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:26.412 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:26.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:25 smithi152 ceph-mon[156819]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:25 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5704/209073 objects degraded (2.728%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:26.883 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:26.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:26.976 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1057,"stamp":"2023-06-10T21:33:23.795523+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4566226854,"num_objects":69691,"num_object_clones":0,"num_object_copies":209073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5704,"num_objects_misplaced":4319,"num_objects_unfound":0,"num_objects_dirty":69691,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":99136,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":613,"num_bytes_recovered":39198418,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96870,"ondisk_log_size":96870,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16207784,"kb_used_data":13545784,"kb_used_omap":30,"kb_used_meta":2661857,"kb_avail":546263128,"statfs":{"total":575970213888,"available":559373443072,"internally_reserved":0,"allocated":13870882816,"data_stored":13865868604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2725741606},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,1,7,2,2,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-287965184,"num_objects":-4394,"num_object_clones":0,"num_object_copies":-13182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":-168,"num_objects_unfound":0,"num_objects_dirty":-4394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4394,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002945"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":287,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.914968+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:33:11.914968+0000","last_peered":"2023-06-10T21:33:11.914968+0000","last_clean":"2023-06-10T21:33:11.914968+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:33:11.914968+0000","last_undegraded":"2023-06-10T21:33:11.914968+0000","last_fullsized":"2023-06-10T21:33:11.914968+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845729+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:33:11.845729+0000","last_peered":"2023-06-10T21:33:11.845729+0000","last_clean":"2023-06-10T21:33:11.845729+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:33:11.845729+0000","last_undegraded":"2023-06-10T21:33:11.845729+0000","last_fullsized":"2023-06-10T21:33:11.845729+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":203,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845314+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:33:11.845314+0000","last_peered":"2023-06-10T21:33:11.845314+0000","last_clean":"2023-06-10T21:33:11.845314+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:33:11.845314+0000","last_undegraded":"2023-06-10T21:33:11.845314+0000","last_fullsized":"2023-06-10T21:33:11.845314+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":151,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896094+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:33:11.896094+0000","last_peered":"2023-06-10T21:33:11.896094+0000","last_clean":"2023-06-10T21:33:11.896094+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:33:11.896094+0000","last_undegraded":"2023-06-10T21:33:11.896094+0000","last_fullsized":"2023-06-10T21:33:11.896094+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845595+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:33:11.845595+0000","last_peered":"2023-06-10T21:33:11.845595+0000","last_clean":"2023-06-10T21:33:11.845595+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:33:11.845595+0000","last_undegraded":"2023-06-10T21:33:11.845595+0000","last_fullsized":"2023-06-10T21:33:11.845595+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":261,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.895866+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:33:11.895866+0000","last_peered":"2023-06-10T21:33:11.895866+0000","last_clean":"2023-06-10T21:33:11.895866+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:33:11.895866+0000","last_undegraded":"2023-06-10T21:33:11.895866+0000","last_fullsized":"2023-06-10T21:33:11.895866+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.879728+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:33:16.879728+0000","last_peered":"2023-06-10T21:33:16.879728+0000","last_clean":"2023-06-10T21:33:16.879728+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:33:16.879728+0000","last_undegraded":"2023-06-10T21:33:16.879728+0000","last_fullsized":"2023-06-10T21:33:16.879728+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":242,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.844736+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:33:11.844736+0000","last_peered":"2023-06-10T21:33:11.844736+0000","last_clean":"2023-06-10T21:33:11.844736+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:33:11.844736+0000","last_undegraded":"2023-06-10T21:33:11.844736+0000","last_fullsized":"2023-06-10T21:33:11.844736+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903785+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:33:11.903785+0000","last_peered":"2023-06-10T21:33:11.903785+0000","last_clean":"2023-06-10T21:33:11.903785+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:33:11.903785+0000","last_undegraded":"2023-06-10T21:33:11.903785+0000","last_fullsized":"2023-06-10T21:33:11.903785+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.844999+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:33:11.844999+0000","last_peered":"2023-06-10T21:33:11.844999+0000","last_clean":"2023-06-10T21:33:11.844999+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:33:11.844999+0000","last_undegraded":"2023-06-10T21:33:11.844999+0000","last_fullsized":"2023-06-10T21:33:11.844999+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":22,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903802+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:33:11.903802+0000","last_peered":"2023-06-10T21:33:11.903802+0000","last_clean":"2023-06-10T21:33:11.903802+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:33:11.903802+0000","last_undegraded":"2023-06-10T21:33:11.903802+0000","last_fullsized":"2023-06-10T21:33:11.903802+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.872779+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:33:16.872779+0000","last_peered":"2023-06-10T21:33:16.872779+0000","last_clean":"2023-06-10T21:33:16.872779+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:33:16.872779+0000","last_undegraded":"2023-06-10T21:33:16.872779+0000","last_fullsized":"2023-06-10T21:33:16.872779+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":165,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904069+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:33:11.904069+0000","last_peered":"2023-06-10T21:33:11.904069+0000","last_clean":"2023-06-10T21:33:11.904069+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:33:11.904069+0000","last_undegraded":"2023-06-10T21:33:11.904069+0000","last_fullsized":"2023-06-10T21:33:11.904069+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":26,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903752+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:33:11.903752+0000","last_peered":"2023-06-10T21:33:11.903752+0000","last_clean":"2023-06-10T21:33:11.903752+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:33:11.903752+0000","last_undegraded":"2023-06-10T21:33:11.903752+0000","last_fullsized":"2023-06-10T21:33:11.903752+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":294,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903845+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:33:11.903845+0000","last_peered":"2023-06-10T21:33:11.903845+0000","last_clean":"2023-06-10T21:33:11.903845+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:33:11.903845+0000","last_undegraded":"2023-06-10T21:33:11.903845+0000","last_fullsized":"2023-06-10T21:33:11.903845+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":228,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845191+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:33:11.845191+0000","last_peered":"2023-06-10T21:33:11.845191+0000","last_clean":"2023-06-10T21:33:11.845191+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:33:11.845191+0000","last_undegraded":"2023-06-10T21:33:11.845191+0000","last_fullsized":"2023-06-10T21:33:11.845191+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":151,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:12.908504+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:33:12.908504+0000","last_peered":"2023-06-10T21:33:12.908504+0000","last_clean":"2023-06-10T21:33:12.908504+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:33:12.908504+0000","last_undegraded":"2023-06-10T21:33:12.908504+0000","last_fullsized":"2023-06-10T21:33:12.908504+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":223,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904035+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:33:11.904035+0000","last_peered":"2023-06-10T21:33:11.904035+0000","last_clean":"2023-06-10T21:33:11.904035+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:33:11.904035+0000","last_undegraded":"2023-06-10T21:33:11.904035+0000","last_fullsized":"2023-06-10T21:33:11.904035+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":271,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903533+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:33:11.903533+0000","last_peered":"2023-06-10T21:33:11.903533+0000","last_clean":"2023-06-10T21:33:11.903533+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:33:11.903533+0000","last_undegraded":"2023-06-10T21:33:11.903533+0000","last_fullsized":"2023-06-10T21:33:11.903533+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":151,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.878851+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:33:16.878851+0000","last_peered":"2023-06-10T21:33:16.878851+0000","last_clean":"2023-06-10T21:33:16.878851+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:33:16.878851+0000","last_undegraded":"2023-06-10T21:33:16.878851+0000","last_fullsized":"2023-06-10T21:33:16.878851+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":270,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:12.908450+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:33:12.908450+0000","last_peered":"2023-06-10T21:33:12.908450+0000","last_clean":"2023-06-10T21:33:12.908450+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:33:12.908450+0000","last_undegraded":"2023-06-10T21:33:12.908450+0000","last_fullsized":"2023-06-10T21:33:12.908450+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":246,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845874+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:33:11.845874+0000","last_peered":"2023-06-10T21:33:11.845874+0000","last_clean":"2023-06-10T21:33:11.845874+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:33:11.845874+0000","last_undegraded":"2023-06-10T21:33:11.845874+0000","last_fullsized":"2023-06-10T21:33:11.845874+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"295'6267","reported_seq":6302,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.916060+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:33:22.916060+0000","last_peered":"2023-06-10T21:33:22.916060+0000","last_clean":"2023-06-10T21:33:22.916060+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:33:22.916060+0000","last_undegraded":"2023-06-10T21:33:22.916060+0000","last_fullsized":"2023-06-10T21:33:22.916060+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6267,"log_dups_size":0,"ondisk_log_size":6267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6267,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":20,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904087+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:33:11.904087+0000","last_peered":"2023-06-10T21:33:11.904087+0000","last_clean":"2023-06-10T21:33:11.904087+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:33:11.904087+0000","last_undegraded":"2023-06-10T21:33:11.904087+0000","last_fullsized":"2023-06-10T21:33:11.904087+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":16,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903494+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:33:11.903494+0000","last_peered":"2023-06-10T21:33:11.903494+0000","last_clean":"2023-06-10T21:33:11.903494+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:33:11.903494+0000","last_undegraded":"2023-06-10T21:33:11.903494+0000","last_fullsized":"2023-06-10T21:33:11.903494+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"295'6280","reported_seq":6316,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.912578+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:33:22.912578+0000","last_peered":"2023-06-10T21:33:22.912578+0000","last_clean":"2023-06-10T21:33:22.912578+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:33:22.912578+0000","last_undegraded":"2023-06-10T21:33:22.912578+0000","last_fullsized":"2023-06-10T21:33:22.912578+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6280,"log_dups_size":0,"ondisk_log_size":6280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6280,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":20,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903880+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:33:11.903880+0000","last_peered":"2023-06-10T21:33:11.903880+0000","last_clean":"2023-06-10T21:33:11.903880+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:33:11.903880+0000","last_undegraded":"2023-06-10T21:33:11.903880+0000","last_fullsized":"2023-06-10T21:33:11.903880+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":168,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.468834+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:33:16.468834+0000","last_peered":"2023-06-10T21:33:16.468834+0000","last_clean":"2023-06-10T21:33:16.468834+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:33:16.468834+0000","last_undegraded":"2023-06-10T21:33:16.468834+0000","last_fullsized":"2023-06-10T21:33:16.468834+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"295'6447","reported_seq":6486,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:22.826427+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:33:22.826427+0000","last_peered":"2023-06-10T21:33:22.826427+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:33:22.826427+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292225024,"num_objects":4459,"num_object_clones":0,"num_object_copies":13377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6447,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":4459},{"shards":"0,3","objects":-935}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":302,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903947+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:33:11.903947+0000","last_peered":"2023-06-10T21:33:11.903947+0000","last_clean":"2023-06-10T21:33:11.903947+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:33:11.903947+0000","last_undegraded":"2023-06-10T21:33:11.903947+0000","last_fullsized":"2023-06-10T21:33:11.903947+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":16,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903557+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:33:11.903557+0000","last_peered":"2023-06-10T21:33:11.903557+0000","last_clean":"2023-06-10T21:33:11.903557+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:33:11.903557+0000","last_undegraded":"2023-06-10T21:33:11.903557+0000","last_fullsized":"2023-06-10T21:33:11.903557+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"295'6081","reported_seq":6115,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.562610+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:33:19.562610+0000","last_peered":"2023-06-10T21:33:19.562610+0000","last_clean":"2023-06-10T21:33:19.562610+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:33:19.562610+0000","last_undegraded":"2023-06-10T21:33:19.562610+0000","last_fullsized":"2023-06-10T21:33:19.562610+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6081,"log_dups_size":0,"ondisk_log_size":6081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288555008,"num_objects":4403,"num_object_clones":0,"num_object_copies":13209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6081,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845333+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:33:11.845333+0000","last_peered":"2023-06-10T21:33:11.845333+0000","last_clean":"2023-06-10T21:33:11.845333+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:33:11.845333+0000","last_undegraded":"2023-06-10T21:33:11.845333+0000","last_fullsized":"2023-06-10T21:33:11.845333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":118,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.897465+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:33:11.897465+0000","last_peered":"2023-06-10T21:33:11.897465+0000","last_clean":"2023-06-10T21:33:11.897465+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:33:11.897465+0000","last_undegraded":"2023-06-10T21:33:11.897465+0000","last_fullsized":"2023-06-10T21:33:11.897465+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"295'6069","reported_seq":7253,"reported_epoch":295,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:19.609510+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:33:19.609510+0000","last_peered":"2023-06-10T21:33:19.609510+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:33:19.609510+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6069,"log_dups_size":0,"ondisk_log_size":6069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283049984,"num_objects":4319,"num_object_clones":0,"num_object_copies":12957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":251,"num_objects_misplaced":4319,"num_objects_unfound":0,"num_objects_dirty":4319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6069,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":572,"num_bytes_recovered":37421056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-251},{"shards":"1,2,3","objects":4319}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":245,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845826+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:33:11.845826+0000","last_peered":"2023-06-10T21:33:11.845826+0000","last_clean":"2023-06-10T21:33:11.845826+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:33:11.845826+0000","last_undegraded":"2023-06-10T21:33:11.845826+0000","last_fullsized":"2023-06-10T21:33:11.845826+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845419+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:33:11.845419+0000","last_peered":"2023-06-10T21:33:11.845419+0000","last_clean":"2023-06-10T21:33:11.845419+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:33:11.845419+0000","last_undegraded":"2023-06-10T21:33:11.845419+0000","last_fullsized":"2023-06-10T21:33:11.845419+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"295'6303","reported_seq":6337,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.634248+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:33:19.634248+0000","last_peered":"2023-06-10T21:33:19.634248+0000","last_clean":"2023-06-10T21:33:19.634248+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:33:19.634248+0000","last_undegraded":"2023-06-10T21:33:19.634248+0000","last_fullsized":"2023-06-10T21:33:19.634248+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6303,"log_dups_size":0,"ondisk_log_size":6303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291700736,"num_objects":4451,"num_object_clones":0,"num_object_copies":13353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6303,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.468847+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:33:16.468847+0000","last_peered":"2023-06-10T21:33:16.468847+0000","last_clean":"2023-06-10T21:33:16.468847+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:33:16.468847+0000","last_undegraded":"2023-06-10T21:33:16.468847+0000","last_fullsized":"2023-06-10T21:33:16.468847+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":286,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.908601+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:33:11.908601+0000","last_peered":"2023-06-10T21:33:11.908601+0000","last_clean":"2023-06-10T21:33:11.908601+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:33:11.908601+0000","last_undegraded":"2023-06-10T21:33:11.908601+0000","last_fullsized":"2023-06-10T21:33:11.908601+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"295'6171","reported_seq":6207,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.417927+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:33:21.417927+0000","last_peered":"2023-06-10T21:33:21.417927+0000","last_clean":"2023-06-10T21:33:21.417927+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:33:21.417927+0000","last_undegraded":"2023-06-10T21:33:21.417927+0000","last_fullsized":"2023-06-10T21:33:21.417927+0000","mapping_epoch":269,"log_start":"276'400","ondisk_log_start":"276'400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5771,"log_dups_size":0,"ondisk_log_size":5771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286195712,"num_objects":4367,"num_object_clones":0,"num_object_copies":13101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6171,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":272,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.902821+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:33:11.902821+0000","last_peered":"2023-06-10T21:33:11.902821+0000","last_clean":"2023-06-10T21:33:11.902821+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:33:11.902821+0000","last_undegraded":"2023-06-10T21:33:11.902821+0000","last_fullsized":"2023-06-10T21:33:11.902821+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":68,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.846022+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:33:11.846022+0000","last_peered":"2023-06-10T21:33:11.846022+0000","last_clean":"2023-06-10T21:33:11.846022+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:33:11.846022+0000","last_undegraded":"2023-06-10T21:33:11.846022+0000","last_fullsized":"2023-06-10T21:33:11.846022+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"295'6214","reported_seq":6250,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.297855+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:33:21.297855+0000","last_peered":"2023-06-10T21:33:21.297855+0000","last_clean":"2023-06-10T21:33:21.297855+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:33:21.297855+0000","last_undegraded":"2023-06-10T21:33:21.297855+0000","last_fullsized":"2023-06-10T21:33:21.297855+0000","mapping_epoch":269,"log_start":"276'400","ondisk_log_start":"276'400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283639808,"num_objects":4328,"num_object_clones":0,"num_object_copies":12984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6214,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":249,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.455617+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:33:16.455617+0000","last_peered":"2023-06-10T21:33:16.455617+0000","last_clean":"2023-06-10T21:33:16.455617+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:33:16.455617+0000","last_undegraded":"2023-06-10T21:33:16.455617+0000","last_fullsized":"2023-06-10T21:33:16.455617+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":204,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.846088+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:33:11.846088+0000","last_peered":"2023-06-10T21:33:11.846088+0000","last_clean":"2023-06-10T21:33:11.846088+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:33:11.846088+0000","last_undegraded":"2023-06-10T21:33:11.846088+0000","last_fullsized":"2023-06-10T21:33:11.846088+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"295'6096","reported_seq":6132,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.624615+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:33:19.624615+0000","last_peered":"2023-06-10T21:33:19.624615+0000","last_clean":"2023-06-10T21:33:19.624615+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:33:19.624615+0000","last_undegraded":"2023-06-10T21:33:19.624615+0000","last_fullsized":"2023-06-10T21:33:19.624615+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283574296,"num_objects":4328,"num_object_clones":0,"num_object_copies":12984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4328,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6096,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":118,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.895675+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:33:11.895675+0000","last_peered":"2023-06-10T21:33:11.895675+0000","last_clean":"2023-06-10T21:33:11.895675+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:33:11.895675+0000","last_undegraded":"2023-06-10T21:33:11.895675+0000","last_fullsized":"2023-06-10T21:33:11.895675+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":298,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845762+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:33:11.845762+0000","last_peered":"2023-06-10T21:33:11.845762+0000","last_clean":"2023-06-10T21:33:11.845762+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:33:11.845762+0000","last_undegraded":"2023-06-10T21:33:11.845762+0000","last_fullsized":"2023-06-10T21:33:11.845762+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":138,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904629+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:33:11.904629+0000","last_peered":"2023-06-10T21:33:11.904629+0000","last_clean":"2023-06-10T21:33:11.904629+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:33:11.904629+0000","last_undegraded":"2023-06-10T21:33:11.904629+0000","last_fullsized":"2023-06-10T21:33:11.904629+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"295'6163","reported_seq":6198,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.421658+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:33:21.421658+0000","last_peered":"2023-06-10T21:33:21.421658+0000","last_clean":"2023-06-10T21:33:21.421658+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:33:21.421658+0000","last_undegraded":"2023-06-10T21:33:21.421658+0000","last_fullsized":"2023-06-10T21:33:21.421658+0000","mapping_epoch":269,"log_start":"276'300","ondisk_log_start":"276'300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5863,"log_dups_size":0,"ondisk_log_size":5863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287375360,"num_objects":4385,"num_object_clones":0,"num_object_copies":13155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6163,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":164,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904704+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:33:11.904704+0000","last_peered":"2023-06-10T21:33:11.904704+0000","last_clean":"2023-06-10T21:33:11.904704+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:33:11.904704+0000","last_undegraded":"2023-06-10T21:33:11.904704+0000","last_fullsized":"2023-06-10T21:33:11.904704+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":238,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.897673+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:33:11.897673+0000","last_peered":"2023-06-10T21:33:11.897673+0000","last_clean":"2023-06-10T21:33:11.897673+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:33:11.897673+0000","last_undegraded":"2023-06-10T21:33:11.897673+0000","last_fullsized":"2023-06-10T21:33:11.897673+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":127,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.909608+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:33:11.909608+0000","last_peered":"2023-06-10T21:33:11.909608+0000","last_clean":"2023-06-10T21:33:11.909608+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:33:11.909608+0000","last_undegraded":"2023-06-10T21:33:11.909608+0000","last_fullsized":"2023-06-10T21:33:11.909608+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"295'6202","reported_seq":6237,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.411269+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:33:21.411269+0000","last_peered":"2023-06-10T21:33:21.411269+0000","last_clean":"2023-06-10T21:33:21.411269+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:33:21.411269+0000","last_undegraded":"2023-06-10T21:33:21.411269+0000","last_fullsized":"2023-06-10T21:33:21.411269+0000","mapping_epoch":269,"log_start":"276'400","ondisk_log_start":"276'400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283639808,"num_objects":4328,"num_object_clones":0,"num_object_copies":12984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6202,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":138,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.908507+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:33:11.908507+0000","last_peered":"2023-06-10T21:33:11.908507+0000","last_clean":"2023-06-10T21:33:11.908507+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:33:11.908507+0000","last_undegraded":"2023-06-10T21:33:11.908507+0000","last_fullsized":"2023-06-10T21:33:11.908507+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.905103+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:33:11.905103+0000","last_peered":"2023-06-10T21:33:11.905103+0000","last_clean":"2023-06-10T21:33:11.905103+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:33:11.905103+0000","last_undegraded":"2023-06-10T21:33:11.905103+0000","last_fullsized":"2023-06-10T21:33:11.905103+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":187,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896247+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:33:11.896247+0000","last_peered":"2023-06-10T21:33:11.896247+0000","last_clean":"2023-06-10T21:33:11.896247+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:33:11.896247+0000","last_undegraded":"2023-06-10T21:33:11.896247+0000","last_fullsized":"2023-06-10T21:33:11.896247+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"295'6093","reported_seq":6128,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.430697+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:33:21.430697+0000","last_peered":"2023-06-10T21:33:21.430697+0000","last_clean":"2023-06-10T21:33:21.430697+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:33:21.430697+0000","last_undegraded":"2023-06-10T21:33:21.430697+0000","last_fullsized":"2023-06-10T21:33:21.430697+0000","mapping_epoch":269,"log_start":"276'300","ondisk_log_start":"276'300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279773184,"num_objects":4269,"num_object_clones":0,"num_object_copies":12807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6093,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":318,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.910864+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:33:11.910864+0000","last_peered":"2023-06-10T21:33:11.910864+0000","last_clean":"2023-06-10T21:33:11.910864+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:33:11.910864+0000","last_undegraded":"2023-06-10T21:33:11.910864+0000","last_fullsized":"2023-06-10T21:33:11.910864+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":458,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903596+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:33:11.903596+0000","last_peered":"2023-06-10T21:33:11.903596+0000","last_clean":"2023-06-10T21:33:11.903596+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:33:11.903596+0000","last_undegraded":"2023-06-10T21:33:11.903596+0000","last_fullsized":"2023-06-10T21:33:11.903596+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":232,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845392+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:33:11.845392+0000","last_peered":"2023-06-10T21:33:11.845392+0000","last_clean":"2023-06-10T21:33:11.845392+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:33:11.845392+0000","last_undegraded":"2023-06-10T21:33:11.845392+0000","last_fullsized":"2023-06-10T21:33:11.845392+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903960+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:33:11.903960+0000","last_peered":"2023-06-10T21:33:11.903960+0000","last_clean":"2023-06-10T21:33:11.903960+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:33:11.903960+0000","last_undegraded":"2023-06-10T21:33:11.903960+0000","last_fullsized":"2023-06-10T21:33:11.903960+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"295'6228","reported_seq":6263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:21.440670+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:33:21.440670+0000","last_peered":"2023-06-10T21:33:21.440670+0000","last_clean":"2023-06-10T21:33:21.440670+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:33:21.440670+0000","last_undegraded":"2023-06-10T21:33:21.440670+0000","last_fullsized":"2023-06-10T21:33:21.440670+0000","mapping_epoch":269,"log_start":"276'400","ondisk_log_start":"276'400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289144832,"num_objects":4412,"num_object_clones":0,"num_object_copies":13236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6228,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":314,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903664+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:33:11.903664+0000","last_peered":"2023-06-10T21:33:11.903664+0000","last_clean":"2023-06-10T21:33:11.903664+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:33:11.903664+0000","last_undegraded":"2023-06-10T21:33:11.903664+0000","last_fullsized":"2023-06-10T21:33:11.903664+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":133,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896949+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:33:11.896949+0000","last_peered":"2023-06-10T21:33:11.896949+0000","last_clean":"2023-06-10T21:33:11.896949+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:33:11.896949+0000","last_undegraded":"2023-06-10T21:33:11.896949+0000","last_fullsized":"2023-06-10T21:33:11.896949+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":118,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.895912+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:33:11.895912+0000","last_peered":"2023-06-10T21:33:11.895912+0000","last_clean":"2023-06-10T21:33:11.895912+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:33:11.895912+0000","last_undegraded":"2023-06-10T21:33:11.895912+0000","last_fullsized":"2023-06-10T21:33:11.895912+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"295'6161","reported_seq":6196,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.536105+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:33:19.536105+0000","last_peered":"2023-06-10T21:33:19.536105+0000","last_clean":"2023-06-10T21:33:19.536105+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:33:19.536105+0000","last_undegraded":"2023-06-10T21:33:19.536105+0000","last_fullsized":"2023-06-10T21:33:19.536105+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286720000,"num_objects":4375,"num_object_clones":0,"num_object_copies":13125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6161,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":299,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903909+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:33:11.903909+0000","last_peered":"2023-06-10T21:33:11.903909+0000","last_clean":"2023-06-10T21:33:11.903909+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:33:11.903909+0000","last_undegraded":"2023-06-10T21:33:11.903909+0000","last_fullsized":"2023-06-10T21:33:11.903909+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.909709+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:33:11.909709+0000","last_peered":"2023-06-10T21:33:11.909709+0000","last_clean":"2023-06-10T21:33:11.909709+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:33:11.909709+0000","last_undegraded":"2023-06-10T21:33:11.909709+0000","last_fullsized":"2023-06-10T21:33:11.909709+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":114,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.897808+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:33:11.897808+0000","last_peered":"2023-06-10T21:33:11.897808+0000","last_clean":"2023-06-10T21:33:11.897808+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:33:11.897808+0000","last_undegraded":"2023-06-10T21:33:11.897808+0000","last_fullsized":"2023-06-10T21:33:11.897808+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"295'6079","reported_seq":6113,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.642510+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:33:19.642510+0000","last_peered":"2023-06-10T21:33:19.642510+0000","last_clean":"2023-06-10T21:33:19.642510+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:33:19.642510+0000","last_undegraded":"2023-06-10T21:33:19.642510+0000","last_fullsized":"2023-06-10T21:33:19.642510+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282787840,"num_objects":4315,"num_object_clones":0,"num_object_copies":12945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6079,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":272,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.469046+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:33:16.469046+0000","last_peered":"2023-06-10T21:33:16.469046+0000","last_clean":"2023-06-10T21:33:16.469046+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:33:16.469046+0000","last_undegraded":"2023-06-10T21:33:16.469046+0000","last_fullsized":"2023-06-10T21:33:16.469046+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":69,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.469021+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:33:16.469021+0000","last_peered":"2023-06-10T21:33:16.469021+0000","last_clean":"2023-06-10T21:33:16.469021+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:33:16.469021+0000","last_undegraded":"2023-06-10T21:33:16.469021+0000","last_fullsized":"2023-06-10T21:33:16.469021+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":253,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.844939+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:33:11.844939+0000","last_peered":"2023-06-10T21:33:11.844939+0000","last_clean":"2023-06-10T21:33:11.844939+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:33:11.844939+0000","last_undegraded":"2023-06-10T21:33:11.844939+0000","last_fullsized":"2023-06-10T21:33:11.844939+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"295'6021","reported_seq":6055,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.605223+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:33:19.605223+0000","last_peered":"2023-06-10T21:33:19.605223+0000","last_clean":"2023-06-10T21:33:19.605223+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:33:19.605223+0000","last_undegraded":"2023-06-10T21:33:19.605223+0000","last_fullsized":"2023-06-10T21:33:19.605223+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6021,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904003+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:33:11.904003+0000","last_peered":"2023-06-10T21:33:11.904003+0000","last_clean":"2023-06-10T21:33:11.904003+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:33:11.904003+0000","last_undegraded":"2023-06-10T21:33:11.904003+0000","last_fullsized":"2023-06-10T21:33:11.904003+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":122,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896075+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:33:11.896075+0000","last_peered":"2023-06-10T21:33:11.896075+0000","last_clean":"2023-06-10T21:33:11.896075+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:33:11.896075+0000","last_undegraded":"2023-06-10T21:33:11.896075+0000","last_fullsized":"2023-06-10T21:33:11.896075+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":250,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896731+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:33:11.896731+0000","last_peered":"2023-06-10T21:33:11.896731+0000","last_clean":"2023-06-10T21:33:11.896731+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:33:11.896731+0000","last_undegraded":"2023-06-10T21:33:11.896731+0000","last_fullsized":"2023-06-10T21:33:11.896731+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":117,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.897886+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:33:11.897886+0000","last_peered":"2023-06-10T21:33:11.897886+0000","last_clean":"2023-06-10T21:33:11.897886+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:33:11.897886+0000","last_undegraded":"2023-06-10T21:33:11.897886+0000","last_fullsized":"2023-06-10T21:33:11.897886+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.909884+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:33:11.909884+0000","last_peered":"2023-06-10T21:33:11.909884+0000","last_clean":"2023-06-10T21:33:11.909884+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:33:11.909884+0000","last_undegraded":"2023-06-10T21:33:11.909884+0000","last_fullsized":"2023-06-10T21:33:11.909884+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":20,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.903605+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:33:11.903605+0000","last_peered":"2023-06-10T21:33:11.903605+0000","last_clean":"2023-06-10T21:33:11.903605+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:33:11.903605+0000","last_undegraded":"2023-06-10T21:33:11.903605+0000","last_fullsized":"2023-06-10T21:33:11.903605+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845668+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:33:11.845668+0000","last_peered":"2023-06-10T21:33:11.845668+0000","last_clean":"2023-06-10T21:33:11.845668+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:33:11.845668+0000","last_undegraded":"2023-06-10T21:33:11.845668+0000","last_fullsized":"2023-06-10T21:33:11.845668+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":72,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845212+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:33:11.845212+0000","last_peered":"2023-06-10T21:33:11.845212+0000","last_clean":"2023-06-10T21:33:11.845212+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:33:11.845212+0000","last_undegraded":"2023-06-10T21:33:11.845212+0000","last_fullsized":"2023-06-10T21:33:11.845212+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":232,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845328+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:33:11.845328+0000","last_peered":"2023-06-10T21:33:11.845328+0000","last_clean":"2023-06-10T21:33:11.845328+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:33:11.845328+0000","last_undegraded":"2023-06-10T21:33:11.845328+0000","last_fullsized":"2023-06-10T21:33:11.845328+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":278,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845821+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:33:11.845821+0000","last_peered":"2023-06-10T21:33:11.845821+0000","last_clean":"2023-06-10T21:33:11.845821+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:33:11.845821+0000","last_undegraded":"2023-06-10T21:33:11.845821+0000","last_fullsized":"2023-06-10T21:33:11.845821+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":281,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.845116+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:33:11.845116+0000","last_peered":"2023-06-10T21:33:11.845116+0000","last_clean":"2023-06-10T21:33:11.845116+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:33:11.845116+0000","last_undegraded":"2023-06-10T21:33:11.845116+0000","last_fullsized":"2023-06-10T21:33:11.845116+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":183,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.896219+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:33:11.896219+0000","last_peered":"2023-06-10T21:33:11.896219+0000","last_clean":"2023-06-10T21:33:11.896219+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:33:11.896219+0000","last_undegraded":"2023-06-10T21:33:11.896219+0000","last_fullsized":"2023-06-10T21:33:11.896219+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":27,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-06-10T21:33:11.904849+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:33:11.904849+0000","last_peered":"2023-06-10T21:33:11.904849+0000","last_clean":"2023-06-10T21:33:11.904849+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:33:11.904849+0000","last_undegraded":"2023-06-10T21:33:11.904849+0000","last_fullsized":"2023-06-10T21:33:11.904849+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4565631000,"num_objects":69667,"num_object_clones":0,"num_object_copies":209001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5704,"num_objects_misplaced":4319,"num_objects_unfound":0,"num_objects_dirty":69667,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98875,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":572,"num_bytes_recovered":37421056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13810544640,"data_stored":13810532424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96675,"ondisk_log_size":96675,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430635,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1684200,"kb_used_data":1121760,"kb_used_omap":0,"kb_used_meta":562432,"kb_avail":92060952,"statfs":{"total":95995035648,"available":94270414848,"internally_reserved":0,"allocated":1148682240,"data_stored":1148119704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":575930368},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":6.9740000000000002,"15min":8.7959999999999994},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":6.4779999999999998,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.2070000000000001,"5min":6.915,"15min":9.0050000000000008},"min":{"1min":0.26400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":6.3810000000000002,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":2.5800000000000001,"15min":3.0350000000000001},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":49.732999999999997,"5min":87.417000000000002,"15min":130.19399999999999},"last":5.5670000000000002},{"interface":"front","average":{"1min":4.883,"5min":2.7210000000000001,"15min":3.573},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":50.060000000000002,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:32:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7450000000000001,"5min":3.2509999999999999,"15min":3.0289999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.138999999999999,"5min":138.36099999999999,"15min":160.99299999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.623,"5min":3.097,"15min":3.6640000000000001},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.4350000000000005,"5min":137.708,"15min":212.13399999999999},"last":4.9240000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:33:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":2.8660000000000001,"15min":2.6219999999999999},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.9489999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.393,"5min":2.4969999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.31,"15min":0.31},"max":{"1min":5.7869999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":1.776}]},{"osd":4,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.442,"15min":0.75},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.41399999999999998,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.59499999999999997,"15min":1.091},"min":{"1min":0.28299999999999997,"5min":0.245,"15min":0.245},"max":{"1min":0.48699999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.53900000000000003,"15min":1.847},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52600000000000002,"5min":3.9729999999999999,"15min":211.99799999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.52400000000000002,"15min":1.7},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.29299999999999998},"max":{"1min":0.499,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.504}]},{"osd":6,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.58899999999999997,"15min":1.1180000000000001},"min":{"1min":0.33800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":3.375,"15min":13.073},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.499,"15min":1.0069999999999999},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.53000000000000003,"5min":1.897,"15min":10.874000000000001},"last":0.65200000000000002}]}]},{"osd":6,"up_from":0,"seq":180388626863,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566306,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4302148,"kb_used_data":3691068,"kb_used_omap":4,"kb_used_meta":611003,"kb_avail":89443004,"statfs":{"total":95995035648,"available":91589636096,"internally_reserved":0,"allocated":3779653632,"data_stored":3778713641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":625667982},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":9.0679999999999996,"15min":12.176},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.0659999999999998,"5min":8.9860000000000007,"15min":11.875999999999999},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":10.244,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.316}]},{"osd":2,"last update":"Sat Jun 10 21:32:31 2023","interfaces":[{"interface":"back","average":{"1min":9.0399999999999991,"5min":8.0719999999999992,"15min":3.8290000000000002},"min":{"1min":0.27700000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":54.578000000000003,"5min":107.857,"15min":107.857},"last":45.639000000000003},{"interface":"front","average":{"1min":7.8310000000000004,"5min":8.1389999999999993,"15min":3.54},"min":{"1min":0.40300000000000002,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":54.140999999999998,"5min":107.958,"15min":107.958},"last":46.380000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:33:07 2023","interfaces":[{"interface":"back","average":{"1min":7.6360000000000001,"5min":3.6960000000000002,"15min":2.5489999999999999},"min":{"1min":0.37,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":80.584999999999994,"5min":80.584999999999994,"15min":80.584999999999994},"last":42.694000000000003},{"interface":"front","average":{"1min":7.5880000000000001,"5min":4.8230000000000004,"15min":3.7290000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":80.873999999999995,"5min":80.873999999999995,"15min":80.873999999999995},"last":42.734999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.4650000000000001,"15min":1.518},"min":{"1min":0.38,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.4260000000000002,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.432},{"interface":"front","average":{"1min":2.536,"5min":1.4490000000000001,"15min":1.4039999999999999},"min":{"1min":0.28799999999999998,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":8.2739999999999991,"5min":8.2739999999999991,"15min":14.460000000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.802,"15min":4.1050000000000004},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5499999999999998,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.472,"15min":3.7949999999999999},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.4820000000000002,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.893,"15min":2.012},"min":{"1min":0.38300000000000001,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":6.5750000000000002,"5min":8.8580000000000005,"15min":14.858000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.655,"5min":1.4850000000000001,"15min":1.698},"min":{"1min":0.47599999999999998,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.5140000000000002,"5min":11.476000000000001,"15min":13.48},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.5980000000000001,"15min":2.4199999999999999},"min":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":5.3769999999999998,"5min":16.113,"15min":162.399},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.9319999999999999,"5min":1.6819999999999999,"15min":2.1960000000000002},"min":{"1min":0.34899999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":162.27199999999999},"last":0.50700000000000001}]}]},{"osd":0,"up_from":244,"seq":1047972020283,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2474964,"kb_used_data":2231244,"kb_used_omap":10,"kb_used_meta":243701,"kb_avail":91270188,"statfs":{"total":95995035648,"available":93460672512,"internally_reserved":0,"allocated":2284793856,"data_stored":2283778400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":249549876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,1,1,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":10.356999999999999,"5min":5.2729999999999997,"15min":4.6970000000000001},"min":{"1min":0.26200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":139.68000000000001,"5min":139.68000000000001,"15min":139.68000000000001},"last":4.9429999999999996},{"interface":"front","average":{"1min":9.1170000000000009,"5min":5.5350000000000001,"15min":5.8860000000000001},"min":{"1min":0.22900000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":141.79499999999999,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.246}]},{"osd":2,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":3.278,"5min":5.9630000000000001,"15min":5.9550000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":8.4440000000000008,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.495},{"interface":"front","average":{"1min":3.0419999999999998,"5min":5.9349999999999996,"15min":5.3819999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.8930000000000007,"5min":208.34299999999999,"15min":208.34299999999999},"last":2.2450000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:33:06 2023","interfaces":[{"interface":"back","average":{"1min":6.7779999999999996,"5min":5.0220000000000002,"15min":5.8650000000000002},"min":{"1min":0.245,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":111.745,"5min":111.745,"15min":111.745},"last":0.42499999999999999},{"interface":"front","average":{"1min":9.2940000000000005,"5min":6.0640000000000001,"15min":6.8769999999999998},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":114.205,"5min":114.205,"15min":114.205},"last":2.5539999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":3.0939999999999999,"15min":4.4290000000000003},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.645,"5min":29.709,"15min":29.709},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.8929999999999998,"15min":3.8140000000000001},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.7350000000000003,"5min":13.872,"15min":13.872},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":2.9470000000000001,"15min":3.9060000000000001},"min":{"1min":0.42799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7859999999999996,"5min":14.105,"15min":14.105},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.5980000000000001,"5min":2.6720000000000002,"15min":3.2999999999999998},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.6020000000000003,"5min":19.846,"15min":19.846},"last":2.3420000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":3.226,"15min":4.6200000000000001},"min":{"1min":0.442,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.605,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.571,"5min":2.98,"15min":3.9420000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.8140000000000001,"5min":13.82,"15min":13.82},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.8700000000000001,"15min":3.6819999999999999},"min":{"1min":0.46899999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.5609999999999999,"5min":19.866,"15min":19.866},"last":17.523},{"interface":"front","average":{"1min":1.143,"5min":3.2280000000000002,"15min":4.6189999999999998},"min":{"1min":0.44900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":3.5529999999999999,"5min":29.77,"15min":29.77},"last":17.792999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075694,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3244540,"kb_used_data":2807220,"kb_used_omap":7,"kb_used_meta":437304,"kb_avail":90500612,"statfs":{"total":95995035648,"available":92672626688,"internally_reserved":0,"allocated":2874593280,"data_stored":2873660969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":447799377},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":4.2190000000000003,"15min":4.5419999999999998},"min":{"1min":0.30299999999999999,"5min":0.222,"15min":0.222},"max":{"1min":12.406000000000001,"5min":66.784000000000006,"15min":66.784000000000006},"last":1.7450000000000001},{"interface":"front","average":{"1min":2.657,"5min":4.9100000000000001,"15min":5.2309999999999999},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":10.458,"5min":67.251999999999995,"15min":67.251999999999995},"last":3.0110000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":11.742000000000001,"5min":4.3499999999999996,"15min":2.5070000000000001},"min":{"1min":0.41399999999999998,"5min":0.19,"15min":0.19},"max":{"1min":147.20099999999999,"5min":147.20099999999999,"15min":147.20099999999999},"last":7.8300000000000001},{"interface":"front","average":{"1min":11.183999999999999,"5min":4.0140000000000002,"15min":2.2930000000000001},"min":{"1min":0.36599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":147.81299999999999,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.38}]},{"osd":3,"last update":"Sat Jun 10 21:33:01 2023","interfaces":[{"interface":"back","average":{"1min":10.353999999999999,"5min":4.8970000000000002,"15min":5.1260000000000003},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":138.03299999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":10.279,"5min":5.444,"15min":6.0099999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":138.095,"5min":138.095,"15min":138.095},"last":3.9609999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.9649999999999999,"15min":2.6880000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.28999999999999998},"max":{"1min":5.2729999999999997,"5min":110.337,"15min":224.00999999999999},"last":3.0539999999999998},{"interface":"front","average":{"1min":1.9450000000000001,"5min":3.3100000000000001,"15min":2.8679999999999999},"min":{"1min":0.439,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.1470000000000002,"5min":110.2,"15min":223.28399999999999},"last":7.7450000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.5049999999999999,"15min":1.3600000000000001},"min":{"1min":0.379,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":2.6259999999999999,"5min":14.180999999999999,"15min":14.180999999999999},"last":1.696},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.708,"15min":1.6040000000000001},"min":{"1min":0.39300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1799999999999997,"5min":16.125,"15min":16.125},"last":3.0950000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.9530000000000001,"15min":1.6759999999999999},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":5.4640000000000004,"5min":16.183,"15min":16.183},"last":3.1309999999999998},{"interface":"front","average":{"1min":1.9810000000000001,"5min":2.2250000000000001,"15min":1.8680000000000001},"min":{"1min":0.52400000000000002,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":5.0650000000000004,"5min":11.029,"15min":17.244},"last":7.7960000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.786,"15min":1.7529999999999999},"min":{"1min":0.443,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.669,"5min":113.913,"15min":113.913},"last":60.517000000000003},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.125,"15min":2.1000000000000001},"min":{"1min":0.47499999999999998,"5min":0.42099999999999999,"15min":0.41399999999999998},"max":{"1min":5.1390000000000002,"5min":113.937,"15min":113.937},"last":60.545999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824054,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2792200,"kb_used_data":2572288,"kb_used_omap":7,"kb_used_meta":219896,"kb_avail":90952952,"statfs":{"total":95995035648,"available":93135822848,"internally_reserved":0,"allocated":2634022912,"data_stored":2633018962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":225173523},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,5,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":4.2039999999999997,"15min":5.0999999999999996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.522,"5min":93.992999999999995,"15min":93.992999999999995},"last":4.0250000000000004},{"interface":"front","average":{"1min":1.387,"5min":3.891,"15min":4.8520000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":5.7729999999999997,"5min":95.623999999999995,"15min":95.623999999999995},"last":12.34}]},{"osd":1,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":3.7759999999999998,"15min":3.6840000000000002},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.422999999999998,"5min":19.859000000000002,"15min":19.859000000000002},"last":6.7110000000000003},{"interface":"front","average":{"1min":5.2939999999999996,"5min":3.028,"15min":3.3319999999999999},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.384,"5min":17.384,"15min":17.384},"last":4.2169999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":6.6130000000000004,"15min":10.819000000000001},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.9580000000000002,"5min":87.555999999999997,"15min":87.555999999999997},"last":12.273},{"interface":"front","average":{"1min":4.806,"5min":8.3859999999999992,"15min":12.356999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.170999999999999,"5min":80.566999999999993,"15min":80.566999999999993},"last":6.1500000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.4430000000000001,"15min":1.397},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.6070000000000002,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":12.218999999999999},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.7570000000000001,"15min":2.7450000000000001},"min":{"1min":0.40100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.384,"5min":19.728000000000002,"15min":19.728000000000002},"last":2.2229999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2010000000000001,"15min":1.329},"min":{"1min":0.39400000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":4.0010000000000003},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.4610000000000001,"15min":1.4399999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6419999999999999,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":12.31}]},{"osd":6,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7810000000000001,"15min":2.7690000000000001},"min":{"1min":0.438,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":14.119,"5min":19.805,"15min":19.805},"last":2.1859999999999999},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.2689999999999999,"15min":1.383},"min":{"1min":0.47999999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9809999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":3.9700000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.7919999999999998,"15min":2.782},"min":{"1min":0.503,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":14.422000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":0.501},{"interface":"front","average":{"1min":1.524,"5min":1.2450000000000001,"15min":1.357},"min":{"1min":0.44900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9379999999999997,"5min":6.9379999999999997,"15min":6.9379999999999997},"last":3.9300000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019319,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1709732,"kb_used_data":1122204,"kb_used_omap":0,"kb_used_meta":587520,"kb_avail":92035420,"statfs":{"total":95995035648,"available":94244270080,"internally_reserved":0,"allocated":1149136896,"data_stored":1148576928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":601620480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":5.0650000000000004,"15min":8.8070000000000004},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.2489999999999997,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.255},{"interface":"front","average":{"1min":1.484,"5min":5.6269999999999998,"15min":8.5020000000000007},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3120000000000003,"5min":144.303,"15min":144.303},"last":6.548}]},{"osd":1,"last update":"Sat Jun 10 21:32:24 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.544,"15min":2.9060000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":6.7569999999999997,"5min":6.7569999999999997,"15min":182.035},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.206,"5min":1.6890000000000001,"15min":2.7170000000000001},"min":{"1min":0.29899999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.2429999999999999,"5min":9.4760000000000009,"15min":181.73099999999999},"last":0.29099999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":1.9510000000000001,"15min":1.554},"min":{"1min":0.34699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.116,"5min":11.116,"15min":22.614000000000001},"last":1.9399999999999999},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.7809999999999999,"15min":1.5309999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.798,"5min":18.329999999999998,"15min":22.326000000000001},"last":1.9199999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.125,"15min":2.3029999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":29.856999999999999,"5min":29.856999999999999,"15min":29.856999999999999},"last":0.499},{"interface":"front","average":{"1min":10.365,"5min":3.5270000000000001,"15min":2.5379999999999998},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":160.06200000000001,"5min":160.06200000000001,"15min":160.06200000000001},"last":2.6179999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.64100000000000001,"15min":1.0820000000000001},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.86299999999999999,"5min":6.742,"15min":13.941000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.72999999999999998,"15min":1.107},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.61199999999999999,"5min":6.0819999999999999,"15min":11.904999999999999},"last":5.25}]},{"osd":6,"last update":"Sat Jun 10 21:32:54 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.58399999999999996,"15min":1.0800000000000001},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.76900000000000002,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":1.7210000000000001},{"interface":"front","average":{"1min":0.439,"5min":0.75900000000000001,"15min":1.143},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.72399999999999998,"5min":8.3109999999999999,"15min":13.845000000000001},"last":1.71}]},{"osd":7,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.64900000000000002,"15min":1.2549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.51100000000000001,"5min":6.798,"15min":16.265000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.64700000000000002,"15min":1.175},"min":{"1min":0.316,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.627,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":1.6060000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823091,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2276065280,"data_stored":2276065280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3752988672,"data_stored":3752984600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2867138560,"data_stored":2867134488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2629566464,"data_stored":2629566464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1143148544,"data_stored":1143144472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1141637120,"data_stored":1141637120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:33:26.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:33:26.994 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:27.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:27.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:26 smithi064 ceph-mon[186706]: pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5748/205956 objects degraded (2.791%); 4155/205956 objects misplaced (2.017%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:27.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:26 smithi064 ceph-mon[180297]: pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5748/205956 objects degraded (2.791%); 4155/205956 objects misplaced (2.017%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:27.310 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:27.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:27.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:27.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:26 smithi152 ceph-mon[156819]: pgmap v1058: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5748/205956 objects degraded (2.791%); 4155/205956 objects misplaced (2.017%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:33:27.558 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:27.580 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:27.637 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:33:27.699 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:27.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:27.711 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:33:27.793 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:33:27.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:27.847 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:27.857 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:33:27.897 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:33:27.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:33:27.993 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:33:27.997 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:28.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:28.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:28.034 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:28.034 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:28.047 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:33:28.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.056 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.057 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.057 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.057 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.057 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:33:28.101 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:28.133 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/968932341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:28.136 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:28.286 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:28.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:28.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:28.363 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/968932341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:28.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/968932341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:28.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:28.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:28.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:33:29.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:29 smithi152 ceph-mon[156819]: pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 5748/203157 objects degraded (2.829%); 4155/203157 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:29.298 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:29.301 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:29 smithi064 ceph-mon[186706]: pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 5748/203157 objects degraded (2.829%); 4155/203157 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:29.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:29 smithi064 ceph-mon[180297]: pgmap v1059: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 4.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 5748/203157 objects degraded (2.829%); 4155/203157 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:29.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:29.307 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:29.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:29.436 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:29.543 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:29.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:29.651 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:29.683 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:29.948 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:29.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:30.053 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:30.086 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:30.136 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:30.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:30.244 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:30.274 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:30.461 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:30.489 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:30.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:30.596 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:30.626 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:30.673 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:30.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:30.780 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:30.900 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:30.988 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:30.988 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:31.062 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:31.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:31.144 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[186706]: pgmap v1060: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 5789/198039 objects degraded (2.923%); 3991/198039 objects misplaced (2.015%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:31.144 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5789/198039 objects degraded (2.923%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:31.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:33:31.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[180297]: pgmap v1060: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 5789/198039 objects degraded (2.923%); 3991/198039 objects misplaced (2.015%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:31.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5789/198039 objects degraded (2.923%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:31.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:33:31.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:31.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:31.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:31 smithi152 ceph-mon[156819]: pgmap v1060: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 520 op/s; 5789/198039 objects degraded (2.923%); 3991/198039 objects misplaced (2.015%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5789/198039 objects degraded (2.923%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:33:31.655 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:31.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:31.760 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:31.793 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:31.844 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:31.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:31.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:31.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:33:31.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:32.294 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:32.493 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:33:32.496 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:32.638 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:32.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:32.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:32.865 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:33.014 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:33.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:33.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:33.109 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:33:33.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:33.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:33 smithi064 ceph-mon[180297]: pgmap v1061: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 5789/195123 objects degraded (2.967%); 3991/195123 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:33.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:33:33.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:33 smithi064 ceph-mon[186706]: pgmap v1061: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 5789/195123 objects degraded (2.967%); 3991/195123 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:33.147 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:33:33.147 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:32] "GET /metrics HTTP/1.1" 200 224073 "" "Prometheus/2.33.4" 2023-06-10T21:33:33.162 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:33 smithi152 ceph-mon[156819]: pgmap v1061: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 4.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 512 op/s; 5789/195123 objects degraded (2.967%); 3991/195123 objects misplaced (2.045%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:33.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:33:33.669 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:33.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:33.775 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:33.806 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:33.944 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:33.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:34.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:34.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:34.336 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:34.337 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:34.360 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:34.361 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.361 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:33:34.361 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: osdmap e297: 8 total, 8 up, 6 in 2023-06-10T21:33:34.361 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.361 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:34.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:34.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:33:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: osdmap e297: 8 total, 8 up, 6 in 2023-06-10T21:33:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:33:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2162307988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:33:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: osdmap e297: 8 total, 8 up, 6 in 2023-06-10T21:33:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: failed to encode map e297 with expected crc 2023-06-10T21:33:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:34.737 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:34.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:34.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:34.959 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:34.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:35.077 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:33:35.088 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:33:35.102 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:35 smithi152 ceph-mon[156819]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5789/193671 objects degraded (2.989%); 3991/193671 objects misplaced (2.061%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:35.133 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:33:35.382 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:35.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:35.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:35.392 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:35 smithi064 ceph-mon[180297]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5789/193671 objects degraded (2.989%); 3991/193671 objects misplaced (2.061%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:35.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:35 smithi064 ceph-mon[186706]: pgmap v1063: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5789/193671 objects degraded (2.989%); 3991/193671 objects misplaced (2.061%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:35.492 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:35.570 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:35.729 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:36 smithi064 ceph-mon[180297]: from='client.56158 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5789/193671 objects degraded (2.989%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:36.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:36 smithi064 ceph-mon[186706]: from='client.56158 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:36.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5789/193671 objects degraded (2.989%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:36 smithi152 ceph-mon[156819]: from='client.56158 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:36.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5789/193671 objects degraded (2.989%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:36.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:36.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:36.624 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1063,"stamp":"2023-06-10T21:33:33.803258+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4229765030,"num_objects":64557,"num_object_clones":0,"num_object_copies":193671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5789,"num_objects_misplaced":3991,"num_objects_unfound":0,"num_objects_dirty":64557,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":104270,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":856,"num_bytes_recovered":55123666,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99404,"ondisk_log_size":99404,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15283304,"kb_used_data":12601592,"kb_used_omap":30,"kb_used_meta":2681249,"kb_avail":547187608,"statfs":{"total":575970213888,"available":560320110592,"internally_reserved":0,"allocated":12904030208,"data_stored":12898999404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2745599014},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,6,3,3,1,2,0,0,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-336461824,"num_objects":-5134,"num_object_clones":0,"num_object_copies":-15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":85,"num_objects_misplaced":-328,"num_objects_unfound":0,"num_objects_dirty":-5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":243,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007734"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":288,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.872832+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:33:16.872832+0000","last_peered":"2023-06-10T21:33:16.872832+0000","last_clean":"2023-06-10T21:33:16.872832+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:33:16.872832+0000","last_undegraded":"2023-06-10T21:33:16.872832+0000","last_fullsized":"2023-06-10T21:33:16.872832+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":229,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.864544+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:33:16.864544+0000","last_peered":"2023-06-10T21:33:16.864544+0000","last_clean":"2023-06-10T21:33:16.864544+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:33:16.864544+0000","last_undegraded":"2023-06-10T21:33:16.864544+0000","last_fullsized":"2023-06-10T21:33:16.864544+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":204,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.864178+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:33:16.864178+0000","last_peered":"2023-06-10T21:33:16.864178+0000","last_clean":"2023-06-10T21:33:16.864178+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:33:16.864178+0000","last_undegraded":"2023-06-10T21:33:16.864178+0000","last_fullsized":"2023-06-10T21:33:16.864178+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":152,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.879006+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:33:16.879006+0000","last_peered":"2023-06-10T21:33:16.879006+0000","last_clean":"2023-06-10T21:33:16.879006+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:33:16.879006+0000","last_undegraded":"2023-06-10T21:33:16.879006+0000","last_fullsized":"2023-06-10T21:33:16.879006+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862030+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:33:16.862030+0000","last_peered":"2023-06-10T21:33:16.862030+0000","last_clean":"2023-06-10T21:33:16.862030+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:33:16.862030+0000","last_undegraded":"2023-06-10T21:33:16.862030+0000","last_fullsized":"2023-06-10T21:33:16.862030+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.879623+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:33:16.879623+0000","last_peered":"2023-06-10T21:33:16.879623+0000","last_clean":"2023-06-10T21:33:16.879623+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:33:16.879623+0000","last_undegraded":"2023-06-10T21:33:16.879623+0000","last_fullsized":"2023-06-10T21:33:16.879623+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.921312+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:33:22.921312+0000","last_peered":"2023-06-10T21:33:22.921312+0000","last_clean":"2023-06-10T21:33:22.921312+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:33:22.921312+0000","last_undegraded":"2023-06-10T21:33:22.921312+0000","last_fullsized":"2023-06-10T21:33:22.921312+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.861544+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:33:16.861544+0000","last_peered":"2023-06-10T21:33:16.861544+0000","last_clean":"2023-06-10T21:33:16.861544+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:33:16.861544+0000","last_undegraded":"2023-06-10T21:33:16.861544+0000","last_fullsized":"2023-06-10T21:33:16.861544+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":17,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002521+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:33:17.002521+0000","last_peered":"2023-06-10T21:33:17.002521+0000","last_clean":"2023-06-10T21:33:17.002521+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:33:17.002521+0000","last_undegraded":"2023-06-10T21:33:17.002521+0000","last_fullsized":"2023-06-10T21:33:17.002521+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":18,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.863565+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:33:16.863565+0000","last_peered":"2023-06-10T21:33:16.863565+0000","last_clean":"2023-06-10T21:33:16.863565+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:33:16.863565+0000","last_undegraded":"2023-06-10T21:33:16.863565+0000","last_fullsized":"2023-06-10T21:33:16.863565+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":23,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.006813+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:33:17.006813+0000","last_peered":"2023-06-10T21:33:17.006813+0000","last_clean":"2023-06-10T21:33:17.006813+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:33:17.006813+0000","last_undegraded":"2023-06-10T21:33:17.006813+0000","last_fullsized":"2023-06-10T21:33:17.006813+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":258,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:19.634353+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:33:19.634353+0000","last_peered":"2023-06-10T21:33:19.634353+0000","last_clean":"2023-06-10T21:33:19.634353+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:33:19.634353+0000","last_undegraded":"2023-06-10T21:33:19.634353+0000","last_fullsized":"2023-06-10T21:33:19.634353+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":166,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002368+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:33:17.002368+0000","last_peered":"2023-06-10T21:33:17.002368+0000","last_clean":"2023-06-10T21:33:17.002368+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:33:17.002368+0000","last_undegraded":"2023-06-10T21:33:17.002368+0000","last_fullsized":"2023-06-10T21:33:17.002368+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":27,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002231+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:33:17.002231+0000","last_peered":"2023-06-10T21:33:17.002231+0000","last_clean":"2023-06-10T21:33:17.002231+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:33:17.002231+0000","last_undegraded":"2023-06-10T21:33:17.002231+0000","last_fullsized":"2023-06-10T21:33:17.002231+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002270+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:33:17.002270+0000","last_peered":"2023-06-10T21:33:17.002270+0000","last_clean":"2023-06-10T21:33:17.002270+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:33:17.002270+0000","last_undegraded":"2023-06-10T21:33:17.002270+0000","last_fullsized":"2023-06-10T21:33:17.002270+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":229,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.861682+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:33:16.861682+0000","last_peered":"2023-06-10T21:33:16.861682+0000","last_clean":"2023-06-10T21:33:16.861682+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:33:16.861682+0000","last_undegraded":"2023-06-10T21:33:16.861682+0000","last_fullsized":"2023-06-10T21:33:16.861682+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":152,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.875181+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:33:16.875181+0000","last_peered":"2023-06-10T21:33:16.875181+0000","last_clean":"2023-06-10T21:33:16.875181+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:33:16.875181+0000","last_undegraded":"2023-06-10T21:33:16.875181+0000","last_fullsized":"2023-06-10T21:33:16.875181+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.013940+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:33:17.013940+0000","last_peered":"2023-06-10T21:33:17.013940+0000","last_clean":"2023-06-10T21:33:17.013940+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:33:17.013940+0000","last_undegraded":"2023-06-10T21:33:17.013940+0000","last_fullsized":"2023-06-10T21:33:17.013940+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002061+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:33:17.002061+0000","last_peered":"2023-06-10T21:33:17.002061+0000","last_clean":"2023-06-10T21:33:17.002061+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:33:17.002061+0000","last_undegraded":"2023-06-10T21:33:17.002061+0000","last_fullsized":"2023-06-10T21:33:17.002061+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":152,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.921259+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:33:22.921259+0000","last_peered":"2023-06-10T21:33:22.921259+0000","last_clean":"2023-06-10T21:33:22.921259+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:33:22.921259+0000","last_undegraded":"2023-06-10T21:33:22.921259+0000","last_fullsized":"2023-06-10T21:33:22.921259+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":271,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.878271+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:33:16.878271+0000","last_peered":"2023-06-10T21:33:16.878271+0000","last_clean":"2023-06-10T21:33:16.878271+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:33:16.878271+0000","last_undegraded":"2023-06-10T21:33:16.878271+0000","last_fullsized":"2023-06-10T21:33:16.878271+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":247,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862340+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:33:16.862340+0000","last_peered":"2023-06-10T21:33:16.862340+0000","last_clean":"2023-06-10T21:33:16.862340+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:33:16.862340+0000","last_undegraded":"2023-06-10T21:33:16.862340+0000","last_fullsized":"2023-06-10T21:33:16.862340+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"296'6577","reported_seq":6614,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:32.876573+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:33:32.876573+0000","last_peered":"2023-06-10T21:33:32.876573+0000","last_clean":"2023-06-10T21:33:32.876573+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:33:32.876573+0000","last_undegraded":"2023-06-10T21:33:32.876573+0000","last_fullsized":"2023-06-10T21:33:32.876573+0000","mapping_epoch":269,"log_start":"276'100","ondisk_log_start":"276'100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259981312,"num_objects":3967,"num_object_clones":0,"num_object_copies":11901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6577,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":21,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.003024+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:33:17.003024+0000","last_peered":"2023-06-10T21:33:17.003024+0000","last_clean":"2023-06-10T21:33:17.003024+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:33:17.003024+0000","last_undegraded":"2023-06-10T21:33:17.003024+0000","last_fullsized":"2023-06-10T21:33:17.003024+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.859559+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:33:16.859559+0000","last_peered":"2023-06-10T21:33:16.859559+0000","last_clean":"2023-06-10T21:33:16.859559+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:33:16.859559+0000","last_undegraded":"2023-06-10T21:33:16.859559+0000","last_fullsized":"2023-06-10T21:33:16.859559+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"296'6603","reported_seq":6640,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:32.862139+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:33:32.862139+0000","last_peered":"2023-06-10T21:33:32.862139+0000","last_clean":"2023-06-10T21:33:32.862139+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:33:32.862139+0000","last_undegraded":"2023-06-10T21:33:32.862139+0000","last_fullsized":"2023-06-10T21:33:32.862139+0000","mapping_epoch":269,"log_start":"276'200","ondisk_log_start":"276'200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261685248,"num_objects":3993,"num_object_clones":0,"num_object_copies":11979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6603,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":21,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002528+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:33:17.002528+0000","last_peered":"2023-06-10T21:33:17.002528+0000","last_clean":"2023-06-10T21:33:17.002528+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:33:17.002528+0000","last_undegraded":"2023-06-10T21:33:17.002528+0000","last_fullsized":"2023-06-10T21:33:17.002528+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":169,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.864208+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:33:16.864208+0000","last_peered":"2023-06-10T21:33:16.864208+0000","last_clean":"2023-06-10T21:33:16.864208+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:33:16.864208+0000","last_undegraded":"2023-06-10T21:33:16.864208+0000","last_fullsized":"2023-06-10T21:33:16.864208+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"296'6792","reported_seq":6832,"reported_epoch":296,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:32.896096+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:33:32.896096+0000","last_peered":"2023-06-10T21:33:32.896096+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:33:32.896096+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"276'400","ondisk_log_start":"276'400","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269615104,"num_objects":4114,"num_object_clones":0,"num_object_copies":12342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6792,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":4114},{"shards":"0,3","objects":-1280}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":303,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.013810+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:33:17.013810+0000","last_peered":"2023-06-10T21:33:17.013810+0000","last_clean":"2023-06-10T21:33:17.013810+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:33:17.013810+0000","last_undegraded":"2023-06-10T21:33:17.013810+0000","last_fullsized":"2023-06-10T21:33:17.013810+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002143+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:33:17.002143+0000","last_peered":"2023-06-10T21:33:17.002143+0000","last_clean":"2023-06-10T21:33:17.002143+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:33:17.002143+0000","last_undegraded":"2023-06-10T21:33:17.002143+0000","last_fullsized":"2023-06-10T21:33:17.002143+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"296'6398","reported_seq":6433,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.581127+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:33:29.581127+0000","last_peered":"2023-06-10T21:33:29.581127+0000","last_clean":"2023-06-10T21:33:29.581127+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:33:29.581127+0000","last_undegraded":"2023-06-10T21:33:29.581127+0000","last_fullsized":"2023-06-10T21:33:29.581127+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267780096,"num_objects":4086,"num_object_clones":0,"num_object_copies":12258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.954651+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:33:22.954651+0000","last_peered":"2023-06-10T21:33:22.954651+0000","last_clean":"2023-06-10T21:33:22.954651+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:33:22.954651+0000","last_undegraded":"2023-06-10T21:33:22.954651+0000","last_fullsized":"2023-06-10T21:33:22.954651+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":119,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876716+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:33:16.876716+0000","last_peered":"2023-06-10T21:33:16.876716+0000","last_clean":"2023-06-10T21:33:16.876716+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:33:16.876716+0000","last_undegraded":"2023-06-10T21:33:16.876716+0000","last_fullsized":"2023-06-10T21:33:16.876716+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"296'6395","reported_seq":8067,"reported_epoch":296,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:29.659727+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:33:29.659727+0000","last_peered":"2023-06-10T21:33:29.659727+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:33:29.659727+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261685248,"num_objects":3993,"num_object_clones":0,"num_object_copies":11979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":336,"num_objects_misplaced":3991,"num_objects_unfound":0,"num_objects_dirty":3993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6395,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":815,"num_bytes_recovered":53346304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-334},{"shards":"1,2,3","objects":3991}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":247,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.954633+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:33:22.954633+0000","last_peered":"2023-06-10T21:33:22.954633+0000","last_clean":"2023-06-10T21:33:22.954633+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:33:22.954633+0000","last_undegraded":"2023-06-10T21:33:22.954633+0000","last_fullsized":"2023-06-10T21:33:22.954633+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.954721+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:33:22.954721+0000","last_peered":"2023-06-10T21:33:22.954721+0000","last_clean":"2023-06-10T21:33:22.954721+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:33:22.954721+0000","last_undegraded":"2023-06-10T21:33:22.954721+0000","last_fullsized":"2023-06-10T21:33:22.954721+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"296'6631","reported_seq":6667,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.652841+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:33:29.652841+0000","last_peered":"2023-06-10T21:33:29.652841+0000","last_clean":"2023-06-10T21:33:29.652841+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:33:29.652841+0000","last_undegraded":"2023-06-10T21:33:29.652841+0000","last_fullsized":"2023-06-10T21:33:29.652841+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270204928,"num_objects":4123,"num_object_clones":0,"num_object_copies":12369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862724+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:33:16.862724+0000","last_peered":"2023-06-10T21:33:16.862724+0000","last_clean":"2023-06-10T21:33:16.862724+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:33:16.862724+0000","last_undegraded":"2023-06-10T21:33:16.862724+0000","last_fullsized":"2023-06-10T21:33:16.862724+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":287,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.877529+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:33:16.877529+0000","last_peered":"2023-06-10T21:33:16.877529+0000","last_clean":"2023-06-10T21:33:16.877529+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:33:16.877529+0000","last_undegraded":"2023-06-10T21:33:16.877529+0000","last_fullsized":"2023-06-10T21:33:16.877529+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"296'6489","reported_seq":6526,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.449466+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:33:31.449466+0000","last_peered":"2023-06-10T21:33:31.449466+0000","last_clean":"2023-06-10T21:33:31.449466+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:33:31.449466+0000","last_undegraded":"2023-06-10T21:33:31.449466+0000","last_fullsized":"2023-06-10T21:33:31.449466+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5789,"log_dups_size":0,"ondisk_log_size":5789,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265355264,"num_objects":4049,"num_object_clones":0,"num_object_copies":12147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6489,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":273,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.874547+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:33:16.874547+0000","last_peered":"2023-06-10T21:33:16.874547+0000","last_clean":"2023-06-10T21:33:16.874547+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:33:16.874547+0000","last_undegraded":"2023-06-10T21:33:16.874547+0000","last_fullsized":"2023-06-10T21:33:16.874547+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":70,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.955150+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:33:22.955150+0000","last_peered":"2023-06-10T21:33:22.955150+0000","last_clean":"2023-06-10T21:33:22.955150+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:33:22.955150+0000","last_undegraded":"2023-06-10T21:33:22.955150+0000","last_fullsized":"2023-06-10T21:33:22.955150+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"296'6546","reported_seq":6583,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.448853+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:33:31.448853+0000","last_peered":"2023-06-10T21:33:31.448853+0000","last_clean":"2023-06-10T21:33:31.448853+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:33:31.448853+0000","last_undegraded":"2023-06-10T21:33:31.448853+0000","last_fullsized":"2023-06-10T21:33:31.448853+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261881856,"num_objects":3996,"num_object_clones":0,"num_object_copies":11988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6546,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":250,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.861908+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:33:16.861908+0000","last_peered":"2023-06-10T21:33:16.861908+0000","last_clean":"2023-06-10T21:33:16.861908+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:33:16.861908+0000","last_undegraded":"2023-06-10T21:33:16.861908+0000","last_fullsized":"2023-06-10T21:33:16.861908+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.957094+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:33:22.957094+0000","last_peered":"2023-06-10T21:33:22.957094+0000","last_clean":"2023-06-10T21:33:22.957094+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:33:22.957094+0000","last_undegraded":"2023-06-10T21:33:22.957094+0000","last_fullsized":"2023-06-10T21:33:22.957094+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"296'6425","reported_seq":6462,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.676920+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:33:29.676920+0000","last_peered":"2023-06-10T21:33:29.676920+0000","last_clean":"2023-06-10T21:33:29.676920+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:33:29.676920+0000","last_undegraded":"2023-06-10T21:33:29.676920+0000","last_fullsized":"2023-06-10T21:33:29.676920+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012952,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6425,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":119,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876941+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:33:16.876941+0000","last_peered":"2023-06-10T21:33:16.876941+0000","last_clean":"2023-06-10T21:33:16.876941+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:33:16.876941+0000","last_undegraded":"2023-06-10T21:33:16.876941+0000","last_fullsized":"2023-06-10T21:33:16.876941+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.864639+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:33:16.864639+0000","last_peered":"2023-06-10T21:33:16.864639+0000","last_clean":"2023-06-10T21:33:16.864639+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:33:16.864639+0000","last_undegraded":"2023-06-10T21:33:16.864639+0000","last_fullsized":"2023-06-10T21:33:16.864639+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.875038+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:33:16.875038+0000","last_peered":"2023-06-10T21:33:16.875038+0000","last_clean":"2023-06-10T21:33:16.875038+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:33:16.875038+0000","last_undegraded":"2023-06-10T21:33:16.875038+0000","last_fullsized":"2023-06-10T21:33:16.875038+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"296'6508","reported_seq":6544,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.336610+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:33:31.336610+0000","last_peered":"2023-06-10T21:33:31.336610+0000","last_clean":"2023-06-10T21:33:31.336610+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:33:31.336610+0000","last_undegraded":"2023-06-10T21:33:31.336610+0000","last_fullsized":"2023-06-10T21:33:31.336610+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5808,"log_dups_size":0,"ondisk_log_size":5808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6508,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":165,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.875108+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:33:16.875108+0000","last_peered":"2023-06-10T21:33:16.875108+0000","last_clean":"2023-06-10T21:33:16.875108+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:33:16.875108+0000","last_undegraded":"2023-06-10T21:33:16.875108+0000","last_fullsized":"2023-06-10T21:33:16.875108+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":239,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876844+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:33:16.876844+0000","last_peered":"2023-06-10T21:33:16.876844+0000","last_clean":"2023-06-10T21:33:16.876844+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:33:16.876844+0000","last_undegraded":"2023-06-10T21:33:16.876844+0000","last_fullsized":"2023-06-10T21:33:16.876844+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.871873+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:33:16.871873+0000","last_peered":"2023-06-10T21:33:16.871873+0000","last_clean":"2023-06-10T21:33:16.871873+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:33:16.871873+0000","last_undegraded":"2023-06-10T21:33:16.871873+0000","last_fullsized":"2023-06-10T21:33:16.871873+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"296'6511","reported_seq":6548,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.370243+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:33:31.370243+0000","last_peered":"2023-06-10T21:33:31.370243+0000","last_clean":"2023-06-10T21:33:31.370243+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:33:31.370243+0000","last_undegraded":"2023-06-10T21:33:31.370243+0000","last_fullsized":"2023-06-10T21:33:31.370243+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263389184,"num_objects":4019,"num_object_clones":0,"num_object_copies":12057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6511,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":139,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.874618+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:33:16.874618+0000","last_peered":"2023-06-10T21:33:16.874618+0000","last_clean":"2023-06-10T21:33:16.874618+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:33:16.874618+0000","last_undegraded":"2023-06-10T21:33:16.874618+0000","last_fullsized":"2023-06-10T21:33:16.874618+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.872002+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:33:16.872002+0000","last_peered":"2023-06-10T21:33:16.872002+0000","last_clean":"2023-06-10T21:33:16.872002+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:33:16.872002+0000","last_undegraded":"2023-06-10T21:33:16.872002+0000","last_fullsized":"2023-06-10T21:33:16.872002+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":188,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.879799+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:33:16.879799+0000","last_peered":"2023-06-10T21:33:16.879799+0000","last_clean":"2023-06-10T21:33:16.879799+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:33:16.879799+0000","last_undegraded":"2023-06-10T21:33:16.879799+0000","last_fullsized":"2023-06-10T21:33:16.879799+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"296'6378","reported_seq":6415,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.440502+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:33:31.440502+0000","last_peered":"2023-06-10T21:33:31.440502+0000","last_clean":"2023-06-10T21:33:31.440502+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:33:31.440502+0000","last_undegraded":"2023-06-10T21:33:31.440502+0000","last_fullsized":"2023-06-10T21:33:31.440502+0000","mapping_epoch":269,"log_start":"276'600","ondisk_log_start":"276'600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5778,"log_dups_size":0,"ondisk_log_size":5778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261095424,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6378,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":319,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.878440+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:33:16.878440+0000","last_peered":"2023-06-10T21:33:16.878440+0000","last_clean":"2023-06-10T21:33:16.878440+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:33:16.878440+0000","last_undegraded":"2023-06-10T21:33:16.878440+0000","last_fullsized":"2023-06-10T21:33:16.878440+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":459,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.859640+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:33:16.859640+0000","last_peered":"2023-06-10T21:33:16.859640+0000","last_clean":"2023-06-10T21:33:16.859640+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:33:16.859640+0000","last_undegraded":"2023-06-10T21:33:16.859640+0000","last_fullsized":"2023-06-10T21:33:16.859640+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":233,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862875+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:33:16.862875+0000","last_peered":"2023-06-10T21:33:16.862875+0000","last_clean":"2023-06-10T21:33:16.862875+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:33:16.862875+0000","last_undegraded":"2023-06-10T21:33:16.862875+0000","last_fullsized":"2023-06-10T21:33:16.862875+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.013882+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:33:17.013882+0000","last_peered":"2023-06-10T21:33:17.013882+0000","last_clean":"2023-06-10T21:33:17.013882+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:33:17.013882+0000","last_undegraded":"2023-06-10T21:33:17.013882+0000","last_fullsized":"2023-06-10T21:33:17.013882+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"296'6544","reported_seq":6581,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:31.382509+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:33:31.382509+0000","last_peered":"2023-06-10T21:33:31.382509+0000","last_clean":"2023-06-10T21:33:31.382509+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:33:31.382509+0000","last_undegraded":"2023-06-10T21:33:31.382509+0000","last_fullsized":"2023-06-10T21:33:31.382509+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5844,"log_dups_size":0,"ondisk_log_size":5844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268435456,"num_objects":4096,"num_object_clones":0,"num_object_copies":12288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6544,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.859621+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:33:16.859621+0000","last_peered":"2023-06-10T21:33:16.859621+0000","last_clean":"2023-06-10T21:33:16.859621+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:33:16.859621+0000","last_undegraded":"2023-06-10T21:33:16.859621+0000","last_fullsized":"2023-06-10T21:33:16.859621+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":134,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876639+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:33:16.876639+0000","last_peered":"2023-06-10T21:33:16.876639+0000","last_clean":"2023-06-10T21:33:16.876639+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:33:16.876639+0000","last_undegraded":"2023-06-10T21:33:16.876639+0000","last_fullsized":"2023-06-10T21:33:16.876639+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.878782+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:33:16.878782+0000","last_peered":"2023-06-10T21:33:16.878782+0000","last_clean":"2023-06-10T21:33:16.878782+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:33:16.878782+0000","last_undegraded":"2023-06-10T21:33:16.878782+0000","last_fullsized":"2023-06-10T21:33:16.878782+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"296'6476","reported_seq":6513,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.677006+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:33:29.677006+0000","last_peered":"2023-06-10T21:33:29.677006+0000","last_clean":"2023-06-10T21:33:29.677006+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:33:29.677006+0000","last_undegraded":"2023-06-10T21:33:29.677006+0000","last_fullsized":"2023-06-10T21:33:29.677006+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6476,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":300,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.013862+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:33:17.013862+0000","last_peered":"2023-06-10T21:33:17.013862+0000","last_clean":"2023-06-10T21:33:17.013862+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:33:17.013862+0000","last_undegraded":"2023-06-10T21:33:17.013862+0000","last_fullsized":"2023-06-10T21:33:17.013862+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":137,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.871961+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:33:16.871961+0000","last_peered":"2023-06-10T21:33:16.871961+0000","last_clean":"2023-06-10T21:33:16.871961+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:33:16.871961+0000","last_undegraded":"2023-06-10T21:33:16.871961+0000","last_fullsized":"2023-06-10T21:33:16.871961+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":115,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876965+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:33:16.876965+0000","last_peered":"2023-06-10T21:33:16.876965+0000","last_clean":"2023-06-10T21:33:16.876965+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:33:16.876965+0000","last_undegraded":"2023-06-10T21:33:16.876965+0000","last_fullsized":"2023-06-10T21:33:16.876965+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"296'6388","reported_seq":6423,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.650310+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:33:29.650310+0000","last_peered":"2023-06-10T21:33:29.650310+0000","last_clean":"2023-06-10T21:33:29.650310+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:33:29.650310+0000","last_undegraded":"2023-06-10T21:33:29.650310+0000","last_fullsized":"2023-06-10T21:33:29.650310+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262537216,"num_objects":4006,"num_object_clones":0,"num_object_copies":12018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6388,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":273,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862455+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:33:16.862455+0000","last_peered":"2023-06-10T21:33:16.862455+0000","last_clean":"2023-06-10T21:33:16.862455+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:33:16.862455+0000","last_undegraded":"2023-06-10T21:33:16.862455+0000","last_fullsized":"2023-06-10T21:33:16.862455+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":70,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862216+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:33:16.862216+0000","last_peered":"2023-06-10T21:33:16.862216+0000","last_clean":"2023-06-10T21:33:16.862216+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:33:16.862216+0000","last_undegraded":"2023-06-10T21:33:16.862216+0000","last_fullsized":"2023-06-10T21:33:16.862216+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":254,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.861753+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:33:16.861753+0000","last_peered":"2023-06-10T21:33:16.861753+0000","last_clean":"2023-06-10T21:33:16.861753+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:33:16.861753+0000","last_undegraded":"2023-06-10T21:33:16.861753+0000","last_fullsized":"2023-06-10T21:33:16.861753+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"296'6348","reported_seq":6383,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:29.620921+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:33:29.620921+0000","last_peered":"2023-06-10T21:33:29.620921+0000","last_clean":"2023-06-10T21:33:29.620921+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:33:29.620921+0000","last_undegraded":"2023-06-10T21:33:29.620921+0000","last_fullsized":"2023-06-10T21:33:29.620921+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262668288,"num_objects":4008,"num_object_clones":0,"num_object_copies":12024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002926+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:33:17.002926+0000","last_peered":"2023-06-10T21:33:17.002926+0000","last_clean":"2023-06-10T21:33:17.002926+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:33:17.002926+0000","last_undegraded":"2023-06-10T21:33:17.002926+0000","last_fullsized":"2023-06-10T21:33:17.002926+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":123,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.878932+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:33:16.878932+0000","last_peered":"2023-06-10T21:33:16.878932+0000","last_clean":"2023-06-10T21:33:16.878932+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:33:16.878932+0000","last_undegraded":"2023-06-10T21:33:16.878932+0000","last_fullsized":"2023-06-10T21:33:16.878932+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":251,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.876434+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:33:16.876434+0000","last_peered":"2023-06-10T21:33:16.876434+0000","last_clean":"2023-06-10T21:33:16.876434+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:33:16.876434+0000","last_undegraded":"2023-06-10T21:33:16.876434+0000","last_fullsized":"2023-06-10T21:33:16.876434+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":119,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-10T21:33:22.952003+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:33:22.952003+0000","last_peered":"2023-06-10T21:33:22.952003+0000","last_clean":"2023-06-10T21:33:22.952003+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:33:22.952003+0000","last_undegraded":"2023-06-10T21:33:22.952003+0000","last_fullsized":"2023-06-10T21:33:22.952003+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.872111+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:33:16.872111+0000","last_peered":"2023-06-10T21:33:16.872111+0000","last_clean":"2023-06-10T21:33:16.872111+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:33:16.872111+0000","last_undegraded":"2023-06-10T21:33:16.872111+0000","last_fullsized":"2023-06-10T21:33:16.872111+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":21,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:17.002203+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:33:17.002203+0000","last_peered":"2023-06-10T21:33:17.002203+0000","last_clean":"2023-06-10T21:33:17.002203+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:33:17.002203+0000","last_undegraded":"2023-06-10T21:33:17.002203+0000","last_fullsized":"2023-06-10T21:33:17.002203+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.865160+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:33:16.865160+0000","last_peered":"2023-06-10T21:33:16.865160+0000","last_clean":"2023-06-10T21:33:16.865160+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:33:16.865160+0000","last_undegraded":"2023-06-10T21:33:16.865160+0000","last_fullsized":"2023-06-10T21:33:16.865160+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":73,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862403+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:33:16.862403+0000","last_peered":"2023-06-10T21:33:16.862403+0000","last_clean":"2023-06-10T21:33:16.862403+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:33:16.862403+0000","last_undegraded":"2023-06-10T21:33:16.862403+0000","last_fullsized":"2023-06-10T21:33:16.862403+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":233,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862798+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:33:16.862798+0000","last_peered":"2023-06-10T21:33:16.862798+0000","last_clean":"2023-06-10T21:33:16.862798+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:33:16.862798+0000","last_undegraded":"2023-06-10T21:33:16.862798+0000","last_fullsized":"2023-06-10T21:33:16.862798+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":279,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.862057+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:33:16.862057+0000","last_peered":"2023-06-10T21:33:16.862057+0000","last_clean":"2023-06-10T21:33:16.862057+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:33:16.862057+0000","last_undegraded":"2023-06-10T21:33:16.862057+0000","last_fullsized":"2023-06-10T21:33:16.862057+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":282,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.863770+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:33:16.863770+0000","last_peered":"2023-06-10T21:33:16.863770+0000","last_clean":"2023-06-10T21:33:16.863770+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:33:16.863770+0000","last_undegraded":"2023-06-10T21:33:16.863770+0000","last_fullsized":"2023-06-10T21:33:16.863770+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":184,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.879878+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:33:16.879878+0000","last_peered":"2023-06-10T21:33:16.879878+0000","last_clean":"2023-06-10T21:33:16.879878+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:33:16.879878+0000","last_undegraded":"2023-06-10T21:33:16.879878+0000","last_fullsized":"2023-06-10T21:33:16.879878+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":28,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-10T21:33:16.871726+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:33:16.871726+0000","last_peered":"2023-06-10T21:33:16.871726+0000","last_clean":"2023-06-10T21:33:16.871726+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:33:16.871726+0000","last_undegraded":"2023-06-10T21:33:16.871726+0000","last_fullsized":"2023-06-10T21:33:16.871726+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4229169176,"num_objects":64533,"num_object_clones":0,"num_object_copies":193599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5789,"num_objects_misplaced":3991,"num_objects_unfound":0,"num_objects_dirty":64533,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":104009,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":815,"num_bytes_recovered":53346304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12845920256,"data_stored":12845908040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99209,"ondisk_log_size":99209,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430637,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596464,"kb_used_data":1040040,"kb_used_omap":0,"kb_used_meta":556416,"kb_avail":92148688,"statfs":{"total":95995035648,"available":94360256512,"internally_reserved":0,"allocated":1065000960,"data_stored":1064438130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":569769984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.603,"5min":6.9740000000000002,"15min":8.7959999999999994},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":6.4779999999999998,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.2070000000000001,"5min":6.915,"15min":9.0050000000000008},"min":{"1min":0.26400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":6.3810000000000002,"5min":185.58000000000001,"15min":185.58000000000001},"last":2.6110000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":2.5800000000000001,"15min":3.0350000000000001},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":49.732999999999997,"5min":87.417000000000002,"15min":130.19399999999999},"last":3.1349999999999998},{"interface":"front","average":{"1min":4.883,"5min":2.7210000000000001,"15min":3.573},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":50.060000000000002,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.505}]},{"osd":2,"last update":"Sat Jun 10 21:33:23 2023","interfaces":[{"interface":"back","average":{"1min":11.731999999999999,"5min":5.4450000000000003,"15min":3.7770000000000001},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":172.33600000000001,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":11.782999999999999,"5min":5.3540000000000001,"15min":4.4139999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":172.185,"5min":172.185,"15min":212.13399999999999},"last":7.3769999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:33:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":2.8660000000000001,"15min":2.6219999999999999},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.9489999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":4.3920000000000003},{"interface":"front","average":{"1min":1.393,"5min":2.4969999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.31,"15min":0.31},"max":{"1min":5.7869999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":1.3}]},{"osd":4,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.442,"15min":0.75},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.41399999999999998,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.374},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.59499999999999997,"15min":1.091},"min":{"1min":0.28299999999999997,"5min":0.245,"15min":0.245},"max":{"1min":0.48699999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.53900000000000003,"15min":1.847},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52600000000000002,"5min":3.9729999999999999,"15min":211.99799999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.52400000000000002,"15min":1.7},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.29299999999999998},"max":{"1min":0.499,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.58899999999999997,"15min":1.1180000000000001},"min":{"1min":0.33800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":3.375,"15min":13.073},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.499,"15min":1.0069999999999999},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.53000000000000003,"5min":1.897,"15min":10.874000000000001},"last":0.443}]}]},{"osd":6,"up_from":0,"seq":180388626865,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566308,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4052176,"kb_used_data":3444936,"kb_used_omap":4,"kb_used_meta":606907,"kb_avail":89692976,"statfs":{"total":95995035648,"available":91845607424,"internally_reserved":0,"allocated":3527614464,"data_stored":3526670472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":621473678},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":9.0679999999999996,"15min":12.176},"min":{"1min":0.28199999999999997,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.20699999999999999},{"interface":"front","average":{"1min":2.0659999999999998,"5min":8.9860000000000007,"15min":11.875999999999999},"min":{"1min":0.22800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":10.244,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.84499999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:32:31 2023","interfaces":[{"interface":"back","average":{"1min":9.0399999999999991,"5min":8.0719999999999992,"15min":3.8290000000000002},"min":{"1min":0.27700000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":54.578000000000003,"5min":107.857,"15min":107.857},"last":0.34799999999999998},{"interface":"front","average":{"1min":7.8310000000000004,"5min":8.1389999999999993,"15min":3.54},"min":{"1min":0.40300000000000002,"5min":0.22800000000000001,"15min":0.218},"max":{"1min":54.140999999999998,"5min":107.958,"15min":107.958},"last":2.4239999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:07 2023","interfaces":[{"interface":"back","average":{"1min":7.6360000000000001,"5min":3.6960000000000002,"15min":2.5489999999999999},"min":{"1min":0.37,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":80.584999999999994,"5min":80.584999999999994,"15min":80.584999999999994},"last":5.819},{"interface":"front","average":{"1min":7.5880000000000001,"5min":4.8230000000000004,"15min":3.7290000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":80.873999999999995,"5min":80.873999999999995,"15min":80.873999999999995},"last":1.25}]},{"osd":4,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.4650000000000001,"15min":1.518},"min":{"1min":0.38,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.4260000000000002,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.536,"5min":1.4490000000000001,"15min":1.4039999999999999},"min":{"1min":0.28799999999999998,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":8.2739999999999991,"5min":8.2739999999999991,"15min":14.460000000000001},"last":0.81000000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.802,"15min":4.1050000000000004},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5499999999999998,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.472,"15min":3.7949999999999999},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.4820000000000002,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.48199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.893,"15min":2.012},"min":{"1min":0.38300000000000001,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":6.5750000000000002,"5min":8.8580000000000005,"15min":14.858000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.655,"5min":1.4850000000000001,"15min":1.698},"min":{"1min":0.47599999999999998,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.5140000000000002,"5min":11.476000000000001,"15min":13.48},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.5980000000000001,"15min":2.4199999999999999},"min":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":5.3769999999999998,"5min":16.113,"15min":162.399},"last":26.937999999999999},{"interface":"front","average":{"1min":2.9319999999999999,"5min":1.6819999999999999,"15min":2.1960000000000002},"min":{"1min":0.34899999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":162.27199999999999},"last":28.553000000000001}]}]},{"osd":0,"up_from":244,"seq":1047972020285,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2315104,"kb_used_data":2063128,"kb_used_omap":10,"kb_used_meta":251893,"kb_avail":91430048,"statfs":{"total":95995035648,"available":93624369152,"internally_reserved":0,"allocated":2112643072,"data_stored":2111623615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":257938484},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":10.356999999999999,"5min":5.2729999999999997,"15min":4.6970000000000001},"min":{"1min":0.26200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":139.68000000000001,"5min":139.68000000000001,"15min":139.68000000000001},"last":4.7160000000000002},{"interface":"front","average":{"1min":9.1170000000000009,"5min":5.5350000000000001,"15min":5.8860000000000001},"min":{"1min":0.22900000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":141.79499999999999,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":3.278,"5min":5.9630000000000001,"15min":5.9550000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":8.4440000000000008,"5min":205.10900000000001,"15min":205.10900000000001},"last":3.9239999999999999},{"interface":"front","average":{"1min":3.0419999999999998,"5min":5.9349999999999996,"15min":5.3819999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.8930000000000007,"5min":208.34299999999999,"15min":208.34299999999999},"last":2.4430000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:33:06 2023","interfaces":[{"interface":"back","average":{"1min":6.7779999999999996,"5min":5.0220000000000002,"15min":5.8650000000000002},"min":{"1min":0.245,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":111.745,"5min":111.745,"15min":111.745},"last":0.57699999999999996},{"interface":"front","average":{"1min":9.2940000000000005,"5min":6.0640000000000001,"15min":6.8769999999999998},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":114.205,"5min":114.205,"15min":114.205},"last":4.6319999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":3.0939999999999999,"15min":4.4290000000000003},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.645,"5min":29.709,"15min":29.709},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.8929999999999998,"15min":3.8140000000000001},"min":{"1min":0.35799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.7350000000000003,"5min":13.872,"15min":13.872},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":2.9470000000000001,"15min":3.9060000000000001},"min":{"1min":0.42799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":4.7859999999999996,"5min":14.105,"15min":14.105},"last":0.379},{"interface":"front","average":{"1min":1.5980000000000001,"5min":2.6720000000000002,"15min":3.2999999999999998},"min":{"1min":0.38500000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.6020000000000003,"5min":19.846,"15min":19.846},"last":2.6499999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":3.226,"15min":4.6200000000000001},"min":{"1min":0.442,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.605,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.571,"5min":2.98,"15min":3.9420000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.8140000000000001,"5min":13.82,"15min":13.82},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:32:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.8700000000000001,"15min":3.6819999999999999},"min":{"1min":0.46899999999999997,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.5609999999999999,"5min":19.866,"15min":19.866},"last":2.585},{"interface":"front","average":{"1min":1.143,"5min":3.2280000000000002,"15min":4.6189999999999998},"min":{"1min":0.44900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":3.5529999999999999,"5min":29.77,"15min":29.77},"last":0.51200000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075696,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3050504,"kb_used_data":2604992,"kb_used_omap":7,"kb_used_meta":445496,"kb_avail":90694648,"statfs":{"total":95995035648,"available":92871319552,"internally_reserved":0,"allocated":2667511808,"data_stored":2666575496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":456187985},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,3,1,1,0,0,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3410000000000002,"5min":4.2190000000000003,"15min":4.5419999999999998},"min":{"1min":0.30299999999999999,"5min":0.222,"15min":0.222},"max":{"1min":12.406000000000001,"5min":66.784000000000006,"15min":66.784000000000006},"last":0.20300000000000001},{"interface":"front","average":{"1min":2.657,"5min":4.9100000000000001,"15min":5.2309999999999999},"min":{"1min":0.26100000000000001,"5min":0.252,"15min":0.252},"max":{"1min":10.458,"5min":67.251999999999995,"15min":67.251999999999995},"last":2.0169999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":11.742000000000001,"5min":4.3499999999999996,"15min":2.5070000000000001},"min":{"1min":0.41399999999999998,"5min":0.19,"15min":0.19},"max":{"1min":147.20099999999999,"5min":147.20099999999999,"15min":147.20099999999999},"last":3.081},{"interface":"front","average":{"1min":11.183999999999999,"5min":4.0140000000000002,"15min":2.2930000000000001},"min":{"1min":0.36599999999999999,"5min":0.214,"15min":0.214},"max":{"1min":147.81299999999999,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:33:01 2023","interfaces":[{"interface":"back","average":{"1min":10.353999999999999,"5min":4.8970000000000002,"15min":5.1260000000000003},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":138.03299999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":10.279,"5min":5.444,"15min":6.0099999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":138.095,"5min":138.095,"15min":138.095},"last":2.2010000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.9649999999999999,"15min":2.6880000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.28999999999999998},"max":{"1min":5.2729999999999997,"5min":110.337,"15min":224.00999999999999},"last":2.081},{"interface":"front","average":{"1min":1.9450000000000001,"5min":3.3100000000000001,"15min":2.8679999999999999},"min":{"1min":0.439,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.1470000000000002,"5min":110.2,"15min":223.28399999999999},"last":2.2210000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.5049999999999999,"15min":1.3600000000000001},"min":{"1min":0.379,"5min":0.32900000000000001,"15min":0.29699999999999999},"max":{"1min":2.6259999999999999,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.708,"15min":1.6040000000000001},"min":{"1min":0.39300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.1799999999999997,"5min":16.125,"15min":16.125},"last":2.121}]},{"osd":6,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.9530000000000001,"15min":1.6759999999999999},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":5.4640000000000004,"5min":16.183,"15min":16.183},"last":2.1579999999999999},{"interface":"front","average":{"1min":1.9810000000000001,"5min":2.2250000000000001,"15min":1.8680000000000001},"min":{"1min":0.52400000000000002,"5min":0.34999999999999998,"15min":0.33600000000000002},"max":{"1min":5.0650000000000004,"5min":11.029,"15min":17.244},"last":2.1760000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.786,"15min":1.7529999999999999},"min":{"1min":0.443,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.669,"5min":113.913,"15min":113.913},"last":73.445999999999998},{"interface":"front","average":{"1min":1.3779999999999999,"5min":3.125,"15min":2.1000000000000001},"min":{"1min":0.47499999999999998,"5min":0.42099999999999999,"15min":0.41399999999999998},"max":{"1min":5.1390000000000002,"5min":113.937,"15min":113.937},"last":73.475999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824056,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2635412,"kb_used_data":2407308,"kb_used_omap":7,"kb_used_meta":228088,"kb_avail":91109740,"statfs":{"total":95995035648,"available":93296373760,"internally_reserved":0,"allocated":2465083392,"data_stored":2464075441,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":233562131},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,3,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":4.2039999999999997,"15min":5.0999999999999996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.522,"5min":93.992999999999995,"15min":93.992999999999995},"last":8.4450000000000003},{"interface":"front","average":{"1min":1.387,"5min":3.891,"15min":4.8520000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":5.7729999999999997,"5min":95.623999999999995,"15min":95.623999999999995},"last":10.358000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":3.7759999999999998,"15min":3.6840000000000002},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.422999999999998,"5min":19.859000000000002,"15min":19.859000000000002},"last":2.5350000000000001},{"interface":"front","average":{"1min":5.2939999999999996,"5min":3.028,"15min":3.3319999999999999},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.384,"5min":17.384,"15min":17.384},"last":5.9779999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":6.6130000000000004,"15min":10.819000000000001},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.9580000000000002,"5min":87.555999999999997,"15min":87.555999999999997},"last":10.324},{"interface":"front","average":{"1min":4.806,"5min":8.3859999999999992,"15min":12.356999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.170999999999999,"5min":80.566999999999993,"15min":80.566999999999993},"last":12.541}]},{"osd":4,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.4430000000000001,"15min":1.397},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.6070000000000002,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":1.645},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.7570000000000001,"15min":2.7450000000000001},"min":{"1min":0.40100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.384,"5min":19.728000000000002,"15min":19.728000000000002},"last":1.6830000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2010000000000001,"15min":1.329},"min":{"1min":0.39400000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":5.71},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.4610000000000001,"15min":1.4399999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6419999999999999,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":1.611}]},{"osd":6,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7810000000000001,"15min":2.7690000000000001},"min":{"1min":0.438,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":14.119,"5min":19.805,"15min":19.805},"last":1.754},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.2689999999999999,"15min":1.383},"min":{"1min":0.47999999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9809999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":5.6550000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.7919999999999998,"15min":2.782},"min":{"1min":0.503,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":14.422000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":1.7789999999999999},{"interface":"front","average":{"1min":1.524,"5min":1.2450000000000001,"15min":1.357},"min":{"1min":0.44900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9379999999999997,"5min":6.9379999999999997,"15min":6.9379999999999997},"last":5.617}]}]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633644,"kb_used_data":1041188,"kb_used_omap":0,"kb_used_meta":592448,"kb_avail":92111508,"statfs":{"total":95995035648,"available":94322184192,"internally_reserved":0,"allocated":1066176512,"data_stored":1065616250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606666752},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":5.0650000000000004,"15min":8.8070000000000004},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.2489999999999997,"5min":144.22399999999999,"15min":144.22399999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.484,"5min":5.6269999999999998,"15min":8.5020000000000007},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3120000000000003,"5min":144.303,"15min":144.303},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":1.9470000000000001,"15min":2.9390000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":13.461,"5min":13.461,"15min":182.035},"last":1.27},{"interface":"front","average":{"1min":1.833,"5min":1.7549999999999999,"15min":2.6800000000000002},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.255000000000001,"5min":10.255000000000001,"15min":181.73099999999999},"last":2.492}]},{"osd":2,"last update":"Sat Jun 10 21:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":1.9510000000000001,"15min":1.554},"min":{"1min":0.34699999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":11.116,"5min":11.116,"15min":22.614000000000001},"last":8.1709999999999994},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.7809999999999999,"15min":1.5309999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.798,"5min":18.329999999999998,"15min":22.326000000000001},"last":1.615}]},{"osd":3,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.125,"15min":2.3029999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":29.856999999999999,"5min":29.856999999999999,"15min":29.856999999999999},"last":1.173},{"interface":"front","average":{"1min":10.365,"5min":3.5270000000000001,"15min":2.5379999999999998},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":160.06200000000001,"5min":160.06200000000001,"15min":160.06200000000001},"last":0.38200000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.64100000000000001,"15min":1.0820000000000001},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.86299999999999999,"5min":6.742,"15min":13.941000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.72999999999999998,"15min":1.107},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":0.61199999999999999,"5min":6.0819999999999999,"15min":11.904999999999999},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:32:54 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.58399999999999996,"15min":1.0800000000000001},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.76900000000000002,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":0.438},{"interface":"front","average":{"1min":0.439,"5min":0.75900000000000001,"15min":1.143},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.72399999999999998,"5min":8.3109999999999999,"15min":13.845000000000001},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.64900000000000002,"15min":1.2549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.51100000000000001,"5min":6.798,"15min":16.265000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.44,"5min":0.64700000000000002,"15min":1.175},"min":{"1min":0.316,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.627,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":0.41199999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823093,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2106720256,"data_stored":2106720256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3500871680,"data_stored":3500867608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2658996224,"data_stored":2658992152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2460745728,"data_stored":2460745728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1059262464,"data_stored":1059258392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1059323904,"data_stored":1059323904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:33:36.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:33:36.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:33:36.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:36.736 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:37.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:37.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:37.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:37.095 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:37 smithi064 ceph-mon[180297]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 5798/190422 objects degraded (3.045%); 3847/190422 objects misplaced (2.020%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:37.096 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:37 smithi064 ceph-mon[186706]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 5798/190422 objects degraded (3.045%); 3847/190422 objects misplaced (2.020%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:37.146 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:37.181 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:37.280 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:37.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:37.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:37.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:37 smithi152 ceph-mon[156819]: pgmap v1064: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 517 op/s; 5798/190422 objects degraded (3.045%); 3847/190422 objects misplaced (2.020%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:37.414 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:37.632 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:37.632 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:37.835 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:37.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:37.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:37.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:38.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:38.163 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:38.218 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:38.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:38.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:38.318 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:38.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:38.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:33:38.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:38.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:38 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:38 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:38 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:38 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:38 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:38 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:38.832 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:38.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:38.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:38.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:39.008 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:33:39.012 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:39.057 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:33:39.118 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:33:39.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:33:39.231 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:33:39.279 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:33:39.284 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:39.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:39.341 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:33:39.385 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:33:39.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:39.404 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.404 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.404 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.405 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.405 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.405 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.406 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:33:39.501 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:39.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:39 smithi064 ceph-mon[186706]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 5798/187347 objects degraded (3.095%); 3847/187347 objects misplaced (2.053%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:39.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2742245858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:39.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:39 smithi064 ceph-mon[180297]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 5798/187347 objects degraded (3.095%); 3847/187347 objects misplaced (2.053%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:39.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2742245858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:39.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:39 smithi152 ceph-mon[156819]: pgmap v1065: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 526 op/s; 5798/187347 objects degraded (3.095%); 3847/187347 objects misplaced (2.053%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2742245858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.452 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166812 9.0 9:0ada01af:::benchmark_data_smithi064_247471_object57824:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 595.12421852299997, 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0161496670000001, 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.453 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166812 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:33:40.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322919+0000", 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:40.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322918+0000", 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9450000000000004e-06 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322924+0000", 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0769999999999997e-06 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322934+0000", 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5289999999999997e-06 2023-06-10T21:33:40.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322940+0000", 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7850999999999995e-05 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323038+0000", 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0801e-05 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323049+0000", 2023-06-10T21:33:40.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0047669809999999 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.327816+0000", 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023938549999999999 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.330210+0000", 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000173513 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.330383+0000", 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030409370000000001 2023-06-10T21:33:40.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333424+0000", 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054907360000000004 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.338915+0000", 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9415e-05 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.338964+0000", 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9045000000000001e-05 2023-06-10T21:33:40.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339023+0000", 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5683000000000003e-05 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339069+0000", 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0161496670000001 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.465 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166815 9.0 9:03235a20:::benchmark_data_smithi064_247471_object57827:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e157)", 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 595.12418418499999, 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0162916630000001, 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.466 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166815 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:33:40.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322954+0000", 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322953+0000", 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7860000000000001e-06 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322956+0000", 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8239999999999996e-06 2023-06-10T21:33:40.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322962+0000", 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2969999999999997e-06 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.322966+0000", 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111656 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323078+0000", 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5869999999999998e-06 2023-06-10T21:33:40.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.323084+0000", 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0103796089999999 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333464+0000", 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0339000000000004e-05 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333544+0000", 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9686999999999996e-05 2023-06-10T21:33:40.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.333644+0000", 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054681870000000002 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339112+0000", 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7512000000000003e-05 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339150+0000", 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8546000000000001e-05 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339178+0000", 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3045e-05 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339211+0000", 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4028e-05 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339245+0000", 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0162916630000001 2023-06-10T21:33:40.477 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166826 9.0 9:04443790:::benchmark_data_smithi064_247471_object57838:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 595.11250334700003, 2023-06-10T21:33:40.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014370735, 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166826 2023-06-10T21:33:40.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334634+0000", 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334633+0000", 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2560000000000008e-06 2023-06-10T21:33:40.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334642+0000", 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6484e-05 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334659+0000", 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9730000000000002e-06 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334665+0000", 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8026999999999999e-05 2023-06-10T21:33:40.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334723+0000", 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4628e-05 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.334737+0000", 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0014236080000001 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336161+0000", 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00050002300000000005 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336661+0000", 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105979 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.336767+0000", 2023-06-10T21:33:40.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024868210000000002 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339254+0000", 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096393050000000008 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348893+0000", 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5008e-05 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348938+0000", 2023-06-10T21:33:40.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.243e-05 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348970+0000", 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4765999999999999e-05 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349005+0000", 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014370735 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166843 9.0 9:06ac37a2:::benchmark_data_smithi064_247471_object57855:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 595.083554316, 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98562871200000002, 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.491 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166843 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:33:40.492 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363583+0000", 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363582+0000", 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3459999999999999e-06 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363588+0000", 2023-06-10T21:33:40.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7246000000000001e-05 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363606+0000", 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4109999999999999e-06 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363613+0000", 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2362999999999999e-05 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363675+0000", 2023-06-10T21:33:40.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1939e-05 2023-06-10T21:33:40.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.363687+0000", 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97560709499999998 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339295+0000", 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0881000000000001e-05 2023-06-10T21:33:40.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339365+0000", 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7531000000000001e-05 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.499 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.339433+0000", 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094019989999999994 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.348835+0000", 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022852899999999999 2023-06-10T21:33:40.500 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349063+0000", 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0859000000000001e-05 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349124+0000", 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1295999999999998e-05 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349176+0000", 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6529000000000002e-05 2023-06-10T21:33:40.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349212+0000", 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98562871200000002 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166894 9.0 9:058fc9da:::benchmark_data_smithi064_247471_object57906:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 594.87694942400003, 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78374433300000002, 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.504 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166894 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570188+0000", 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:40.506 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570186+0000", 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.789e-06 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570195+0000", 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9400000000000001e-05 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570215+0000", 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5470000000000002e-06 2023-06-10T21:33:40.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570221+0000", 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2519000000000003e-05 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570274+0000", 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8919999999999995e-06 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.570284+0000", 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77249051099999999 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.342774+0000", 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013168899999999999 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.342906+0000", 2023-06-10T21:33:40.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000158128 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.343064+0000", 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059511130000000001 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349015+0000", 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041870379999999997 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353202+0000", 2023-06-10T21:33:40.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062741800000000003 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353829+0000", 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9527000000000001e-05 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353889+0000", 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3646999999999997e-05 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353933+0000", 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.78374433300000002 2023-06-10T21:33:40.515 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:166905 9.0 9:01f9f1d0:::benchmark_data_smithi064_247471_object57917:head [delete] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 594.85582568100006, 2023-06-10T21:33:40.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76680837999999996, 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166905 2023-06-10T21:33:40.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591312+0000", 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591311+0000", 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3470000000000004e-06 2023-06-10T21:33:40.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591317+0000", 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1277000000000001e-05 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591328+0000", 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1000000000000003e-06 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591333+0000", 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1403999999999999e-05 2023-06-10T21:33:40.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591375+0000", 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.636e-06 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:45.591383+0000", 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75494583900000001 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346329+0000", 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6804000000000005e-05 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346406+0000", 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8774000000000002e-05 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.346485+0000", 2023-06-10T21:33:40.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027402619999999998 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.349225+0000", 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047691249999999999 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.353994+0000", 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040310190000000003 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358025+0000", 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9750000000000003e-05 2023-06-10T21:33:40.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358075+0000", 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5550999999999998e-05 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:23:46.358120+0000", 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.76680837999999996 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.528 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35400.0:199852 9.0 9:0219fd3c:::benchmark_data_smithi064_247471_object90864:head [delete] snapc 0=[] ondisk+write+known_if_redirected e171)", 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 538.91454718099999, 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30757901700000001, 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35400", 2023-06-10T21:33:40.529 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1486171525", 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199852 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:33:40.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532591+0000", 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532589+0000", 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2009999999999999e-06 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532597+0000", 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5182e-05 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532612+0000", 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5780000000000004e-06 2023-06-10T21:33:40.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532619+0000", 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8724999999999999e-05 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532658+0000", 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118538 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532776+0000", 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119059 2023-06-10T21:33:40.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.532895+0000", 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041169240000000001 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.537012+0000", 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013852160000000001 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.550864+0000", 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.28918148999999999 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840046+0000", 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6958000000000004e-05 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840123+0000", 2023-06-10T21:33:40.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6752000000000002e-05 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:24:41.840170+0000", 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.30757901700000001 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.539 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13226 10.a 10:57a745d6:::benchmark_data_smithi064_312713_object13225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.19776412499999, 2023-06-10T21:33:40.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.416610181, 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13226 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.541 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249374+0000", 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249372+0000", 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015133799999999999 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249524+0000", 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0832000000000001e-05 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249534+0000", 2023-06-10T21:33:40.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6910000000000002e-06 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249540+0000", 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000273119 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249813+0000", 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114503 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.249928+0000", 2023-06-10T21:33:40.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014703399999999999 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.250075+0000", 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00083144899999999995 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.250906+0000", 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00097972100000000002 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.251886+0000", 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014250977999999999 2023-06-10T21:33:40.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266137+0000", 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0433e-05 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266147+0000", 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36071926900000001 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.626867+0000", 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1214e-05 2023-06-10T21:33:40.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.626878+0000", 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0389994629999999 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665877+0000", 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106587 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665984+0000", 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.416610181 2023-06-10T21:33:40.552 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13230 10.a 10:56398bc8:::benchmark_data_smithi064_312713_object13229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.19456993799997, 2023-06-10T21:33:40.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.413825726, 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13230 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.554 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252568+0000", 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252567+0000", 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8826999999999999e-05 2023-06-10T21:33:40.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252606+0000", 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3930000000000008e-06 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252614+0000", 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5149999999999997e-06 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252620+0000", 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0380000000000003e-05 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252660+0000", 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102132 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252762+0000", 2023-06-10T21:33:40.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015171800000000001 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.252914+0000", 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031348970000000002 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.256049+0000", 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001809043 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.257858+0000", 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0083186690000000008 2023-06-10T21:33:40.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266177+0000", 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1099999999999997e-06 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266184+0000", 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36171270300000002 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627896+0000", 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.901e-06 2023-06-10T21:33:40.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627905+0000", 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0384316309999999 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666337+0000", 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6521999999999999e-05 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666394+0000", 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.413825726 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.565 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13234 10.a 10:5db0cd49:::benchmark_data_smithi064_312713_object13233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.19211921900001, 2023-06-10T21:33:40.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4210149110000001, 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13234 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.567 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255019+0000", 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255017+0000", 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8182000000000001e-05 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255055+0000", 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9320000000000008e-06 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255065+0000", 2023-06-10T21:33:40.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.468e-06 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255070+0000", 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4666999999999998e-05 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255115+0000", 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012085 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255236+0000", 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000144413 2023-06-10T21:33:40.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,5", 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.255380+0000", 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00088783000000000004 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.256268+0000", 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017292430000000001 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.257997+0000", 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082125259999999995 2023-06-10T21:33:40.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266210+0000", 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8719999999999996e-06 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266217+0000", 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36172231999999999 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627939+0000", 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3130000000000001e-06 2023-06-10T21:33:40.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627947+0000", 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0384744699999999 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666421+0000", 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00080771 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667229+0000", 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087343360000000005 2023-06-10T21:33:40.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.675963+0000", 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0401999999999994e-05 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676033+0000", 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4210149110000001 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13237 10.a 10:52604c11:::benchmark_data_smithi064_312713_object13236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e200)", 2023-06-10T21:33:40.580 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.18843804199997, 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.417774082, 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.581 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13237 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258700+0000", 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258699+0000", 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2926999999999998e-05 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258732+0000", 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4180000000000003e-06 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258741+0000", 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9060000000000001e-06 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.258746+0000", 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074915210000000001 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266238+0000", 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2489999999999998e-06 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.266244+0000", 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.36174977400000002 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627994+0000", 2023-06-10T21:33:40.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9409999999999996e-06 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628003+0000", 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0392761150000001 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667279+0000", 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118165 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667397+0000", 2023-06-10T21:33:40.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121712 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667519+0000", 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002026927 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669546+0000", 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053423530000000002 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674888+0000", 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0011963340000000001 2023-06-10T21:33:40.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676084+0000", 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.015e-05 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676125+0000", 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034917100000000001 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676474+0000", 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.417774082 2023-06-10T21:33:40.593 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13245 10.a 10:577fa6ae:::benchmark_data_smithi064_312713_object13244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.17906108099999, 2023-06-10T21:33:40.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4089718099999999, 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13245 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.595 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268077+0000", 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268076+0000", 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0794999999999998e-05 2023-06-10T21:33:40.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268116+0000", 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.151e-05 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268128+0000", 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5259999999999999e-06 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.268134+0000", 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00186199 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.269996+0000", 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9990000000000003e-06 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.270006+0000", 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35802179899999997 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628028+0000", 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9039999999999992e-06 2023-06-10T21:33:40.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628036+0000", 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.040087349 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668124+0000", 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014886000000000001 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668272+0000", 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021805899999999999 2023-06-10T21:33:40.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668490+0000", 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001063549 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669554+0000", 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053829209999999997 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674937+0000", 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001595632 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676533+0000", 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8981000000000003e-05 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.676592+0000", 2023-06-10T21:33:40.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045696000000000002 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677048+0000", 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4089718099999999 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.607 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13248 10.7 10:e5a76f8f:::benchmark_data_smithi064_312713_object13247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.175271614, 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3988524149999999, 2023-06-10T21:33:40.608 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13248 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271866+0000", 2023-06-10T21:33:40.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271864+0000", 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104888 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271969+0000", 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8029999999999998e-05 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271987+0000", 2023-06-10T21:33:40.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8689999999999998e-06 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.271994+0000", 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031337380000000001 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.275128+0000", 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1113e-05 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.275139+0000", 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.35235471299999999 2023-06-10T21:33:40.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627494+0000", 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2689e-05 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.627506+0000", 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.036269895 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.663776+0000", 2023-06-10T21:33:40.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012765699999999999 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.663904+0000", 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000138367 2023-06-10T21:33:40.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.664042+0000", 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035549879999999998 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667597+0000", 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120649 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667718+0000", 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023842529999999998 2023-06-10T21:33:40.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670102+0000", 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4106000000000001e-05 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670146+0000", 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00057218899999999997 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670719+0000", 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3988524149999999 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.621 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13271 10.a 10:5728c7b4:::benchmark_data_smithi064_312713_object13270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.160593818, 2023-06-10T21:33:40.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3910280420000001, 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.623 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13271 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286544+0000", 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286543+0000", 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.872e-05 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286602+0000", 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3389999999999993e-06 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286610+0000", 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8359999999999999e-06 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286615+0000", 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2231999999999996e-05 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286677+0000", 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3670000000000003e-06 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.286686+0000", 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.343093494 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.629780+0000", 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6810000000000004e-06 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.629789+0000", 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0391585459999999 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.668947+0000", 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012422299999999999 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669071+0000", 2023-06-10T21:33:40.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121916 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669193+0000", 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053272599999999997 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.669726+0000", 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052600779999999996 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674986+0000", 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002138872 2023-06-10T21:33:40.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677125+0000", 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4203999999999998e-05 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677179+0000", 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039273699999999999 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.677572+0000", 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3910280420000001 2023-06-10T21:33:40.635 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13274 10.7 10:e090e432:::benchmark_data_smithi064_312713_object13273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.15810778500003, 2023-06-10T21:33:40.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.384448001, 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13274 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.637 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289030+0000", 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6611999999999999e-05 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289056+0000", 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4040000000000003e-06 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289062+0000", 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1099999999999999e-06 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289065+0000", 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7024999999999997e-05 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289122+0000", 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0210000000000001e-05 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.289132+0000", 2023-06-10T21:33:40.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33959756699999999 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628729+0000", 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9420000000000007e-06 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628739+0000", 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0367496140000001 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665489+0000", 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016709400000000001 2023-06-10T21:33:40.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.665656+0000", 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037900500000000002 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:33:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666035+0000", 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016913609999999999 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667726+0000", 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030370800000000002 2023-06-10T21:33:40.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670764+0000", 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022778709999999999 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673041+0000", 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9855999999999997e-05 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673101+0000", 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000376707 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.673478+0000", 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.384448001 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:13276 10.7 10:e43325ff:::benchmark_data_smithi064_312713_object13275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 475.15551137599999, 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.383298761, 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:40.650 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 13276 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:40.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291626+0000", 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291625+0000", 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8028000000000007e-05 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291693+0000", 2023-06-10T21:33:40.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5150999999999999e-05 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291708+0000", 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3890000000000002e-06 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291715+0000", 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0940000000000003e-05 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291775+0000", 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3091000000000001e-05 2023-06-10T21:33:40.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.291789+0000", 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33697318300000001 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628762+0000", 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2109999999999998e-06 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:45.628770+0000", 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.037982368 2023-06-10T21:33:40.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666752+0000", 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013466800000000001 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:40.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.666887+0000", 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019130800000000001 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667078+0000", 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065763300000000002 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.667736+0000", 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030568829999999998 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.670793+0000", 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036619259999999998 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674455+0000", 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6567000000000002e-05 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674511+0000", 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041383799999999998 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:25:46.674925+0000", 2023-06-10T21:33:40.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.383298761 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:40.663 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:40.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:40.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:40.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:40.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:40.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:41.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:33:41.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:41 smithi152 ceph-mon[156819]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5847/182628 objects degraded (3.202%); 3679/182628 objects misplaced (2.014%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5847/182628 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:41.566 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:41.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:41 smithi064 ceph-mon[186706]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5847/182628 objects degraded (3.202%); 3679/182628 objects misplaced (2.014%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:41.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5847/182628 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:41 smithi064 ceph-mon[180297]: pgmap v1066: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5847/182628 objects degraded (3.202%); 3679/182628 objects misplaced (2.014%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5847/182628 objects degraded (3.202%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:42.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:42.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:42.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:42.434 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:42.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:42.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:42.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:42.866 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:42.968 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:42.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:43.020 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:42] "GET /metrics HTTP/1.1" 200 224058 "" "Prometheus/2.33.4" 2023-06-10T21:33:43.073 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:43.156 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110204 11.4 11:239c8bb2:::benchmark_data_smithi064_385725_object25931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.149920+0000", 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.003486804, 2023-06-10T21:33:43.157 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035232039999999998, 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.158 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110204 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149920+0000", 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.159 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149920+0000", 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.160 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149918+0000", 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4240000000000004e-06 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149926+0000", 2023-06-10T21:33:43.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4092e-05 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149940+0000", 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9580000000000001e-06 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.149947+0000", 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5661999999999997e-05 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.150003+0000", 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6008999999999995e-05 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.150099+0000", 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1138999999999996e-05 2023-06-10T21:33:43.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.150190+0000", 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026901519999999999 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.152880+0000", 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029599449999999999 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36159.0:110194 11.f e297/269)", 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.153476+0000", 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9999309, 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7358999999999999e-05, 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153476+0000", 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.168 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153476+0000", 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153475+0000", 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1389999999999998e-06 2023-06-10T21:33:43.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153482+0000", 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4575e-05 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153496+0000", 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.135e-06 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153501+0000", 2023-06-10T21:33:43.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.584e-05 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36159.0:110199 11.f e297/269)", 2023-06-10T21:33:43.172 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.153522+0000", 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9998846, 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8982e-05, 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153522+0000", 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153522+0000", 2023-06-10T21:33:43.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153522+0000", 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7329999999999999e-06 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153525+0000", 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7169999999999999e-06 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153529+0000", 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7199999999999998e-06 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153532+0000", 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8670000000000006e-06 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.177 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.36159.0:110195 11.e e297/269 ondisk, result = 0)", 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.153535+0000", 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4294967295.9998717, 2023-06-10T21:33:43.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8494999999999998e-05, 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153535+0000", 2023-06-10T21:33:43.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153535+0000", 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153535+0000", 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.407e-06 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153536+0000", 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8320000000000002e-06 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153539+0000", 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3329999999999999e-06 2023-06-10T21:33:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153542+0000", 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.413e-05 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153576+0000", 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6469999999999998e-06 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153583+0000", 2023-06-10T21:33:43.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8219e-05 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110194 11.f 11:f9667e1a:::benchmark_data_smithi064_385725_object25921:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.134785+0000", 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018621399, 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018923175, 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.186 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110194 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134785+0000", 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134785+0000", 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:43.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134784+0000", 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0514e-05 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134794+0000", 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3774999999999999e-05 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134808+0000", 2023-06-10T21:33:43.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.84e-06 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134814+0000", 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0275000000000001e-05 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134874+0000", 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000131712 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.135006+0000", 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014348099999999999 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.135149+0000", 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010488539 2023-06-10T21:33:43.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.145638+0000", 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079347270000000008 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153573+0000", 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018787333999999999 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110197 11.4 11:2ac97eee:::benchmark_data_smithi064_385725_object25924:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.134838+0000", 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018568833, 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018957841, 2023-06-10T21:33:43.196 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110197 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134838+0000", 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134838+0000", 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134838+0000", 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8279999999999999e-06 2023-06-10T21:33:43.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134839+0000", 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9919999999999999e-06 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134842+0000", 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.371e-06 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134846+0000", 2023-06-10T21:33:43.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0590999999999997e-05 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134896+0000", 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010126 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134998+0000", 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2928000000000004e-05 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.135071+0000", 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077224829999999996 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.142793+0000", 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079551740000000006 2023-06-10T21:33:43.204 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110198 11.4 11:2939867b:::benchmark_data_smithi064_385725_object25925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.134859+0000", 2023-06-10T21:33:43.205 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018547561000000001, 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019017253000000001, 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.206 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110198 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134859+0000", 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134859+0000", 2023-06-10T21:33:43.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:43.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134859+0000", 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.734e-06 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134861+0000", 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5040000000000002e-06 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134867+0000", 2023-06-10T21:33:43.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.687e-06 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134871+0000", 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036894179999999999 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.138560+0000", 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116981 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.138677+0000", 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011845600000000001 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.138796+0000", 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040070590000000003 2023-06-10T21:33:43.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.142803+0000", 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079434710000000006 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.214 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110199 11.f 11:f2ab91ca:::benchmark_data_smithi064_385725_object25926:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.134875+0000", 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018531381, 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019084073999999999, 2023-06-10T21:33:43.215 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110199 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.216 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134875+0000", 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134875+0000", 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134875+0000", 2023-06-10T21:33:43.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0189999999999998e-06 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134878+0000", 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.49e-06 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134883+0000", 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9530000000000001e-06 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134887+0000", 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039143600000000004 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.138802+0000", 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5402000000000006e-05 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.138897+0000", 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108509 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.139006+0000", 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066382589999999997 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.145644+0000", 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079881659999999997 2023-06-10T21:33:43.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153632+0000", 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018756748 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:110202 11.f 11:f775a9bb:::benchmark_data_smithi064_385725_object25929:head [delete] snapc 0=[] ondisk+write+known_if_redirected e297)", 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:43.134895+0000", 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.018512061999999999, 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019159447, 2023-06-10T21:33:43.225 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 110202 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.226 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134895+0000", 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134895+0000", 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134894+0000", 2023-06-10T21:33:43.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0047999999999999e-05 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134904+0000", 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:43.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134911+0000", 2023-06-10T21:33:43.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6179999999999997e-06 2023-06-10T21:33:43.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:43.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.134915+0000", 2023-06-10T21:33:43.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070940259999999998 2023-06-10T21:33:43.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:43.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.142009+0000", 2023-06-10T21:33:43.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104817 2023-06-10T21:33:43.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.142114+0000", 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113828 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.142228+0000", 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035358149999999999 2023-06-10T21:33:43.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.145764+0000", 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078930349999999996 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:43.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:43.153657+0000", 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018762078000000001 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:33:43.237 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 9 2023-06-10T21:33:43.238 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:43.238 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:43.549 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:43.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:43 smithi064 ceph-mon[180297]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5847/180009 objects degraded (3.248%); 3679/180009 objects misplaced (2.044%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:43.559 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:43 smithi064 ceph-mon[186706]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5847/180009 objects degraded (3.248%); 3679/180009 objects misplaced (2.044%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:43.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:43.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:43 smithi152 ceph-mon[156819]: pgmap v1067: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5847/180009 objects degraded (3.248%); 3679/180009 objects misplaced (2.044%); 1.6 MiB/s, 25 objects/s recovering 2023-06-10T21:33:43.762 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:43.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:33:44.069 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:44.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:44.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:44.293 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:44.310 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:33:44.310 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5847/180009 objects degraded (3.248%), 2 pgs degraded","count":2},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1497,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":297,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":60003,"data_bytes":3931314086,"bytes_used":14793924608,"bytes_avail":561176289280,"bytes_total":575970213888,"degraded_objects":5847,"degraded_total":180009,"degraded_ratio":0.032481709247870937,"misplaced_objects":3679,"misplaced_total":180009,"misplaced_ratio":0.020437866995539112,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1664137,"recovering_keys_per_sec":0,"num_objects_recovered":254,"num_bytes_recovered":16646144,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":503},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":24,"modified":"2023-06-10T21:29:57.688996+0000","services":{}},"progress_events":{"08ad03ac-ce84-41c1-8854-b18550dc6087":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:33:44.311 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:44.415 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:44.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:44.525 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:33:44.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:33:44.538 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:44.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:44.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3349651628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:33:44.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:44.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3349651628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:33:44.711 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:44.853 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:33:44.854 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3349651628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:33:45.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 5847/178479 objects degraded (3.276%); 3679/178479 objects misplaced (2.061%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:33:45.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:33:45.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:45.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: osdmap e298: 8 total, 8 up, 6 in 2023-06-10T21:33:45.788 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.789 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.789 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.789 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:45 smithi152 ceph-mon[156819]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 5847/178479 objects degraded (3.276%); 3679/178479 objects misplaced (2.061%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:33:45.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:33:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: osdmap e298: 8 total, 8 up, 6 in 2023-06-10T21:33:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[186706]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: pgmap v1068: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 5847/178479 objects degraded (3.276%); 3679/178479 objects misplaced (2.061%); 1.5 MiB/s, 23 objects/s recovering 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2438394504' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: osdmap e298: 8 total, 8 up, 6 in 2023-06-10T21:33:45.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:45 smithi064 ceph-mon[180297]: failed to encode map e298 with expected crc 2023-06-10T21:33:45.996 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:45.996 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:33:45.997 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:33:45.997 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:33:45.997 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5847/178479 objects degraded (3.276%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5847/178479 objects degraded (3.276%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:46.762 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5847/178479 objects degraded (3.276%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:46.780 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:33:47.111 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:33:47.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:33:47.133 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:47.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:47.193 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:33:47.241 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:47.390 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:47.543 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:47.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:47.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:47.593 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:47.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[180297]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5861/175137 objects degraded (3.347%); 3508/175137 objects misplaced (2.003%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[186706]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5861/175137 objects degraded (3.347%); 3508/175137 objects misplaced (2.003%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:47.692 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:47 smithi152 ceph-mon[156819]: pgmap v1070: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 509 op/s; 5861/175137 objects degraded (3.347%); 3508/175137 objects misplaced (2.003%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:47.692 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:47.695 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:47.803 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:47.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:47.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:47.935 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:47.977 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:47.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:48.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:48.109 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:48.283 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:48.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:48.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:48.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:48.478 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:48.837 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:33:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95683 10.1 10:82260f40:::benchmark_data_smithi064_312713_object4927:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:33:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 391.38309318199998, 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604, 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95683 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:33:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449476+0000", 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449475+0000", 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3629999999999998e-06 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449478+0000", 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4200000000000007e-06 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449487+0000", 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8169999999999998e-06 2023-06-10T21:33:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.449492+0000", 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034418460000000001 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.452934+0000", 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123182 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453057+0000", 2023-06-10T21:33:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012961900000000001 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453186+0000", 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058584609999999997 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459045+0000", 2023-06-10T21:33:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6512e-05 2023-06-10T21:33:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459061+0000", 2023-06-10T21:33:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32578302999999997 2023-06-10T21:33:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784844+0000", 2023-06-10T21:33:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3819000000000001e-05 2023-06-10T21:33:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784898+0000", 2023-06-10T21:33:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2079999999999997e-05 2023-06-10T21:33:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784940+0000", 2023-06-10T21:33:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.335464604 2023-06-10T21:33:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:95687 10.1 10:884db9c7:::benchmark_data_smithi064_312713_object4931:head [delete] snapc 0=[] ondisk+write+known_if_redirected e227)", 2023-06-10T21:33:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:33:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 391.37952678900001, 2023-06-10T21:33:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002, 2023-06-10T21:33:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 95687 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453042+0000", 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453041+0000", 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5729999999999999e-06 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453045+0000", 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.665e-06 2023-06-10T21:33:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453051+0000", 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.253e-06 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.453054+0000", 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027574850000000001 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455811+0000", 2023-06-10T21:33:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2564999999999994e-05 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455884+0000", 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6180999999999994e-05 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.455960+0000", 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031903819999999999 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.459151+0000", 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015862529999999999 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.460737+0000", 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32425166599999999 2023-06-10T21:33:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.784988+0000", 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6248999999999997e-05 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785035+0000", 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8260000000000003e-05 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:17.785073+0000", 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33203093900000002 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111471 10.1 10:85cc9677:::benchmark_data_smithi064_312713_object20715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 367.542984139, 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999, 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111471 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289583+0000", 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0890000000000002e-06 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289590+0000", 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6314e-05 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289607+0000", 2023-06-10T21:33:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.267e-06 2023-06-10T21:33:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289614+0000", 2023-06-10T21:33:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3952000000000001e-05 2023-06-10T21:33:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289668+0000", 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001314 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289799+0000", 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9462999999999994e-05 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289889+0000", 2023-06-10T21:33:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029815369999999998 2023-06-10T21:33:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.292870+0000", 2023-06-10T21:33:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065814400000000005 2023-06-10T21:33:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.293528+0000", 2023-06-10T21:33:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030991312 2023-06-10T21:33:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324520+0000", 2023-06-10T21:33:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3043e-05 2023-06-10T21:33:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324533+0000", 2023-06-10T21:33:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.00200007 2023-06-10T21:33:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326533+0000", 2023-06-10T21:33:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6407000000000003e-05 2023-06-10T21:33:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326599+0000", 2023-06-10T21:33:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999 2023-06-10T21:33:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111487 10.1 10:8be62f36:::benchmark_data_smithi064_312713_object20731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:33:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:33:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 367.50084291299999, 2023-06-10T21:33:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997, 2023-06-10T21:33:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111487 2023-06-10T21:33:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331724+0000", 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7449999999999995e-06 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331732+0000", 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5709000000000002e-05 2023-06-10T21:33:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0659999999999998e-06 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331755+0000", 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1113999999999999e-05 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331806+0000", 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0156999999999999e-05 2023-06-10T21:33:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331816+0000", 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99503633400000002 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326852+0000", 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9278999999999996e-05 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326952+0000", 2023-06-10T21:33:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015410900000000001 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4", 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.327106+0000", 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035978350000000002 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.330703+0000", 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051919100000000001 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331223+0000", 2023-06-10T21:33:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1488999999999994e-05 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331284+0000", 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112025 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331396+0000", 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85133 11.3 11.3e76a33 (undecoded) ondisk+retry+write+known_if_redirected e288)", 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.311743812000003, 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995, 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85133 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520824+0000", 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.792e-06 2023-06-10T21:33:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520829+0000", 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0971999999999999e-05 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.906 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520840+0000", 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9069999999999998e-06 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520845+0000", 2023-06-10T21:33:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032454789999999999 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524090+0000", 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.843e-06 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524095+0000", 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98421531600000001 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508311+0000", 2023-06-10T21:33:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7920000000000004e-06 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508320+0000", 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85152 11.5 11.363bf865 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.291056050000002, 2023-06-10T21:33:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995, 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85152 2023-06-10T21:33:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541512+0000", 2023-06-10T21:33:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9899999999999999e-06 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541516+0000", 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0390000000000005e-06 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541524+0000", 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2599999999999999e-06 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541528+0000", 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4838000000000001e-05 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541563+0000", 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.95e-06 2023-06-10T21:33:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541568+0000", 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96671271599999997 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508281+0000", 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1725999999999999e-05 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508293+0000", 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995 2023-06-10T21:33:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85160 11.3 11.f67e3553 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:33:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.271545998999997, 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94735323500000002, 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85160 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561022+0000", 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1909999999999997e-06 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561028+0000", 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3774999999999999e-05 2023-06-10T21:33:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561042+0000", 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1970000000000004e-06 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561047+0000", 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1782000000000001e-05 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561089+0000", 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4460000000000007e-06 2023-06-10T21:33:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561098+0000", 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94724917200000003 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508347+0000", 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8926000000000001e-05 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508376+0000", 2023-06-10T21:33:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94735323500000002 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85191 11.5 11.7bbb6cb5 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.234372731000001, 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91013171800000003, 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85191 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598195+0000", 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5589999999999999e-06 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598200+0000", 2023-06-10T21:33:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3920000000000003e-06 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598208+0000", 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2100000000000001e-06 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598213+0000", 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598250+0000", 2023-06-10T21:33:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6789999999999996e-06 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598255+0000", 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91006529300000005 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508320+0000", 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5769999999999996e-06 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508328+0000", 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91013171800000003 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85195 11.3 11.919bb373 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.231737844999998, 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758057299999995, 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85195 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8479999999999999e-06 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600833+0000", 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2070000000000003e-06 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600838+0000", 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0210000000000001e-06 2023-06-10T21:33:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600842+0000", 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2150000000000001e-05 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600884+0000", 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8730000000000004e-06 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600890+0000", 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90751452799999999 2023-06-10T21:33:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508404+0000", 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1849999999999998e-06 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508412+0000", 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758057299999995 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85196 11.3 11.67cdda23 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.231712834, 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758501300000005, 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85196 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7930000000000002e-06 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600858+0000", 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6449999999999999e-06 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600865+0000", 2023-06-10T21:33:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0100000000000003e-06 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600870+0000", 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00046476800000000001 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.601335+0000", 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3800000000000002e-06 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.601340+0000", 2023-06-10T21:33:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90709262499999999 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508433+0000", 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2449999999999998e-06 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508441+0000", 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758501300000005 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85222 11.3 11.2dc991d3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.197243606000001, 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314513999999999, 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85222 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:33:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635324+0000", 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5709999999999999e-06 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635329+0000", 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1164e-05 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635340+0000", 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6739999999999996e-06 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635345+0000", 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3868000000000005e-05 2023-06-10T21:33:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635418+0000", 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4380000000000003e-06 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635424+0000", 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87303943799999995 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508463+0000", 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0779999999999998e-06 2023-06-10T21:33:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508470+0000", 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314513999999999 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85223 11.3 11.8bf7f803 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.197213947999998, 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314413300000004, 2023-06-10T21:33:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85223 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635354+0000", 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1090000000000001e-06 2023-06-10T21:33:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635357+0000", 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3224e-05 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635380+0000", 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7760000000000002e-06 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635383+0000", 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00044493000000000001 2023-06-10T21:33:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635828+0000", 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3069999999999996e-06 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635832+0000", 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87265963000000002 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508491+0000", 2023-06-10T21:33:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.542e-06 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508499+0000", 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314413300000004 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85224 11.3 11.3ab6c8a3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635386+0000", 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.197182693999999, 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314312999999999, 2023-06-10T21:33:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85224 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635386+0000", 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635386+0000", 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635386+0000", 2023-06-10T21:33:48.978 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7400000000000001e-06 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635388+0000", 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7779999999999998e-06 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635392+0000", 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2409999999999998e-06 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635395+0000", 2023-06-10T21:33:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00089670899999999998 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636291+0000", 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.779e-06 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636294+0000", 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87222703499999998 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508521+0000", 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1130000000000001e-06 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508529+0000", 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314312999999999 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85227 11.3 11.edeb40b3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.197153817999997, 2023-06-10T21:33:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87315411499999995, 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85227 2023-06-10T21:33:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0999999999999998e-06 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635417+0000", 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4450000000000001e-06 2023-06-10T21:33:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635420+0000", 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2230000000000001e-06 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635422+0000", 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001176156 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636599+0000", 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.481e-06 2023-06-10T21:33:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636601+0000", 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87194879999999997 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508550+0000", 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9221000000000001e-05 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508569+0000", 2023-06-10T21:33:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87315411499999995 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85250 11.5 11.c30edcb5 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.671462+0000", 2023-06-10T21:33:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.161106644999997, 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83690767200000005, 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85250 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671462+0000", 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671462+0000", 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671461+0000", 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6359999999999999e-06 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671467+0000", 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2281e-05 2023-06-10T21:33:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671479+0000", 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6150000000000002e-06 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671486+0000", 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5423999999999997e-05 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671531+0000", 2023-06-10T21:33:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3039999999999999e-06 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.671539+0000", 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83681101899999999 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508350+0000", 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0319e-05 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508370+0000", 2023-06-10T21:33:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83690767200000005 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85263 11.5 11.411bc185 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.692894+0000", 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.139674966000001, 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81550692899999999, 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85263 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692894+0000", 2023-06-10T21:33:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692894+0000", 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692893+0000", 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8470000000000001e-06 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692898+0000", 2023-06-10T21:33:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0216e-05 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692929+0000", 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8689999999999998e-06 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692936+0000", 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3315e-05 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692989+0000", 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4039999999999998e-06 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692996+0000", 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81539732200000004 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508394+0000", 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.17e-06 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508401+0000", 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81550692899999999 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85264 11.5 11.a4ed8545 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.692952+0000", 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.139616678000003, 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81547886300000005, 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85264 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692952+0000", 2023-06-10T21:33:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692952+0000", 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692952+0000", 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.755e-06 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692955+0000", 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0899999999999999e-06 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692962+0000", 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5650000000000003e-06 2023-06-10T21:33:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.692967+0000", 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045615299999999999 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.693423+0000", 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.614e-06 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.693427+0000", 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.814997321 2023-06-10T21:33:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508424+0000", 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9789999999999996e-06 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508431+0000", 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.81547886300000005 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85269 11.5 11.87ceef15 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.703453+0000", 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.129116193999998, 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80546517299999998, 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85269 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703453+0000", 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703453+0000", 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:33:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703450+0000", 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.306e-05 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703463+0000", 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1891999999999999e-05 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703495+0000", 2023-06-10T21:33:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0144000000000001e-05 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703526+0000", 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010737499999999999 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703633+0000", 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.153e-05 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.703644+0000", 2023-06-10T21:33:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80526125400000004 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508906+0000", 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2125e-05 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508918+0000", 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80546517299999998 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85276 11.3 11.5193c1c3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.717105+0000", 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.115464056999997, 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79149550199999996, 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85276 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717105+0000", 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717105+0000", 2023-06-10T21:33:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717104+0000", 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7800000000000003e-06 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717111+0000", 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2799999999999999e-05 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717123+0000", 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4200000000000004e-06 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717130+0000", 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6480000000000002e-05 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717176+0000", 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0679999999999996e-06 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.717181+0000", 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79141147599999995 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508593+0000", 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.413e-06 2023-06-10T21:33:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508600+0000", 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79149550199999996 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85286 11.3 11.80c4cd03 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.750201+0000", 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 54.082367445000003, 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75860318400000004, 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:33:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85286 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:33:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750201+0000", 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:33:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750201+0000", 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750199+0000", 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3087e-05 2023-06-10T21:33:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750212+0000", 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1813000000000002e-05 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750244+0000", 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9715999999999999e-05 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750264+0000", 2023-06-10T21:33:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2978999999999994e-05 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750337+0000", 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6819999999999997e-06 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.750343+0000", 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75845115399999996 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508795+0000", 2023-06-10T21:33:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0063e-05 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508805+0000", 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.75860318400000004 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:33:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:33:49.048 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:33:49.258 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:33:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5861/172299 objects degraded (3.402%); 3508/172299 objects misplaced (2.036%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5861/172299 objects degraded (3.402%); 3508/172299 objects misplaced (2.036%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:33:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:33:49.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: pgmap v1071: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5861/172299 objects degraded (3.402%); 3508/172299 objects misplaced (2.036%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:33:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:33:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:33:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:33:49.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:33:49.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:49.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:49.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:49.783 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:49.833 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:49.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:49.943 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:33:50.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:50.616 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:33:50.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:50.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:33:50.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:50.832 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:51 smithi152 ceph-mon[156819]: pgmap v1072: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 5861/169407 objects degraded (3.460%); 3508/169407 objects misplaced (2.071%); 1004 KiB/s, 15 objects/s recovering 2023-06-10T21:33:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5861/169407 objects degraded (3.460%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:51.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:51.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:51 smithi064 ceph-mon[180297]: pgmap v1072: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 5861/169407 objects degraded (3.460%); 3508/169407 objects misplaced (2.071%); 1004 KiB/s, 15 objects/s recovering 2023-06-10T21:33:51.550 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5861/169407 objects degraded (3.460%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:51.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:51.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:51 smithi064 ceph-mon[186706]: pgmap v1072: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 5861/169407 objects degraded (3.460%); 3508/169407 objects misplaced (2.071%); 1004 KiB/s, 15 objects/s recovering 2023-06-10T21:33:51.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5861/169407 objects degraded (3.460%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:51.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:51.680 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:51.735 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:51.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:51.843 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:33:51.870 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:51.918 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:33:51.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:51.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:33:51.953 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:51.994 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:33:52.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:52.088 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:33:52.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:52.108 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:33:52.193 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:33:52.194 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:52.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:33:52.271 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:33:52.278 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/387816029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:52.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/387816029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:52.290 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:52.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:52.315 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:33:52.357 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:33:52.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:33:52.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/387816029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:33:52.495 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:53.002 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:33:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:33:52] "GET /metrics HTTP/1.1" 200 224062 "" "Prometheus/2.33.4" 2023-06-10T21:33:53.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:53.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:53.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:53.236 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:53 smithi064 ceph-mon[180297]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 5893/164676 objects degraded (3.579%); 3354/164676 objects misplaced (2.037%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:53.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:53 smithi064 ceph-mon[186706]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 5893/164676 objects degraded (3.579%); 3354/164676 objects misplaced (2.037%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:53 smithi152 ceph-mon[156819]: pgmap v1073: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 5893/164676 objects degraded (3.579%); 3354/164676 objects misplaced (2.037%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:54.175 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:33:54.196 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:33:55.290 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:55.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:55.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:55.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:33:55.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:55 smithi064 ceph-mon[186706]: pgmap v1074: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 5893/163254 objects degraded (3.610%); 3354/163254 objects misplaced (2.054%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:55.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:55 smithi064 ceph-mon[186706]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:55 smithi064 ceph-mon[180297]: pgmap v1074: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 5893/163254 objects degraded (3.610%); 3354/163254 objects misplaced (2.054%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:55 smithi064 ceph-mon[180297]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:55.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:55 smithi152 ceph-mon[156819]: pgmap v1074: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 5893/163254 objects degraded (3.610%); 3354/163254 objects misplaced (2.054%); 1.7 MiB/s, 27 objects/s recovering 2023-06-10T21:33:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:55 smithi152 ceph-mon[156819]: from='client.36303 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:33:56.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5893/163254 objects degraded (3.610%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5893/163254 objects degraded (3.610%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5893/163254 objects degraded (3.610%), 2 pgs degraded (PG_DEGRADED) 2023-06-10T21:33:56.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:33:56.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:33:56.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:56.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:33:57.064 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:33:57.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:57.145 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:33:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:57.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:33:57.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:57.283 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:57.332 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:57.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:57.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:57.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:57 smithi064 ceph-mon[180297]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 5898/159816 objects degraded (3.690%); 3180/159816 objects misplaced (1.990%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:33:57.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:57 smithi064 ceph-mon[186706]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 5898/159816 objects degraded (3.690%); 3180/159816 objects misplaced (1.990%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:33:57.438 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:33:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:57 smithi152 ceph-mon[156819]: pgmap v1075: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 548 op/s; 5898/159816 objects degraded (3.690%); 3180/159816 objects misplaced (1.990%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:33:57.466 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:57.515 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:33:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:57.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:33:57.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:57.658 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1074,"stamp":"2023-06-10T21:33:53.814765+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3565295526,"num_objects":54418,"num_object_clones":0,"num_object_copies":163254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5893,"num_objects_misplaced":3354,"num_objects_unfound":0,"num_objects_dirty":54418,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":114409,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1389,"num_bytes_recovered":90054354,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103943,"ondisk_log_size":103943,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13462360,"kb_used_data":10752616,"kb_used_omap":30,"kb_used_meta":2709537,"kb_avail":549008552,"statfs":{"total":575970213888,"available":562184757248,"internally_reserved":0,"allocated":11010678784,"data_stored":11005629624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2774565926},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,2,2,1,3,0,0,12,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-332595200,"num_objects":-5075,"num_object_clones":0,"num_object_copies":-15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":46,"num_objects_misplaced":-325,"num_objects_unfound":0,"num_objects_dirty":-5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":279,"num_bytes_recovered":18284544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008761"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":291,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:34.726433+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:33:34.726433+0000","last_peered":"2023-06-10T21:33:34.726433+0000","last_clean":"2023-06-10T21:33:34.726433+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:33:34.726433+0000","last_undegraded":"2023-06-10T21:33:34.726433+0000","last_fullsized":"2023-06-10T21:33:34.726433+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092808+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:33:33.092808+0000","last_peered":"2023-06-10T21:33:33.092808+0000","last_clean":"2023-06-10T21:33:33.092808+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:33:33.092808+0000","last_undegraded":"2023-06-10T21:33:33.092808+0000","last_fullsized":"2023-06-10T21:33:33.092808+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":207,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.490264+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:33:44.490264+0000","last_peered":"2023-06-10T21:33:44.490264+0000","last_clean":"2023-06-10T21:33:44.490264+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:33:44.490264+0000","last_undegraded":"2023-06-10T21:33:44.490264+0000","last_fullsized":"2023-06-10T21:33:44.490264+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099854+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:33:33.099854+0000","last_peered":"2023-06-10T21:33:33.099854+0000","last_clean":"2023-06-10T21:33:33.099854+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:33:33.099854+0000","last_undegraded":"2023-06-10T21:33:33.099854+0000","last_fullsized":"2023-06-10T21:33:33.099854+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":272,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.491314+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:33:44.491314+0000","last_peered":"2023-06-10T21:33:44.491314+0000","last_clean":"2023-06-10T21:33:44.491314+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:33:44.491314+0000","last_undegraded":"2023-06-10T21:33:44.491314+0000","last_fullsized":"2023-06-10T21:33:44.491314+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":264,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099503+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:33:33.099503+0000","last_peered":"2023-06-10T21:33:33.099503+0000","last_clean":"2023-06-10T21:33:33.099503+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:33:33.099503+0000","last_undegraded":"2023-06-10T21:33:33.099503+0000","last_fullsized":"2023-06-10T21:33:33.099503+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":154,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099593+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:33:33.099593+0000","last_peered":"2023-06-10T21:33:33.099593+0000","last_clean":"2023-06-10T21:33:33.099593+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:33:33.099593+0000","last_undegraded":"2023-06-10T21:33:33.099593+0000","last_fullsized":"2023-06-10T21:33:33.099593+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":246,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:36.485639+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:33:36.485639+0000","last_peered":"2023-06-10T21:33:36.485639+0000","last_clean":"2023-06-10T21:33:36.485639+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:33:36.485639+0000","last_undegraded":"2023-06-10T21:33:36.485639+0000","last_fullsized":"2023-06-10T21:33:36.485639+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":19,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061735+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:33:33.061735+0000","last_peered":"2023-06-10T21:33:33.061735+0000","last_clean":"2023-06-10T21:33:33.061735+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:33:33.061735+0000","last_undegraded":"2023-06-10T21:33:33.061735+0000","last_fullsized":"2023-06-10T21:33:33.061735+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":20,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092333+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:33:33.092333+0000","last_peered":"2023-06-10T21:33:33.092333+0000","last_clean":"2023-06-10T21:33:33.092333+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:33:33.092333+0000","last_undegraded":"2023-06-10T21:33:33.092333+0000","last_fullsized":"2023-06-10T21:33:33.092333+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":25,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061557+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:33:33.061557+0000","last_peered":"2023-06-10T21:33:33.061557+0000","last_clean":"2023-06-10T21:33:33.061557+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:33:33.061557+0000","last_undegraded":"2023-06-10T21:33:33.061557+0000","last_fullsized":"2023-06-10T21:33:33.061557+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":260,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.086183+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:33:33.086183+0000","last_peered":"2023-06-10T21:33:33.086183+0000","last_clean":"2023-06-10T21:33:33.086183+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:33:33.086183+0000","last_undegraded":"2023-06-10T21:33:33.086183+0000","last_fullsized":"2023-06-10T21:33:33.086183+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":168,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061559+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:33:33.061559+0000","last_peered":"2023-06-10T21:33:33.061559+0000","last_clean":"2023-06-10T21:33:33.061559+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:33:33.061559+0000","last_undegraded":"2023-06-10T21:33:33.061559+0000","last_fullsized":"2023-06-10T21:33:33.061559+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":29,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.064668+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:33:33.064668+0000","last_peered":"2023-06-10T21:33:33.064668+0000","last_clean":"2023-06-10T21:33:33.064668+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:33:33.064668+0000","last_undegraded":"2023-06-10T21:33:33.064668+0000","last_fullsized":"2023-06-10T21:33:33.064668+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":297,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.064738+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:33:33.064738+0000","last_peered":"2023-06-10T21:33:33.064738+0000","last_clean":"2023-06-10T21:33:33.064738+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:33:33.064738+0000","last_undegraded":"2023-06-10T21:33:33.064738+0000","last_fullsized":"2023-06-10T21:33:33.064738+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":232,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:36.506754+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:33:36.506754+0000","last_peered":"2023-06-10T21:33:36.506754+0000","last_clean":"2023-06-10T21:33:36.506754+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:33:36.506754+0000","last_undegraded":"2023-06-10T21:33:36.506754+0000","last_fullsized":"2023-06-10T21:33:36.506754+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":154,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.100932+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:33:33.100932+0000","last_peered":"2023-06-10T21:33:33.100932+0000","last_clean":"2023-06-10T21:33:33.100932+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:33:33.100932+0000","last_undegraded":"2023-06-10T21:33:33.100932+0000","last_fullsized":"2023-06-10T21:33:33.100932+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":226,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061759+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:33:33.061759+0000","last_peered":"2023-06-10T21:33:33.061759+0000","last_clean":"2023-06-10T21:33:33.061759+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:33:33.061759+0000","last_undegraded":"2023-06-10T21:33:33.061759+0000","last_fullsized":"2023-06-10T21:33:33.061759+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":274,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061386+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:33:33.061386+0000","last_peered":"2023-06-10T21:33:33.061386+0000","last_clean":"2023-06-10T21:33:33.061386+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:33:33.061386+0000","last_undegraded":"2023-06-10T21:33:33.061386+0000","last_fullsized":"2023-06-10T21:33:33.061386+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099682+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:33:33.099682+0000","last_peered":"2023-06-10T21:33:33.099682+0000","last_clean":"2023-06-10T21:33:33.099682+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:33:33.099682+0000","last_undegraded":"2023-06-10T21:33:33.099682+0000","last_fullsized":"2023-06-10T21:33:33.099682+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":273,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.098341+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:33:33.098341+0000","last_peered":"2023-06-10T21:33:33.098341+0000","last_clean":"2023-06-10T21:33:33.098341+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:33:33.098341+0000","last_undegraded":"2023-06-10T21:33:33.098341+0000","last_fullsized":"2023-06-10T21:33:33.098341+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":250,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488705+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:33:44.488705+0000","last_peered":"2023-06-10T21:33:44.488705+0000","last_clean":"2023-06-10T21:33:44.488705+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:33:44.488705+0000","last_undegraded":"2023-06-10T21:33:44.488705+0000","last_fullsized":"2023-06-10T21:33:44.488705+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"298'7208","reported_seq":7247,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:52.881174+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:33:52.881174+0000","last_peered":"2023-06-10T21:33:52.881174+0000","last_clean":"2023-06-10T21:33:52.881174+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:33:52.881174+0000","last_undegraded":"2023-06-10T21:33:52.881174+0000","last_fullsized":"2023-06-10T21:33:52.881174+0000","mapping_epoch":269,"log_start":"276'800","ondisk_log_start":"276'800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218628096,"num_objects":3336,"num_object_clones":0,"num_object_copies":10008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7208,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":23,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061730+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:33:33.061730+0000","last_peered":"2023-06-10T21:33:33.061730+0000","last_clean":"2023-06-10T21:33:33.061730+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:33:33.061730+0000","last_undegraded":"2023-06-10T21:33:33.061730+0000","last_fullsized":"2023-06-10T21:33:33.061730+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":19,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061227+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:33:33.061227+0000","last_peered":"2023-06-10T21:33:33.061227+0000","last_clean":"2023-06-10T21:33:33.061227+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:33:33.061227+0000","last_undegraded":"2023-06-10T21:33:33.061227+0000","last_fullsized":"2023-06-10T21:33:33.061227+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"298'7267","reported_seq":7306,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:52.935227+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:33:52.935227+0000","last_peered":"2023-06-10T21:33:52.935227+0000","last_clean":"2023-06-10T21:33:52.935227+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:33:52.935227+0000","last_undegraded":"2023-06-10T21:33:52.935227+0000","last_fullsized":"2023-06-10T21:33:52.935227+0000","mapping_epoch":269,"log_start":"277'800","ondisk_log_start":"277'800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7267,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":23,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061747+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:33:33.061747+0000","last_peered":"2023-06-10T21:33:33.061747+0000","last_clean":"2023-06-10T21:33:33.061747+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:33:33.061747+0000","last_undegraded":"2023-06-10T21:33:33.061747+0000","last_fullsized":"2023-06-10T21:33:33.061747+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092906+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:33:33.092906+0000","last_peered":"2023-06-10T21:33:33.092906+0000","last_clean":"2023-06-10T21:33:33.092906+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:33:33.092906+0000","last_undegraded":"2023-06-10T21:33:33.092906+0000","last_fullsized":"2023-06-10T21:33:33.092906+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"298'7415","reported_seq":7457,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:52.910204+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:33:52.910204+0000","last_peered":"2023-06-10T21:33:52.910204+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:33:52.910204+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"277'1000","ondisk_log_start":"277'1000","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228786176,"num_objects":3491,"num_object_clones":0,"num_object_copies":10473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7415,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":3491},{"shards":"0,3","objects":-1903}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":305,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061667+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:33:33.061667+0000","last_peered":"2023-06-10T21:33:33.061667+0000","last_clean":"2023-06-10T21:33:33.061667+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:33:33.061667+0000","last_undegraded":"2023-06-10T21:33:33.061667+0000","last_fullsized":"2023-06-10T21:33:33.061667+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061404+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:33:33.061404+0000","last_peered":"2023-06-10T21:33:33.061404+0000","last_clean":"2023-06-10T21:33:33.061404+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:33:33.061404+0000","last_undegraded":"2023-06-10T21:33:33.061404+0000","last_fullsized":"2023-06-10T21:33:33.061404+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"298'7019","reported_seq":7056,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.542834+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:33:49.542834+0000","last_peered":"2023-06-10T21:33:49.542834+0000","last_clean":"2023-06-10T21:33:49.542834+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:33:49.542834+0000","last_undegraded":"2023-06-10T21:33:49.542834+0000","last_fullsized":"2023-06-10T21:33:49.542834+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7019,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":179,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092086+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:33:33.092086+0000","last_peered":"2023-06-10T21:33:33.092086+0000","last_clean":"2023-06-10T21:33:33.092086+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:33:33.092086+0000","last_undegraded":"2023-06-10T21:33:33.092086+0000","last_fullsized":"2023-06-10T21:33:33.092086+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.098450+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:33:33.098450+0000","last_peered":"2023-06-10T21:33:33.098450+0000","last_clean":"2023-06-10T21:33:33.098450+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:33:33.098450+0000","last_undegraded":"2023-06-10T21:33:33.098450+0000","last_fullsized":"2023-06-10T21:33:33.098450+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"298'7030","reported_seq":9771,"reported_epoch":298,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:33:49.816238+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:33:49.816238+0000","last_peered":"2023-06-10T21:33:49.816238+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:33:49.816238+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220069888,"num_objects":3358,"num_object_clones":0,"num_object_copies":10074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":440,"num_objects_misplaced":3354,"num_objects_unfound":0,"num_objects_dirty":3358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7030,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1348,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-436},{"shards":"1,2,3","objects":3354}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":249,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.102772+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:33:33.102772+0000","last_peered":"2023-06-10T21:33:33.102772+0000","last_clean":"2023-06-10T21:33:33.102772+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:33:33.102772+0000","last_undegraded":"2023-06-10T21:33:33.102772+0000","last_fullsized":"2023-06-10T21:33:33.102772+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092160+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:33:33.092160+0000","last_peered":"2023-06-10T21:33:33.092160+0000","last_clean":"2023-06-10T21:33:33.092160+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:33:33.092160+0000","last_undegraded":"2023-06-10T21:33:33.092160+0000","last_fullsized":"2023-06-10T21:33:33.092160+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"298'7261","reported_seq":7300,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.799100+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:33:49.799100+0000","last_peered":"2023-06-10T21:33:49.799100+0000","last_clean":"2023-06-10T21:33:49.799100+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:33:49.799100+0000","last_undegraded":"2023-06-10T21:33:49.799100+0000","last_fullsized":"2023-06-10T21:33:49.799100+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7261,"log_dups_size":0,"ondisk_log_size":7261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228917248,"num_objects":3493,"num_object_clones":0,"num_object_copies":10479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7261,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":72,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.106694+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:33:33.106694+0000","last_peered":"2023-06-10T21:33:33.106694+0000","last_clean":"2023-06-10T21:33:33.106694+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:33:33.106694+0000","last_undegraded":"2023-06-10T21:33:33.106694+0000","last_fullsized":"2023-06-10T21:33:33.106694+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":289,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.087907+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:33:33.087907+0000","last_peered":"2023-06-10T21:33:33.087907+0000","last_clean":"2023-06-10T21:33:33.087907+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:33:33.087907+0000","last_undegraded":"2023-06-10T21:33:33.087907+0000","last_fullsized":"2023-06-10T21:33:33.087907+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"298'7115","reported_seq":7154,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.313900+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:33:51.313900+0000","last_peered":"2023-06-10T21:33:51.313900+0000","last_clean":"2023-06-10T21:33:51.313900+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:33:51.313900+0000","last_undegraded":"2023-06-10T21:33:51.313900+0000","last_fullsized":"2023-06-10T21:33:51.313900+0000","mapping_epoch":269,"log_start":"279'1300","ondisk_log_start":"279'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5815,"log_dups_size":0,"ondisk_log_size":5815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7115,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":275,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.084548+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:33:33.084548+0000","last_peered":"2023-06-10T21:33:33.084548+0000","last_clean":"2023-06-10T21:33:33.084548+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:33:33.084548+0000","last_undegraded":"2023-06-10T21:33:33.084548+0000","last_fullsized":"2023-06-10T21:33:33.084548+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092465+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:33:33.092465+0000","last_peered":"2023-06-10T21:33:33.092465+0000","last_clean":"2023-06-10T21:33:33.092465+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:33:33.092465+0000","last_undegraded":"2023-06-10T21:33:33.092465+0000","last_fullsized":"2023-06-10T21:33:33.092465+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"298'7194","reported_seq":7233,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.483306+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:33:51.483306+0000","last_peered":"2023-06-10T21:33:51.483306+0000","last_clean":"2023-06-10T21:33:51.483306+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:33:51.483306+0000","last_undegraded":"2023-06-10T21:33:51.483306+0000","last_fullsized":"2023-06-10T21:33:51.483306+0000","mapping_epoch":269,"log_start":"279'1400","ondisk_log_start":"279'1400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219414528,"num_objects":3348,"num_object_clones":0,"num_object_copies":10044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.103577+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:33:33.103577+0000","last_peered":"2023-06-10T21:33:33.103577+0000","last_clean":"2023-06-10T21:33:33.103577+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:33:33.103577+0000","last_undegraded":"2023-06-10T21:33:33.103577+0000","last_fullsized":"2023-06-10T21:33:33.103577+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.102501+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:33:33.102501+0000","last_peered":"2023-06-10T21:33:33.102501+0000","last_clean":"2023-06-10T21:33:33.102501+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:33:33.102501+0000","last_undegraded":"2023-06-10T21:33:33.102501+0000","last_fullsized":"2023-06-10T21:33:33.102501+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"298'7084","reported_seq":7124,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.657765+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:33:49.657765+0000","last_peered":"2023-06-10T21:33:49.657765+0000","last_clean":"2023-06-10T21:33:49.657765+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:33:49.657765+0000","last_undegraded":"2023-06-10T21:33:49.657765+0000","last_fullsized":"2023-06-10T21:33:49.657765+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7084,"log_dups_size":0,"ondisk_log_size":7084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218824728,"num_objects":3340,"num_object_clones":0,"num_object_copies":10020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3340,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7084,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":121,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.097810+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:33:33.097810+0000","last_peered":"2023-06-10T21:33:33.097810+0000","last_clean":"2023-06-10T21:33:33.097810+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:33:33.097810+0000","last_undegraded":"2023-06-10T21:33:33.097810+0000","last_fullsized":"2023-06-10T21:33:33.097810+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":301,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092892+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:33:33.092892+0000","last_peered":"2023-06-10T21:33:33.092892+0000","last_clean":"2023-06-10T21:33:33.092892+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:33:33.092892+0000","last_undegraded":"2023-06-10T21:33:33.092892+0000","last_fullsized":"2023-06-10T21:33:33.092892+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":141,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.100789+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:33:33.100789+0000","last_peered":"2023-06-10T21:33:33.100789+0000","last_clean":"2023-06-10T21:33:33.100789+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:33:33.100789+0000","last_undegraded":"2023-06-10T21:33:33.100789+0000","last_fullsized":"2023-06-10T21:33:33.100789+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"298'7114","reported_seq":7153,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.471376+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:33:51.471376+0000","last_peered":"2023-06-10T21:33:51.471376+0000","last_clean":"2023-06-10T21:33:51.471376+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:33:51.471376+0000","last_undegraded":"2023-06-10T21:33:51.471376+0000","last_fullsized":"2023-06-10T21:33:51.471376+0000","mapping_epoch":269,"log_start":"279'1300","ondisk_log_start":"279'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225050624,"num_objects":3434,"num_object_clones":0,"num_object_copies":10302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7114,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":167,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.100859+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:33:33.100859+0000","last_peered":"2023-06-10T21:33:33.100859+0000","last_clean":"2023-06-10T21:33:33.100859+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:33:33.100859+0000","last_undegraded":"2023-06-10T21:33:33.100859+0000","last_fullsized":"2023-06-10T21:33:33.100859+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":241,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.098606+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:33:33.098606+0000","last_peered":"2023-06-10T21:33:33.098606+0000","last_clean":"2023-06-10T21:33:33.098606+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:33:33.098606+0000","last_undegraded":"2023-06-10T21:33:33.098606+0000","last_fullsized":"2023-06-10T21:33:33.098606+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":130,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.086621+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:33:33.086621+0000","last_peered":"2023-06-10T21:33:33.086621+0000","last_clean":"2023-06-10T21:33:33.086621+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:33:33.086621+0000","last_undegraded":"2023-06-10T21:33:33.086621+0000","last_fullsized":"2023-06-10T21:33:33.086621+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"298'7140","reported_seq":7177,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.471616+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:33:51.471616+0000","last_peered":"2023-06-10T21:33:51.471616+0000","last_clean":"2023-06-10T21:33:51.471616+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:33:51.471616+0000","last_undegraded":"2023-06-10T21:33:51.471616+0000","last_fullsized":"2023-06-10T21:33:51.471616+0000","mapping_epoch":269,"log_start":"279'1300","ondisk_log_start":"279'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222167040,"num_objects":3390,"num_object_clones":0,"num_object_copies":10170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7140,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":141,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.084615+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:33:33.084615+0000","last_peered":"2023-06-10T21:33:33.084615+0000","last_clean":"2023-06-10T21:33:33.084615+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:33:33.084615+0000","last_undegraded":"2023-06-10T21:33:33.084615+0000","last_fullsized":"2023-06-10T21:33:33.084615+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:34.707543+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:33:34.707543+0000","last_peered":"2023-06-10T21:33:34.707543+0000","last_clean":"2023-06-10T21:33:34.707543+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:33:34.707543+0000","last_undegraded":"2023-06-10T21:33:34.707543+0000","last_fullsized":"2023-06-10T21:33:34.707543+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099671+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:33:33.099671+0000","last_peered":"2023-06-10T21:33:33.099671+0000","last_clean":"2023-06-10T21:33:33.099671+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:33:33.099671+0000","last_undegraded":"2023-06-10T21:33:33.099671+0000","last_fullsized":"2023-06-10T21:33:33.099671+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"298'7037","reported_seq":7076,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.337780+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:33:51.337780+0000","last_peered":"2023-06-10T21:33:51.337780+0000","last_clean":"2023-06-10T21:33:51.337780+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:33:51.337780+0000","last_undegraded":"2023-06-10T21:33:51.337780+0000","last_fullsized":"2023-06-10T21:33:51.337780+0000","mapping_epoch":269,"log_start":"278'1200","ondisk_log_start":"278'1200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5837,"log_dups_size":0,"ondisk_log_size":5837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907200,"num_objects":3325,"num_object_clones":0,"num_object_copies":9975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7037,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":321,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.085528+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:33:33.085528+0000","last_peered":"2023-06-10T21:33:33.085528+0000","last_clean":"2023-06-10T21:33:33.085528+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:33:33.085528+0000","last_undegraded":"2023-06-10T21:33:33.085528+0000","last_fullsized":"2023-06-10T21:33:33.085528+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":461,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061264+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:33:33.061264+0000","last_peered":"2023-06-10T21:33:33.061264+0000","last_clean":"2023-06-10T21:33:33.061264+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:33:33.061264+0000","last_undegraded":"2023-06-10T21:33:33.061264+0000","last_fullsized":"2023-06-10T21:33:33.061264+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":235,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.106836+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:33:33.106836+0000","last_peered":"2023-06-10T21:33:33.106836+0000","last_clean":"2023-06-10T21:33:33.106836+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:33:33.106836+0000","last_undegraded":"2023-06-10T21:33:33.106836+0000","last_fullsized":"2023-06-10T21:33:33.106836+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":112,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061741+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:33:33.061741+0000","last_peered":"2023-06-10T21:33:33.061741+0000","last_clean":"2023-06-10T21:33:33.061741+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:33:33.061741+0000","last_undegraded":"2023-06-10T21:33:33.061741+0000","last_fullsized":"2023-06-10T21:33:33.061741+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"298'7164","reported_seq":7203,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:51.507699+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:33:51.507699+0000","last_peered":"2023-06-10T21:33:51.507699+0000","last_clean":"2023-06-10T21:33:51.507699+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:33:51.507699+0000","last_undegraded":"2023-06-10T21:33:51.507699+0000","last_fullsized":"2023-06-10T21:33:51.507699+0000","mapping_epoch":269,"log_start":"279'1300","ondisk_log_start":"279'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227803136,"num_objects":3476,"num_object_clones":0,"num_object_copies":10428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7164,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":317,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061345+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:33:33.061345+0000","last_peered":"2023-06-10T21:33:33.061345+0000","last_clean":"2023-06-10T21:33:33.061345+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:33:33.061345+0000","last_undegraded":"2023-06-10T21:33:33.061345+0000","last_fullsized":"2023-06-10T21:33:33.061345+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":136,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.101695+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:33:33.101695+0000","last_peered":"2023-06-10T21:33:33.101695+0000","last_clean":"2023-06-10T21:33:33.101695+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:33:33.101695+0000","last_undegraded":"2023-06-10T21:33:33.101695+0000","last_fullsized":"2023-06-10T21:33:33.101695+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":121,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099592+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:33:33.099592+0000","last_peered":"2023-06-10T21:33:33.099592+0000","last_clean":"2023-06-10T21:33:33.099592+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:33:33.099592+0000","last_undegraded":"2023-06-10T21:33:33.099592+0000","last_fullsized":"2023-06-10T21:33:33.099592+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"298'7094","reported_seq":7133,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.774297+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:33:49.774297+0000","last_peered":"2023-06-10T21:33:49.774297+0000","last_clean":"2023-06-10T21:33:49.774297+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:33:49.774297+0000","last_undegraded":"2023-06-10T21:33:49.774297+0000","last_fullsized":"2023-06-10T21:33:49.774297+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7094,"log_dups_size":0,"ondisk_log_size":7094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225574912,"num_objects":3442,"num_object_clones":0,"num_object_copies":10326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7094,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":302,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061662+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:33:33.061662+0000","last_peered":"2023-06-10T21:33:33.061662+0000","last_clean":"2023-06-10T21:33:33.061662+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:33:33.061662+0000","last_undegraded":"2023-06-10T21:33:33.061662+0000","last_fullsized":"2023-06-10T21:33:33.061662+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.086700+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:33:33.086700+0000","last_peered":"2023-06-10T21:33:33.086700+0000","last_clean":"2023-06-10T21:33:33.086700+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:33:33.086700+0000","last_undegraded":"2023-06-10T21:33:33.086700+0000","last_fullsized":"2023-06-10T21:33:33.086700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":117,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.098748+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:33:33.098748+0000","last_peered":"2023-06-10T21:33:33.098748+0000","last_clean":"2023-06-10T21:33:33.098748+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:33:33.098748+0000","last_undegraded":"2023-06-10T21:33:33.098748+0000","last_fullsized":"2023-06-10T21:33:33.098748+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"298'7024","reported_seq":7061,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.616798+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:33:49.616798+0000","last_peered":"2023-06-10T21:33:49.616798+0000","last_clean":"2023-06-10T21:33:49.616798+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:33:49.616798+0000","last_undegraded":"2023-06-10T21:33:49.616798+0000","last_fullsized":"2023-06-10T21:33:49.616798+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220856320,"num_objects":3370,"num_object_clones":0,"num_object_copies":10110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7024,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":275,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.102319+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:33:33.102319+0000","last_peered":"2023-06-10T21:33:33.102319+0000","last_clean":"2023-06-10T21:33:33.102319+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:33:33.102319+0000","last_undegraded":"2023-06-10T21:33:33.102319+0000","last_fullsized":"2023-06-10T21:33:33.102319+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":72,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092280+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:33:33.092280+0000","last_peered":"2023-06-10T21:33:33.092280+0000","last_clean":"2023-06-10T21:33:33.092280+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:33:33.092280+0000","last_undegraded":"2023-06-10T21:33:33.092280+0000","last_fullsized":"2023-06-10T21:33:33.092280+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.103428+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:33:33.103428+0000","last_peered":"2023-06-10T21:33:33.103428+0000","last_clean":"2023-06-10T21:33:33.103428+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:33:33.103428+0000","last_undegraded":"2023-06-10T21:33:33.103428+0000","last_fullsized":"2023-06-10T21:33:33.103428+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"298'6982","reported_seq":7019,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:49.547541+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:33:49.547541+0000","last_peered":"2023-06-10T21:33:49.547541+0000","last_clean":"2023-06-10T21:33:49.547541+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:33:49.547541+0000","last_undegraded":"2023-06-10T21:33:49.547541+0000","last_fullsized":"2023-06-10T21:33:49.547541+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221118464,"num_objects":3374,"num_object_clones":0,"num_object_copies":10122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6982,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061714+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:33:33.061714+0000","last_peered":"2023-06-10T21:33:33.061714+0000","last_clean":"2023-06-10T21:33:33.061714+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:33:33.061714+0000","last_undegraded":"2023-06-10T21:33:33.061714+0000","last_fullsized":"2023-06-10T21:33:33.061714+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":125,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099775+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:33:33.099775+0000","last_peered":"2023-06-10T21:33:33.099775+0000","last_clean":"2023-06-10T21:33:33.099775+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:33:33.099775+0000","last_undegraded":"2023-06-10T21:33:33.099775+0000","last_fullsized":"2023-06-10T21:33:33.099775+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":253,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.101461+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:33:33.101461+0000","last_peered":"2023-06-10T21:33:33.101461+0000","last_clean":"2023-06-10T21:33:33.101461+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:33:33.101461+0000","last_undegraded":"2023-06-10T21:33:33.101461+0000","last_fullsized":"2023-06-10T21:33:33.101461+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":121,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.098837+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:33:33.098837+0000","last_peered":"2023-06-10T21:33:33.098837+0000","last_clean":"2023-06-10T21:33:33.098837+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:33:33.098837+0000","last_undegraded":"2023-06-10T21:33:33.098837+0000","last_fullsized":"2023-06-10T21:33:33.098837+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":139,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.086848+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:33:33.086848+0000","last_peered":"2023-06-10T21:33:33.086848+0000","last_clean":"2023-06-10T21:33:33.086848+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:33:33.086848+0000","last_undegraded":"2023-06-10T21:33:33.086848+0000","last_fullsized":"2023-06-10T21:33:33.086848+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":23,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.061455+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:33:33.061455+0000","last_peered":"2023-06-10T21:33:33.061455+0000","last_clean":"2023-06-10T21:33:33.061455+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:33:33.061455+0000","last_undegraded":"2023-06-10T21:33:33.061455+0000","last_fullsized":"2023-06-10T21:33:33.061455+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.102636+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:33:33.102636+0000","last_peered":"2023-06-10T21:33:33.102636+0000","last_clean":"2023-06-10T21:33:33.102636+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:33:33.102636+0000","last_undegraded":"2023-06-10T21:33:33.102636+0000","last_fullsized":"2023-06-10T21:33:33.102636+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":75,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092581+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:33:33.092581+0000","last_peered":"2023-06-10T21:33:33.092581+0000","last_clean":"2023-06-10T21:33:33.092581+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:33:33.092581+0000","last_undegraded":"2023-06-10T21:33:33.092581+0000","last_fullsized":"2023-06-10T21:33:33.092581+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":235,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.106763+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:33:33.106763+0000","last_peered":"2023-06-10T21:33:33.106763+0000","last_clean":"2023-06-10T21:33:33.106763+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:33:33.106763+0000","last_undegraded":"2023-06-10T21:33:33.106763+0000","last_fullsized":"2023-06-10T21:33:33.106763+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":281,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.092096+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:33:33.092096+0000","last_peered":"2023-06-10T21:33:33.092096+0000","last_clean":"2023-06-10T21:33:33.092096+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:33:33.092096+0000","last_undegraded":"2023-06-10T21:33:33.092096+0000","last_fullsized":"2023-06-10T21:33:33.092096+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":285,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.490045+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:33:44.490045+0000","last_peered":"2023-06-10T21:33:44.490045+0000","last_clean":"2023-06-10T21:33:44.490045+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:33:44.490045+0000","last_undegraded":"2023-06-10T21:33:44.490045+0000","last_fullsized":"2023-06-10T21:33:44.490045+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":186,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.099763+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:33:33.099763+0000","last_peered":"2023-06-10T21:33:33.099763+0000","last_clean":"2023-06-10T21:33:33.099763+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:33:33.099763+0000","last_undegraded":"2023-06-10T21:33:33.099763+0000","last_fullsized":"2023-06-10T21:33:33.099763+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":30,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-10T21:33:33.083913+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:33:33.083913+0000","last_peered":"2023-06-10T21:33:33.083913+0000","last_clean":"2023-06-10T21:33:33.083913+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:33:33.083913+0000","last_undegraded":"2023-06-10T21:33:33.083913+0000","last_fullsized":"2023-06-10T21:33:33.083913+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3564699672,"num_objects":54394,"num_object_clones":0,"num_object_copies":163182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5893,"num_objects_misplaced":3354,"num_objects_unfound":0,"num_objects_dirty":54394,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114148,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1348,"num_bytes_recovered":88276992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10929319936,"data_stored":10929307720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103748,"ondisk_log_size":103748,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430641,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1435840,"kb_used_data":875064,"kb_used_omap":0,"kb_used_meta":560768,"kb_avail":92309312,"statfs":{"total":95995035648,"available":94524735488,"internally_reserved":0,"allocated":896065536,"data_stored":895502118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":574226432},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":5.6710000000000003,"15min":8.3620000000000001},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":52.100000000000001,"5min":184.83699999999999,"15min":184.83699999999999},"last":2.573},{"interface":"front","average":{"1min":3.677,"5min":5.641,"15min":8.5800000000000001},"min":{"1min":0.31900000000000001,"5min":0.223,"15min":0.223},"max":{"1min":51.146999999999998,"5min":185.58000000000001,"15min":185.58000000000001},"last":3.0099999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":2.5800000000000001,"15min":3.0350000000000001},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":49.732999999999997,"5min":87.417000000000002,"15min":130.19399999999999},"last":1.2669999999999999},{"interface":"front","average":{"1min":4.883,"5min":2.7210000000000001,"15min":3.573},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":50.060000000000002,"5min":87.438000000000002,"15min":212.20599999999999},"last":1.6040000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:33:23 2023","interfaces":[{"interface":"back","average":{"1min":11.731999999999999,"5min":5.4450000000000003,"15min":3.7770000000000001},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":172.33600000000001,"5min":172.33600000000001,"15min":172.33600000000001},"last":2.2519999999999998},{"interface":"front","average":{"1min":11.782999999999999,"5min":5.3540000000000001,"15min":4.4139999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":172.185,"5min":172.185,"15min":212.13399999999999},"last":1.3999999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":2.8660000000000001,"15min":2.6219999999999999},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.9489999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":13.598000000000001},{"interface":"front","average":{"1min":1.393,"5min":2.4969999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.31,"15min":0.31},"max":{"1min":5.7869999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.442,"15min":0.75},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.41399999999999998,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.59499999999999997,"15min":1.091},"min":{"1min":0.28299999999999997,"5min":0.245,"15min":0.245},"max":{"1min":0.48699999999999999,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.36599999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.53900000000000003,"15min":1.847},"min":{"1min":0.309,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52600000000000002,"5min":3.9729999999999999,"15min":211.99799999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.52400000000000002,"15min":1.7},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.29299999999999998},"max":{"1min":0.499,"5min":6.7530000000000001,"15min":212.23699999999999},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.58899999999999997,"15min":1.1180000000000001},"min":{"1min":0.33800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.621,"5min":3.375,"15min":13.073},"last":0.435},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.499,"15min":1.0069999999999999},"min":{"1min":0.33900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.53000000000000003,"5min":1.897,"15min":10.874000000000001},"last":0.44400000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626869,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566312,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3569380,"kb_used_data":2954268,"kb_used_omap":4,"kb_used_meta":615099,"kb_avail":90175772,"statfs":{"total":95995035648,"available":92339990528,"internally_reserved":0,"allocated":3025170432,"data_stored":3024222145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":629862286},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3789999999999996,"5min":7.5979999999999999,"15min":11.686},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":70.254000000000005,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.20399999999999999},{"interface":"front","average":{"1min":5.7859999999999996,"5min":7.4790000000000001,"15min":11.372999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":70.100999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:33:35 2023","interfaces":[{"interface":"back","average":{"1min":15.648999999999999,"5min":10.077,"15min":4.8390000000000004},"min":{"1min":0.317,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":191.101,"5min":191.101,"15min":191.101},"last":0.22500000000000001},{"interface":"front","average":{"1min":16.170000000000002,"5min":10.291,"15min":4.5869999999999997},"min":{"1min":0.40500000000000003,"5min":0.249,"15min":0.218},"max":{"1min":190.87899999999999,"5min":190.87899999999999,"15min":190.87899999999999},"last":5.3819999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:33:07 2023","interfaces":[{"interface":"back","average":{"1min":7.6360000000000001,"5min":3.6960000000000002,"15min":2.5489999999999999},"min":{"1min":0.37,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":80.584999999999994,"5min":80.584999999999994,"15min":80.584999999999994},"last":2.5030000000000001},{"interface":"front","average":{"1min":7.5880000000000001,"5min":4.8230000000000004,"15min":3.7290000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":80.873999999999995,"5min":80.873999999999995,"15min":80.873999999999995},"last":0.81200000000000006}]},{"osd":4,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.4650000000000001,"15min":1.518},"min":{"1min":0.38,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.4260000000000002,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.536,"5min":1.4490000000000001,"15min":1.4039999999999999},"min":{"1min":0.28799999999999998,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":8.2739999999999991,"5min":8.2739999999999991,"15min":14.460000000000001},"last":0.30099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.802,"15min":4.1050000000000004},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5499999999999998,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.472,"15min":3.7949999999999999},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.4820000000000002,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.751}]},{"osd":6,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.893,"15min":2.012},"min":{"1min":0.38300000000000001,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":6.5750000000000002,"5min":8.8580000000000005,"15min":14.858000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.655,"5min":1.4850000000000001,"15min":1.698},"min":{"1min":0.47599999999999998,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.5140000000000002,"5min":11.476000000000001,"15min":13.48},"last":0.78500000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.5980000000000001,"15min":2.4199999999999999},"min":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":5.3769999999999998,"5min":16.113,"15min":162.399},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.9319999999999999,"5min":1.6819999999999999,"15min":2.1960000000000002},"min":{"1min":0.34899999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":162.27199999999999},"last":0.379}]}]},{"osd":0,"up_from":244,"seq":1047972020289,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2004084,"kb_used_data":1752172,"kb_used_omap":10,"kb_used_meta":251893,"kb_avail":91741068,"statfs":{"total":95995035648,"available":93942853632,"internally_reserved":0,"allocated":1794224128,"data_stored":1793200376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":257938484},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":5.056,"15min":4.625},"min":{"1min":0.215,"5min":0.191,"15min":0.191},"max":{"1min":8.6750000000000007,"5min":139.68000000000001,"15min":139.68000000000001},"last":1.4430000000000001},{"interface":"front","average":{"1min":3.2909999999999999,"5min":4.9800000000000004,"15min":5.7009999999999996},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.409000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":3.984}]},{"osd":2,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":15.375,"5min":7.8479999999999999,"15min":6.5839999999999996},"min":{"1min":0.28499999999999998,"5min":0.216,"15min":0.216},"max":{"1min":132.245,"5min":205.10900000000001,"15min":205.10900000000001},"last":132.245},{"interface":"front","average":{"1min":15.032,"5min":7.9210000000000003,"15min":6.0439999999999996},"min":{"1min":0.46600000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":132.26599999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":132.26599999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:06 2023","interfaces":[{"interface":"back","average":{"1min":6.7779999999999996,"5min":5.0220000000000002,"15min":5.8650000000000002},"min":{"1min":0.245,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":111.745,"5min":111.745,"15min":111.745},"last":4.5549999999999997},{"interface":"front","average":{"1min":9.2940000000000005,"5min":6.0640000000000001,"15min":6.8769999999999998},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":114.205,"5min":114.205,"15min":114.205},"last":1.101}]},{"osd":4,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.617,"15min":4.2699999999999996},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.744999999999999,"5min":29.709,"15min":29.709},"last":3.891},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.2669999999999999,"15min":3.605},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.9630000000000001,"5min":13.872,"15min":13.872},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.302,"15min":3.6909999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0270000000000001,"5min":14.105,"15min":14.105},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.294,"5min":2.4079999999999999,"15min":3.2120000000000002},"min":{"1min":0.441,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.532,"5min":19.846,"15min":19.846},"last":1.1559999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.7400000000000002,"15min":4.4580000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.771000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":3.931},{"interface":"front","average":{"1min":1.175,"5min":2.3300000000000001,"15min":3.7250000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.0650000000000004,"5min":13.82,"15min":13.82},"last":0.433}]},{"osd":7,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.21,"5min":2.6949999999999998,"15min":3.6230000000000002},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":17.523,"5min":19.866,"15min":19.866},"last":1.1299999999999999},{"interface":"front","average":{"1min":3.6720000000000002,"5min":2.899,"15min":4.5090000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":17.792999999999999,"5min":29.77,"15min":29.77},"last":3.9590000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075700,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2662428,"kb_used_data":2194644,"kb_used_omap":7,"kb_used_meta":467640,"kb_avail":91082724,"statfs":{"total":95995035648,"available":93268709376,"internally_reserved":0,"allocated":2247315456,"data_stored":2246374849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":478863441},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,12,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":4.0060000000000002,"15min":4.4710000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":18.210999999999999,"5min":66.784000000000006,"15min":66.784000000000006},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.7639999999999998,"5min":4.3840000000000003,"15min":5.056},"min":{"1min":0.317,"5min":0.252,"15min":0.252},"max":{"1min":7.6109999999999998,"5min":67.251999999999995,"15min":67.251999999999995},"last":1.3400000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":5.1619999999999999,"15min":2.8380000000000001},"min":{"1min":0.309,"5min":0.218,"15min":0.19},"max":{"1min":44.625,"5min":147.20099999999999,"15min":147.20099999999999},"last":5.6630000000000003},{"interface":"front","average":{"1min":2.0950000000000002,"5min":4.2089999999999996,"15min":2.4049999999999998},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":9.7729999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.74399999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:01 2023","interfaces":[{"interface":"back","average":{"1min":10.353999999999999,"5min":4.8970000000000002,"15min":5.1260000000000003},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":138.03299999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":2.044},{"interface":"front","average":{"1min":10.279,"5min":5.444,"15min":6.0099999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":138.095,"5min":138.095,"15min":138.095},"last":2.1099999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":3.1970000000000001,"15min":2.778},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":6.1689999999999996,"5min":110.337,"15min":224.00999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":4.2080000000000002,"5min":3.972,"15min":3.105},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.486999999999998,"5min":110.2,"15min":223.28399999999999},"last":5.5910000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.671,"15min":1.4379999999999999},"min":{"1min":0.44700000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":9.7460000000000004,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.968,"15min":1.6950000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.242,"5min":16.125,"15min":16.125},"last":1.3740000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.0590000000000002,"15min":1.7669999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":6.1920000000000002,"5min":16.183,"15min":16.183},"last":1.3140000000000001},{"interface":"front","average":{"1min":4.3940000000000001,"5min":2.782,"15min":2.113},"min":{"1min":0.48299999999999998,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":29.538,"5min":29.538,"15min":29.538},"last":5.6349999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":7.9089999999999998,"5min":4.1289999999999996,"15min":2.2309999999999999},"min":{"1min":0.41199999999999998,"5min":0.34799999999999998,"15min":0.33400000000000002},"max":{"1min":73.445999999999998,"5min":113.913,"15min":113.913},"last":0.58699999999999997},{"interface":"front","average":{"1min":7.8849999999999998,"5min":4.4939999999999998,"15min":2.577},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":73.475999999999999,"5min":113.937,"15min":113.937},"last":1.4059999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824060,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323240,"kb_used_data":2099232,"kb_used_omap":7,"kb_used_meta":223992,"kb_avail":91421912,"statfs":{"total":95995035648,"available":93616037888,"internally_reserved":0,"allocated":2149613568,"data_stored":2148601322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":229367827},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":4.2039999999999997,"15min":5.0999999999999996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.522,"5min":93.992999999999995,"15min":93.992999999999995},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.387,"5min":3.891,"15min":4.8520000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":5.7729999999999997,"5min":95.623999999999995,"15min":95.623999999999995},"last":0.48399999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":3.7759999999999998,"15min":3.6840000000000002},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.422999999999998,"5min":19.859000000000002,"15min":19.859000000000002},"last":6.258},{"interface":"front","average":{"1min":5.2939999999999996,"5min":3.028,"15min":3.3319999999999999},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.384,"5min":17.384,"15min":17.384},"last":6.617}]},{"osd":2,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":6.6130000000000004,"15min":10.819000000000001},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.9580000000000002,"5min":87.555999999999997,"15min":87.555999999999997},"last":2.6520000000000001},{"interface":"front","average":{"1min":4.806,"5min":8.3859999999999992,"15min":12.356999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.170999999999999,"5min":80.566999999999993,"15min":80.566999999999993},"last":10.243}]},{"osd":4,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.4430000000000001,"15min":1.397},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.6070000000000002,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":2.6909999999999998},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.7570000000000001,"15min":2.7450000000000001},"min":{"1min":0.40100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.384,"5min":19.728000000000002,"15min":19.728000000000002},"last":6.194}]},{"osd":5,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2010000000000001,"15min":1.329},"min":{"1min":0.39400000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":1.218},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.4610000000000001,"15min":1.4399999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6419999999999999,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7810000000000001,"15min":2.7690000000000001},"min":{"1min":0.438,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":14.119,"5min":19.805,"15min":19.805},"last":6.1399999999999997},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.2689999999999999,"15min":1.383},"min":{"1min":0.47999999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9809999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":0.71399999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.7919999999999998,"15min":2.782},"min":{"1min":0.503,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":14.422000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":6.2290000000000001},{"interface":"front","average":{"1min":1.524,"5min":1.2450000000000001,"15min":1.357},"min":{"1min":0.44900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9379999999999997,"5min":6.9379999999999997,"15min":6.9379999999999997},"last":0.55800000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019325,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1467388,"kb_used_data":877236,"kb_used_omap":0,"kb_used_meta":590144,"kb_avail":92277764,"statfs":{"total":95995035648,"available":94492430336,"internally_reserved":0,"allocated":898289664,"data_stored":897728814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":604307456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:37 2023","interfaces":[{"interface":"back","average":{"1min":10.459,"5min":5.0209999999999999,"15min":8.7919999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":234.203,"5min":234.203,"15min":234.203},"last":3.234},{"interface":"front","average":{"1min":11.193,"5min":5.8780000000000001,"15min":8.5860000000000003},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":234.19,"5min":234.19,"15min":234.19},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":1.9470000000000001,"15min":2.9390000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":13.461,"5min":13.461,"15min":182.035},"last":3.7170000000000001},{"interface":"front","average":{"1min":1.833,"5min":1.7549999999999999,"15min":2.6800000000000002},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.255000000000001,"5min":10.255000000000001,"15min":181.73099999999999},"last":8.593}]},{"osd":2,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3380000000000001,"5min":2.4700000000000002,"15min":1.8080000000000001},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":40.311,"5min":40.311,"15min":40.311},"last":3.6669999999999998},{"interface":"front","average":{"1min":3.3690000000000002,"5min":2.3300000000000001,"15min":1.714},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":40.268999999999998,"5min":40.268999999999998,"15min":40.268999999999998},"last":3.9369999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.125,"15min":2.3029999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":29.856999999999999,"5min":29.856999999999999,"15min":29.856999999999999},"last":7.3460000000000001},{"interface":"front","average":{"1min":10.365,"5min":3.5270000000000001,"15min":2.5379999999999998},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":160.06200000000001,"5min":160.06200000000001,"15min":160.06200000000001},"last":10.82}]},{"osd":5,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.58999999999999997,"15min":1.022},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.23999999999999999},"max":{"1min":0.55200000000000005,"5min":6.742,"15min":13.941000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.61399999999999999,"15min":1.0469999999999999},"min":{"1min":0.32400000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":5.25,"5min":5.4000000000000004,"15min":11.904999999999999},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:32:54 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.58399999999999996,"15min":1.0800000000000001},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.76900000000000002,"5min":4.2729999999999997,"15min":8.2599999999999998},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.75900000000000001,"15min":1.143},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.72399999999999998,"5min":8.3109999999999999,"15min":13.845000000000001},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.64900000000000002,"15min":1.2549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.51100000000000001,"5min":6.798,"15min":16.265000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.64700000000000002,"15min":1.175},"min":{"1min":0.316,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.627,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":0.69799999999999995}]}]},{"osd":5,"up_from":0,"seq":154618823097,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1773797376,"data_stored":1773797376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2998734848,"data_stored":2998730776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2240286720,"data_stored":2240282648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2131427328,"data_stored":2131427328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":892538880,"data_stored":892534808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":892534784,"data_stored":892534784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:33:57.742 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:58.108 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:58.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:33:58.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:33:58.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:58.371 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:33:58.371 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:33:58.400 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:33:58.648 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:33:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:58.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:33:58.758 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:33:58.785 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:58.831 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:33:58.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:58.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:33:58.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:33:58.998 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:33:59.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[180297]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5898/156702 objects degraded (3.764%); 3180/156702 objects misplaced (2.029%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:59.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3886892272' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.259 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:33:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[186706]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5898/156702 objects degraded (3.764%); 3180/156702 objects misplaced (2.029%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3886892272' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:33:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:59 smithi152 ceph-mon[156819]: pgmap v1076: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 3.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5898/156702 objects degraded (3.764%); 3180/156702 objects misplaced (2.029%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:33:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3886892272' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:33:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:33:59.928 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:33:59.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:33:59.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:00.035 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:00.063 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:00.116 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:00.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:00.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:00.255 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:00.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:00.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[180297]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[180297]: osdmap e299: 8 total, 8 up, 6 in 2023-06-10T21:34:00.269 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[180297]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.306 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:00.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:00.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:00.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:00 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:00 smithi152 ceph-mon[156819]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:00 smithi152 ceph-mon[156819]: osdmap e299: 8 total, 8 up, 6 in 2023-06-10T21:34:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:00 smithi152 ceph-mon[156819]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:00.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[186706]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:00.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[186706]: osdmap e299: 8 total, 8 up, 6 in 2023-06-10T21:34:00.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:00 smithi064 ceph-mon[186706]: failed to encode map e299 with expected crc 2023-06-10T21:34:00.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:34:00.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:34:00.981 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:01.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:01.109 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:01.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:01.346 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:01.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:01.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:01.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:01.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:01 smithi064 ceph-mon[180297]: pgmap v1078: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 5898/152304 objects degraded (3.873%); 3055/152304 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:01.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5898/152304 objects degraded (3.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:01.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:01 smithi064 ceph-mon[186706]: pgmap v1078: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 5898/152304 objects degraded (3.873%); 3055/152304 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:01.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5898/152304 objects degraded (3.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:01.575 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:01 smithi152 ceph-mon[156819]: pgmap v1078: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 569 op/s; 5898/152304 objects degraded (3.873%); 3055/152304 objects misplaced (2.006%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5898/152304 objects degraded (3.873%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:01.737 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:01.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:01.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:01.871 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:01.912 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:01.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:02.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:02.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:02.090 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:02.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:02.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:02.225 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:02.277 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:02.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:02.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:02.482 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:02.576 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:02.584 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:02.644 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:02.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:02.761 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:02.790 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:02.836 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:02.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:02.895 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:02] "GET /metrics HTTP/1.1" 200 224062 "" "Prometheus/2.33.4" 2023-06-10T21:34:02.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:02.968 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:03.014 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:03.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:03.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:03.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:34:03.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:03.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[180297]: pgmap v1079: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 5898/149193 objects degraded (3.953%); 3055/149193 objects misplaced (2.048%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:34:03.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:03.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:03.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[186706]: pgmap v1079: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 5898/149193 objects degraded (3.953%); 3055/149193 objects misplaced (2.048%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:34:03.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:03.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:03.582 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:03.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:03 smithi152 ceph-mon[156819]: pgmap v1079: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 5898/149193 objects degraded (3.953%); 3055/149193 objects misplaced (2.048%); 1.8 MiB/s, 29 objects/s recovering 2023-06-10T21:34:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:03.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:03.723 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:03.771 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:03.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:03.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:03.908 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:03.958 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:03.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:04.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:04.094 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:04.141 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:04.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:04.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:04.280 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:04.332 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:04.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:04.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:04.467 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:04.512 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:04.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:04.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:04.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:05.455 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:34:05.455 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:34:05.506 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:34:05.590 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:34:05.627 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:34:05.645 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:34:05.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:05 smithi064 ceph-mon[180297]: pgmap v1080: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5908/145638 objects degraded (4.057%); 2901/145638 objects misplaced (1.992%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:05 smithi064 ceph-mon[186706]: pgmap v1080: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5908/145638 objects degraded (4.057%); 2901/145638 objects misplaced (1.992%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:05.663 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:34:05.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:34:05.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.683 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.683 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.683 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:34:05.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:05 smithi152 ceph-mon[156819]: pgmap v1080: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 3.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 5908/145638 objects degraded (4.057%); 2901/145638 objects misplaced (1.992%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:06.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:06.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:06.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:06.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:06.416 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:06.465 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:06.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:06.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:06.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1325350506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:06.629 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5908/145638 objects degraded (4.057%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:06.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1325350506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:06.630 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5908/145638 objects degraded (4.057%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:06.637 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:06.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1325350506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5908/145638 objects degraded (4.057%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:07.215 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:07.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:07.325 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:07.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:07 smithi064 ceph-mon[180297]: pgmap v1081: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 5908/144240 objects degraded (4.096%); 2901/144240 objects misplaced (2.011%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:07.518 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:07 smithi064 ceph-mon[186706]: pgmap v1081: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 5908/144240 objects degraded (4.096%); 2901/144240 objects misplaced (2.011%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:07.596 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:07.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:07 smithi152 ceph-mon[156819]: pgmap v1081: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 5908/144240 objects degraded (4.096%); 2901/144240 objects misplaced (2.011%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:07.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:07.733 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:07.782 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:07.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:07.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:07.991 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:08.019 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:08.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:08.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:08.187 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:34:08.187 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:34:08.187 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:34:08.187 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:34:08.187 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:34:08.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:08.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:08.464 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:34:08.493 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:34:09.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:34:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:09 smithi064 ceph-mon[186706]: pgmap v1082: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5908/141363 objects degraded (4.179%); 2901/141363 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:09.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:09 smithi064 ceph-mon[186706]: from='client.36318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:09.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:09 smithi064 ceph-mon[180297]: pgmap v1082: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5908/141363 objects degraded (4.179%); 2901/141363 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:09.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:09 smithi064 ceph-mon[180297]: from='client.36318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:09 smithi152 ceph-mon[156819]: pgmap v1082: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 511 op/s; 5908/141363 objects degraded (4.179%); 2901/141363 objects misplaced (2.052%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:09 smithi152 ceph-mon[156819]: from='client.36318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:10.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:10.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:10.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:10.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:10.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:34:10.877 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:10.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:10.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:11.014 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:11.062 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:11.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:11.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:11.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:11 smithi064 ceph-mon[186706]: pgmap v1083: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 5917/136263 objects degraded (4.342%); 2716/136263 objects misplaced (1.993%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:34:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/136263 objects degraded (4.342%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:11 smithi064 ceph-mon[180297]: pgmap v1083: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 5917/136263 objects degraded (4.342%); 2716/136263 objects misplaced (1.993%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:34:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/136263 objects degraded (4.342%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:11 smithi152 ceph-mon[156819]: pgmap v1083: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 5917/136263 objects degraded (4.342%); 2716/136263 objects misplaced (1.993%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:34:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/136263 objects degraded (4.342%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:12.269 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:12.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:12.299 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1082,"stamp":"2023-06-10T21:34:07.824732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3087079334,"num_objects":47121,"num_object_clones":0,"num_object_copies":141363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5908,"num_objects_misplaced":2901,"num_objects_unfound":0,"num_objects_dirty":47121,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":121706,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1827,"num_bytes_recovered":118759122,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106640,"ondisk_log_size":106640,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12112584,"kb_used_data":9391832,"kb_used_omap":30,"kb_used_meta":2720097,"kb_avail":550358328,"statfs":{"total":575970213888,"available":563566927872,"internally_reserved":0,"allocated":9617235968,"data_stored":9612185044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2785379366},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-335085568,"num_objects":-5113,"num_object_clones":0,"num_object_copies":-15339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":-279,"num_objects_unfound":0,"num_objects_dirty":-5113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5113,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":269,"num_bytes_recovered":17629184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003027"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":292,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.511429+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:33:44.511429+0000","last_peered":"2023-06-10T21:33:44.511429+0000","last_clean":"2023-06-10T21:33:44.511429+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:33:44.511429+0000","last_undegraded":"2023-06-10T21:33:44.511429+0000","last_fullsized":"2023-06-10T21:33:44.511429+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":233,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:56.520390+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:33:56.520390+0000","last_peered":"2023-06-10T21:33:56.520390+0000","last_clean":"2023-06-10T21:33:56.520390+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:33:56.520390+0000","last_undegraded":"2023-06-10T21:33:56.520390+0000","last_fullsized":"2023-06-10T21:33:56.520390+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:46.513843+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:33:46.513843+0000","last_peered":"2023-06-10T21:33:46.513843+0000","last_clean":"2023-06-10T21:33:46.513843+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:33:46.513843+0000","last_undegraded":"2023-06-10T21:33:46.513843+0000","last_fullsized":"2023-06-10T21:33:46.513843+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":155,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508617+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:33:44.508617+0000","last_peered":"2023-06-10T21:33:44.508617+0000","last_clean":"2023-06-10T21:33:44.508617+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:33:44.508617+0000","last_undegraded":"2023-06-10T21:33:44.508617+0000","last_fullsized":"2023-06-10T21:33:44.508617+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":273,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:46.513783+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:33:46.513783+0000","last_peered":"2023-06-10T21:33:46.513783+0000","last_clean":"2023-06-10T21:33:46.513783+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:33:46.513783+0000","last_undegraded":"2023-06-10T21:33:46.513783+0000","last_fullsized":"2023-06-10T21:33:46.513783+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":265,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.512922+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:33:44.512922+0000","last_peered":"2023-06-10T21:33:44.512922+0000","last_clean":"2023-06-10T21:33:44.512922+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:33:44.512922+0000","last_undegraded":"2023-06-10T21:33:44.512922+0000","last_fullsized":"2023-06-10T21:33:44.512922+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":155,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.513007+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:33:44.513007+0000","last_peered":"2023-06-10T21:33:44.513007+0000","last_clean":"2023-06-10T21:33:44.513007+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:33:44.513007+0000","last_undegraded":"2023-06-10T21:33:44.513007+0000","last_fullsized":"2023-06-10T21:33:44.513007+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":247,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.490698+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:33:44.490698+0000","last_peered":"2023-06-10T21:33:44.490698+0000","last_clean":"2023-06-10T21:33:44.490698+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:33:44.490698+0000","last_undegraded":"2023-06-10T21:33:44.490698+0000","last_fullsized":"2023-06-10T21:33:44.490698+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":20,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.504058+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:33:44.504058+0000","last_peered":"2023-06-10T21:33:44.504058+0000","last_clean":"2023-06-10T21:33:44.504058+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:33:44.504058+0000","last_undegraded":"2023-06-10T21:33:44.504058+0000","last_fullsized":"2023-06-10T21:33:44.504058+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":21,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.489823+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:33:44.489823+0000","last_peered":"2023-06-10T21:33:44.489823+0000","last_clean":"2023-06-10T21:33:44.489823+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:33:44.489823+0000","last_undegraded":"2023-06-10T21:33:44.489823+0000","last_fullsized":"2023-06-10T21:33:44.489823+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":26,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503849+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:33:44.503849+0000","last_peered":"2023-06-10T21:33:44.503849+0000","last_clean":"2023-06-10T21:33:44.503849+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:33:44.503849+0000","last_undegraded":"2023-06-10T21:33:44.503849+0000","last_fullsized":"2023-06-10T21:33:44.503849+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":261,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.511323+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:33:44.511323+0000","last_peered":"2023-06-10T21:33:44.511323+0000","last_clean":"2023-06-10T21:33:44.511323+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:33:44.511323+0000","last_undegraded":"2023-06-10T21:33:44.511323+0000","last_fullsized":"2023-06-10T21:33:44.511323+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":169,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503871+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:33:44.503871+0000","last_peered":"2023-06-10T21:33:44.503871+0000","last_clean":"2023-06-10T21:33:44.503871+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:33:44.503871+0000","last_undegraded":"2023-06-10T21:33:44.503871+0000","last_fullsized":"2023-06-10T21:33:44.503871+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":30,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503760+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:33:44.503760+0000","last_peered":"2023-06-10T21:33:44.503760+0000","last_clean":"2023-06-10T21:33:44.503760+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:33:44.503760+0000","last_undegraded":"2023-06-10T21:33:44.503760+0000","last_fullsized":"2023-06-10T21:33:44.503760+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":298,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503860+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:33:44.503860+0000","last_peered":"2023-06-10T21:33:44.503860+0000","last_clean":"2023-06-10T21:33:44.503860+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:33:44.503860+0000","last_undegraded":"2023-06-10T21:33:44.503860+0000","last_fullsized":"2023-06-10T21:33:44.503860+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":233,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.487977+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:33:44.487977+0000","last_peered":"2023-06-10T21:33:44.487977+0000","last_clean":"2023-06-10T21:33:44.487977+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:33:44.487977+0000","last_undegraded":"2023-06-10T21:33:44.487977+0000","last_fullsized":"2023-06-10T21:33:44.487977+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":155,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503747+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:33:44.503747+0000","last_peered":"2023-06-10T21:33:44.503747+0000","last_clean":"2023-06-10T21:33:44.503747+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:33:44.503747+0000","last_undegraded":"2023-06-10T21:33:44.503747+0000","last_fullsized":"2023-06-10T21:33:44.503747+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.504041+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:33:44.504041+0000","last_peered":"2023-06-10T21:33:44.504041+0000","last_clean":"2023-06-10T21:33:44.504041+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:33:44.504041+0000","last_undegraded":"2023-06-10T21:33:44.504041+0000","last_fullsized":"2023-06-10T21:33:44.504041+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":275,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503770+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:33:44.503770+0000","last_peered":"2023-06-10T21:33:44.503770+0000","last_clean":"2023-06-10T21:33:44.503770+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:33:44.503770+0000","last_undegraded":"2023-06-10T21:33:44.503770+0000","last_fullsized":"2023-06-10T21:33:44.503770+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":155,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508442+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:33:44.508442+0000","last_peered":"2023-06-10T21:33:44.508442+0000","last_clean":"2023-06-10T21:33:44.508442+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:33:44.508442+0000","last_undegraded":"2023-06-10T21:33:44.508442+0000","last_fullsized":"2023-06-10T21:33:44.508442+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":274,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.500026+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:33:44.500026+0000","last_peered":"2023-06-10T21:33:44.500026+0000","last_clean":"2023-06-10T21:33:44.500026+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:33:44.500026+0000","last_undegraded":"2023-06-10T21:33:44.500026+0000","last_fullsized":"2023-06-10T21:33:44.500026+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:46.513991+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:33:46.513991+0000","last_peered":"2023-06-10T21:33:46.513991+0000","last_clean":"2023-06-10T21:33:46.513991+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:33:46.513991+0000","last_undegraded":"2023-06-10T21:33:46.513991+0000","last_fullsized":"2023-06-10T21:33:46.513991+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"299'7555","reported_seq":7595,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:02.883891+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:34:02.883891+0000","last_peered":"2023-06-10T21:34:02.883891+0000","last_clean":"2023-06-10T21:34:02.883891+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:34:02.883891+0000","last_undegraded":"2023-06-10T21:34:02.883891+0000","last_fullsized":"2023-06-10T21:34:02.883891+0000","mapping_epoch":269,"log_start":"277'1100","ondisk_log_start":"277'1100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6455,"log_dups_size":0,"ondisk_log_size":6455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195887104,"num_objects":2989,"num_object_clones":0,"num_object_copies":8967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7555,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":24,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.504228+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:33:44.504228+0000","last_peered":"2023-06-10T21:33:44.504228+0000","last_clean":"2023-06-10T21:33:44.504228+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:33:44.504228+0000","last_undegraded":"2023-06-10T21:33:44.504228+0000","last_fullsized":"2023-06-10T21:33:44.504228+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":20,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503484+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:33:44.503484+0000","last_peered":"2023-06-10T21:33:44.503484+0000","last_clean":"2023-06-10T21:33:44.503484+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:33:44.503484+0000","last_undegraded":"2023-06-10T21:33:44.503484+0000","last_fullsized":"2023-06-10T21:33:44.503484+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"299'7587","reported_seq":7627,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:02.869197+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:34:02.869197+0000","last_peered":"2023-06-10T21:34:02.869197+0000","last_clean":"2023-06-10T21:34:02.869197+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:34:02.869197+0000","last_undegraded":"2023-06-10T21:34:02.869197+0000","last_fullsized":"2023-06-10T21:34:02.869197+0000","mapping_epoch":269,"log_start":"277'1200","ondisk_log_start":"277'1200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197197824,"num_objects":3009,"num_object_clones":0,"num_object_copies":9027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7587,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":24,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.504109+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:33:44.504109+0000","last_peered":"2023-06-10T21:33:44.504109+0000","last_clean":"2023-06-10T21:33:44.504109+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:33:44.504109+0000","last_undegraded":"2023-06-10T21:33:44.504109+0000","last_fullsized":"2023-06-10T21:33:44.504109+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.490340+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:33:44.490340+0000","last_peered":"2023-06-10T21:33:44.490340+0000","last_clean":"2023-06-10T21:33:44.490340+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:33:44.490340+0000","last_undegraded":"2023-06-10T21:33:44.490340+0000","last_fullsized":"2023-06-10T21:33:44.490340+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"299'7758","reported_seq":7801,"reported_epoch":299,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:02.899823+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:34:02.899823+0000","last_peered":"2023-06-10T21:34:02.899823+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:34:02.899823+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"278'1300","ondisk_log_start":"278'1300","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206307328,"num_objects":3148,"num_object_clones":0,"num_object_copies":9444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":3148},{"shards":"0,3","objects":-2246}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":306,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503945+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:33:44.503945+0000","last_peered":"2023-06-10T21:33:44.503945+0000","last_clean":"2023-06-10T21:33:44.503945+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:33:44.503945+0000","last_undegraded":"2023-06-10T21:33:44.503945+0000","last_fullsized":"2023-06-10T21:33:44.503945+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":20,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503665+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:33:44.503665+0000","last_peered":"2023-06-10T21:33:44.503665+0000","last_clean":"2023-06-10T21:33:44.503665+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:33:44.503665+0000","last_undegraded":"2023-06-10T21:33:44.503665+0000","last_fullsized":"2023-06-10T21:33:44.503665+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"299'7513","reported_seq":7551,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:04.646818+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:34:04.646818+0000","last_peered":"2023-06-10T21:34:04.646818+0000","last_clean":"2023-06-10T21:34:04.646818+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:34:04.646818+0000","last_undegraded":"2023-06-10T21:34:04.646818+0000","last_fullsized":"2023-06-10T21:34:04.646818+0000","mapping_epoch":269,"log_start":"276'100","ondisk_log_start":"276'100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7413,"log_dups_size":0,"ondisk_log_size":7413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194707456,"num_objects":2971,"num_object_clones":0,"num_object_copies":8913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7513,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488119+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:33:44.488119+0000","last_peered":"2023-06-10T21:33:44.488119+0000","last_clean":"2023-06-10T21:33:44.488119+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:33:44.488119+0000","last_undegraded":"2023-06-10T21:33:44.488119+0000","last_fullsized":"2023-06-10T21:33:44.488119+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":122,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.499908+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:33:44.499908+0000","last_peered":"2023-06-10T21:33:44.499908+0000","last_clean":"2023-06-10T21:33:44.499908+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:33:44.499908+0000","last_undegraded":"2023-06-10T21:33:44.499908+0000","last_fullsized":"2023-06-10T21:33:44.499908+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"299'7486","reported_seq":11103,"reported_epoch":299,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:03.571805+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:34:03.571805+0000","last_peered":"2023-06-10T21:34:03.571805+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:34:03.571805+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7486,"log_dups_size":0,"ondisk_log_size":7486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":455,"num_objects_misplaced":2901,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7486,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1786,"num_bytes_recovered":116981760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-454},{"shards":"1,2,3","objects":2901}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":250,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.489464+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:33:44.489464+0000","last_peered":"2023-06-10T21:33:44.489464+0000","last_clean":"2023-06-10T21:33:44.489464+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:33:44.489464+0000","last_undegraded":"2023-06-10T21:33:44.489464+0000","last_fullsized":"2023-06-10T21:33:44.489464+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488188+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:33:44.488188+0000","last_peered":"2023-06-10T21:33:44.488188+0000","last_clean":"2023-06-10T21:33:44.488188+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:33:44.488188+0000","last_undegraded":"2023-06-10T21:33:44.488188+0000","last_fullsized":"2023-06-10T21:33:44.488188+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"299'7725","reported_seq":7764,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:03.517557+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:34:03.517557+0000","last_peered":"2023-06-10T21:34:03.517557+0000","last_clean":"2023-06-10T21:34:03.517557+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:34:03.517557+0000","last_undegraded":"2023-06-10T21:34:03.517557+0000","last_fullsized":"2023-06-10T21:34:03.517557+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7725,"log_dups_size":0,"ondisk_log_size":7725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198508544,"num_objects":3029,"num_object_clones":0,"num_object_copies":9087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7725,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":73,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.489656+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:33:44.489656+0000","last_peered":"2023-06-10T21:33:44.489656+0000","last_clean":"2023-06-10T21:33:44.489656+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:33:44.489656+0000","last_undegraded":"2023-06-10T21:33:44.489656+0000","last_fullsized":"2023-06-10T21:33:44.489656+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":290,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.509004+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:33:44.509004+0000","last_peered":"2023-06-10T21:33:44.509004+0000","last_clean":"2023-06-10T21:33:44.509004+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:33:44.509004+0000","last_undegraded":"2023-06-10T21:33:44.509004+0000","last_fullsized":"2023-06-10T21:33:44.509004+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"299'7602","reported_seq":7642,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.447944+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:34:06.447944+0000","last_peered":"2023-06-10T21:34:06.447944+0000","last_clean":"2023-06-10T21:34:06.447944+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:34:06.447944+0000","last_undegraded":"2023-06-10T21:34:06.447944+0000","last_fullsized":"2023-06-10T21:34:06.447944+0000","mapping_epoch":269,"log_start":"280'1800","ondisk_log_start":"280'1800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192413696,"num_objects":2936,"num_object_clones":0,"num_object_copies":8808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7602,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":276,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508118+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:33:44.508118+0000","last_peered":"2023-06-10T21:33:44.508118+0000","last_clean":"2023-06-10T21:33:44.508118+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:33:44.508118+0000","last_undegraded":"2023-06-10T21:33:44.508118+0000","last_fullsized":"2023-06-10T21:33:44.508118+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":73,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488619+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:33:44.488619+0000","last_peered":"2023-06-10T21:33:44.488619+0000","last_clean":"2023-06-10T21:33:44.488619+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:33:44.488619+0000","last_undegraded":"2023-06-10T21:33:44.488619+0000","last_fullsized":"2023-06-10T21:33:44.488619+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"299'7671","reported_seq":7711,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.488569+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:34:06.488569+0000","last_peered":"2023-06-10T21:34:06.488569+0000","last_clean":"2023-06-10T21:34:06.488569+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:34:06.488569+0000","last_undegraded":"2023-06-10T21:34:06.488569+0000","last_fullsized":"2023-06-10T21:34:06.488569+0000","mapping_epoch":269,"log_start":"280'1900","ondisk_log_start":"280'1900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5771,"log_dups_size":0,"ondisk_log_size":5771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7671,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":253,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.491223+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:33:44.491223+0000","last_peered":"2023-06-10T21:33:44.491223+0000","last_clean":"2023-06-10T21:33:44.491223+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:33:44.491223+0000","last_undegraded":"2023-06-10T21:33:44.491223+0000","last_fullsized":"2023-06-10T21:33:44.491223+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":209,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488974+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:33:44.488974+0000","last_peered":"2023-06-10T21:33:44.488974+0000","last_clean":"2023-06-10T21:33:44.488974+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:33:44.488974+0000","last_undegraded":"2023-06-10T21:33:44.488974+0000","last_fullsized":"2023-06-10T21:33:44.488974+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"299'7503","reported_seq":7544,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:03.562068+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:34:03.562068+0000","last_peered":"2023-06-10T21:34:03.562068+0000","last_clean":"2023-06-10T21:34:03.562068+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:34:03.562068+0000","last_undegraded":"2023-06-10T21:34:03.562068+0000","last_fullsized":"2023-06-10T21:34:03.562068+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191365144,"num_objects":2921,"num_object_clones":0,"num_object_copies":8763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2921,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7503,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":122,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.500426+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:33:44.500426+0000","last_peered":"2023-06-10T21:33:44.500426+0000","last_clean":"2023-06-10T21:33:44.500426+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:33:44.500426+0000","last_undegraded":"2023-06-10T21:33:44.500426+0000","last_fullsized":"2023-06-10T21:33:44.500426+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.489047+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:33:44.489047+0000","last_peered":"2023-06-10T21:33:44.489047+0000","last_clean":"2023-06-10T21:33:44.489047+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:33:44.489047+0000","last_undegraded":"2023-06-10T21:33:44.489047+0000","last_fullsized":"2023-06-10T21:33:44.489047+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":142,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503605+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:33:44.503605+0000","last_peered":"2023-06-10T21:33:44.503605+0000","last_clean":"2023-06-10T21:33:44.503605+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:33:44.503605+0000","last_undegraded":"2023-06-10T21:33:44.503605+0000","last_fullsized":"2023-06-10T21:33:44.503605+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"299'7638","reported_seq":7678,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.483837+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:34:06.483837+0000","last_peered":"2023-06-10T21:34:06.483837+0000","last_clean":"2023-06-10T21:34:06.483837+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:34:06.483837+0000","last_undegraded":"2023-06-10T21:34:06.483837+0000","last_fullsized":"2023-06-10T21:34:06.483837+0000","mapping_epoch":269,"log_start":"280'1800","ondisk_log_start":"280'1800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190709760,"num_objects":2910,"num_object_clones":0,"num_object_copies":8730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7638,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":168,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503678+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:33:44.503678+0000","last_peered":"2023-06-10T21:33:44.503678+0000","last_clean":"2023-06-10T21:33:44.503678+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:33:44.503678+0000","last_undegraded":"2023-06-10T21:33:44.503678+0000","last_fullsized":"2023-06-10T21:33:44.503678+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":242,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.500060+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:33:44.500060+0000","last_peered":"2023-06-10T21:33:44.500060+0000","last_clean":"2023-06-10T21:33:44.500060+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:33:44.500060+0000","last_undegraded":"2023-06-10T21:33:44.500060+0000","last_fullsized":"2023-06-10T21:33:44.500060+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":132,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:54.869684+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:33:54.869684+0000","last_peered":"2023-06-10T21:33:54.869684+0000","last_clean":"2023-06-10T21:33:54.869684+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:33:54.869684+0000","last_undegraded":"2023-06-10T21:33:54.869684+0000","last_fullsized":"2023-06-10T21:33:54.869684+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"299'7683","reported_seq":7722,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.498429+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:34:06.498429+0000","last_peered":"2023-06-10T21:34:06.498429+0000","last_clean":"2023-06-10T21:34:06.498429+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:34:06.498429+0000","last_undegraded":"2023-06-10T21:34:06.498429+0000","last_fullsized":"2023-06-10T21:34:06.498429+0000","mapping_epoch":269,"log_start":"280'1900","ondisk_log_start":"280'1900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186580992,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7683,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":143,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:54.869613+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:33:54.869613+0000","last_peered":"2023-06-10T21:33:54.869613+0000","last_clean":"2023-06-10T21:33:54.869613+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:33:54.869613+0000","last_undegraded":"2023-06-10T21:33:54.869613+0000","last_fullsized":"2023-06-10T21:33:54.869613+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.507951+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:33:44.507951+0000","last_peered":"2023-06-10T21:33:44.507951+0000","last_clean":"2023-06-10T21:33:44.507951+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:33:44.507951+0000","last_undegraded":"2023-06-10T21:33:44.507951+0000","last_fullsized":"2023-06-10T21:33:44.507951+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":191,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.513075+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:33:44.513075+0000","last_peered":"2023-06-10T21:33:44.513075+0000","last_clean":"2023-06-10T21:33:44.513075+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:33:44.513075+0000","last_undegraded":"2023-06-10T21:33:44.513075+0000","last_fullsized":"2023-06-10T21:33:44.513075+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"299'7531","reported_seq":7571,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.390288+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:34:06.390288+0000","last_peered":"2023-06-10T21:34:06.390288+0000","last_clean":"2023-06-10T21:34:06.390288+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:34:06.390288+0000","last_undegraded":"2023-06-10T21:34:06.390288+0000","last_fullsized":"2023-06-10T21:34:06.390288+0000","mapping_epoch":269,"log_start":"280'1700","ondisk_log_start":"280'1700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5831,"log_dups_size":0,"ondisk_log_size":5831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185532416,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7531,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":322,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.512583+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:33:44.512583+0000","last_peered":"2023-06-10T21:33:44.512583+0000","last_clean":"2023-06-10T21:33:44.512583+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:33:44.512583+0000","last_undegraded":"2023-06-10T21:33:44.512583+0000","last_fullsized":"2023-06-10T21:33:44.512583+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":462,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503518+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:33:44.503518+0000","last_peered":"2023-06-10T21:33:44.503518+0000","last_clean":"2023-06-10T21:33:44.503518+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:33:44.503518+0000","last_undegraded":"2023-06-10T21:33:44.503518+0000","last_fullsized":"2023-06-10T21:33:44.503518+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":236,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.489873+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:33:44.489873+0000","last_peered":"2023-06-10T21:33:44.489873+0000","last_clean":"2023-06-10T21:33:44.489873+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:33:44.489873+0000","last_undegraded":"2023-06-10T21:33:44.489873+0000","last_fullsized":"2023-06-10T21:33:44.489873+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.504016+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:33:44.504016+0000","last_peered":"2023-06-10T21:33:44.504016+0000","last_clean":"2023-06-10T21:33:44.504016+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:33:44.504016+0000","last_undegraded":"2023-06-10T21:33:44.504016+0000","last_fullsized":"2023-06-10T21:33:44.504016+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"299'7673","reported_seq":7713,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:06.490654+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:34:06.490654+0000","last_peered":"2023-06-10T21:34:06.490654+0000","last_clean":"2023-06-10T21:34:06.490654+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:34:06.490654+0000","last_undegraded":"2023-06-10T21:34:06.490654+0000","last_fullsized":"2023-06-10T21:34:06.490654+0000","mapping_epoch":269,"log_start":"280'1900","ondisk_log_start":"280'1900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194445312,"num_objects":2967,"num_object_clones":0,"num_object_copies":8901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7673,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":318,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503573+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:33:44.503573+0000","last_peered":"2023-06-10T21:33:44.503573+0000","last_clean":"2023-06-10T21:33:44.503573+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:33:44.503573+0000","last_undegraded":"2023-06-10T21:33:44.503573+0000","last_fullsized":"2023-06-10T21:33:44.503573+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":137,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.502778+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:33:44.502778+0000","last_peered":"2023-06-10T21:33:44.502778+0000","last_clean":"2023-06-10T21:33:44.502778+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:33:44.502778+0000","last_undegraded":"2023-06-10T21:33:44.502778+0000","last_fullsized":"2023-06-10T21:33:44.502778+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508364+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:33:44.508364+0000","last_peered":"2023-06-10T21:33:44.508364+0000","last_clean":"2023-06-10T21:33:44.508364+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:33:44.508364+0000","last_undegraded":"2023-06-10T21:33:44.508364+0000","last_fullsized":"2023-06-10T21:33:44.508364+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"299'7520","reported_seq":7560,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:03.435991+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:34:03.435991+0000","last_peered":"2023-06-10T21:34:03.435991+0000","last_clean":"2023-06-10T21:34:03.435991+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:34:03.435991+0000","last_undegraded":"2023-06-10T21:34:03.435991+0000","last_fullsized":"2023-06-10T21:34:03.435991+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7520,"log_dups_size":0,"ondisk_log_size":7520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197656576,"num_objects":3016,"num_object_clones":0,"num_object_copies":9048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7520,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":303,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503947+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:33:44.503947+0000","last_peered":"2023-06-10T21:33:44.503947+0000","last_clean":"2023-06-10T21:33:44.503947+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:33:44.503947+0000","last_undegraded":"2023-06-10T21:33:44.503947+0000","last_fullsized":"2023-06-10T21:33:44.503947+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:54.869702+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:33:54.869702+0000","last_peered":"2023-06-10T21:33:54.869702+0000","last_clean":"2023-06-10T21:33:54.869702+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:33:54.869702+0000","last_undegraded":"2023-06-10T21:33:54.869702+0000","last_fullsized":"2023-06-10T21:33:54.869702+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":118,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.500173+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:33:44.500173+0000","last_peered":"2023-06-10T21:33:44.500173+0000","last_clean":"2023-06-10T21:33:44.500173+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:33:44.500173+0000","last_undegraded":"2023-06-10T21:33:44.500173+0000","last_fullsized":"2023-06-10T21:33:44.500173+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"299'7523","reported_seq":7561,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:04.603699+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:34:04.603699+0000","last_peered":"2023-06-10T21:34:04.603699+0000","last_clean":"2023-06-10T21:34:04.603699+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:34:04.603699+0000","last_undegraded":"2023-06-10T21:34:04.603699+0000","last_fullsized":"2023-06-10T21:34:04.603699+0000","mapping_epoch":269,"log_start":"276'200","ondisk_log_start":"276'200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7323,"log_dups_size":0,"ondisk_log_size":7323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7523,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":276,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488787+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:33:44.488787+0000","last_peered":"2023-06-10T21:33:44.488787+0000","last_clean":"2023-06-10T21:33:44.488787+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:33:44.488787+0000","last_undegraded":"2023-06-10T21:33:44.488787+0000","last_fullsized":"2023-06-10T21:33:44.488787+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":73,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488500+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:33:44.488500+0000","last_peered":"2023-06-10T21:33:44.488500+0000","last_clean":"2023-06-10T21:33:44.488500+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:33:44.488500+0000","last_undegraded":"2023-06-10T21:33:44.488500+0000","last_fullsized":"2023-06-10T21:33:44.488500+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":257,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.491139+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:33:44.491139+0000","last_peered":"2023-06-10T21:33:44.491139+0000","last_clean":"2023-06-10T21:33:44.491139+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:33:44.491139+0000","last_undegraded":"2023-06-10T21:33:44.491139+0000","last_fullsized":"2023-06-10T21:33:44.491139+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"299'7477","reported_seq":7515,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:04.652862+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:34:04.652862+0000","last_peered":"2023-06-10T21:34:04.652862+0000","last_clean":"2023-06-10T21:34:04.652862+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:34:04.652862+0000","last_undegraded":"2023-06-10T21:34:04.652862+0000","last_fullsized":"2023-06-10T21:34:04.652862+0000","mapping_epoch":269,"log_start":"276'100","ondisk_log_start":"276'100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7377,"log_dups_size":0,"ondisk_log_size":7377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188678144,"num_objects":2879,"num_object_clones":0,"num_object_copies":8637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7477,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503890+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:33:44.503890+0000","last_peered":"2023-06-10T21:33:44.503890+0000","last_clean":"2023-06-10T21:33:44.503890+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:33:44.503890+0000","last_undegraded":"2023-06-10T21:33:44.503890+0000","last_fullsized":"2023-06-10T21:33:44.503890+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":126,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508540+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:33:44.508540+0000","last_peered":"2023-06-10T21:33:44.508540+0000","last_clean":"2023-06-10T21:33:44.508540+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:33:44.508540+0000","last_undegraded":"2023-06-10T21:33:44.508540+0000","last_fullsized":"2023-06-10T21:33:44.508540+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.499995+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:33:44.499995+0000","last_peered":"2023-06-10T21:33:44.499995+0000","last_clean":"2023-06-10T21:33:44.499995+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:33:44.499995+0000","last_undegraded":"2023-06-10T21:33:44.499995+0000","last_fullsized":"2023-06-10T21:33:44.499995+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":122,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.500259+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:33:44.500259+0000","last_peered":"2023-06-10T21:33:44.500259+0000","last_clean":"2023-06-10T21:33:44.500259+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:33:44.500259+0000","last_undegraded":"2023-06-10T21:33:44.500259+0000","last_fullsized":"2023-06-10T21:33:44.500259+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":140,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.508745+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:33:44.508745+0000","last_peered":"2023-06-10T21:33:44.508745+0000","last_clean":"2023-06-10T21:33:44.508745+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:33:44.508745+0000","last_undegraded":"2023-06-10T21:33:44.508745+0000","last_fullsized":"2023-06-10T21:33:44.508745+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":24,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.503683+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:33:44.503683+0000","last_peered":"2023-06-10T21:33:44.503683+0000","last_clean":"2023-06-10T21:33:44.503683+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:33:44.503683+0000","last_undegraded":"2023-06-10T21:33:44.503683+0000","last_fullsized":"2023-06-10T21:33:44.503683+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":181,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:56.520315+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:33:56.520315+0000","last_peered":"2023-06-10T21:33:56.520315+0000","last_clean":"2023-06-10T21:33:56.520315+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:33:56.520315+0000","last_undegraded":"2023-06-10T21:33:56.520315+0000","last_fullsized":"2023-06-10T21:33:56.520315+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":77,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:56.520372+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:33:56.520372+0000","last_peered":"2023-06-10T21:33:56.520372+0000","last_clean":"2023-06-10T21:33:56.520372+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:33:56.520372+0000","last_undegraded":"2023-06-10T21:33:56.520372+0000","last_fullsized":"2023-06-10T21:33:56.520372+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":237,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:56.520271+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:33:56.520271+0000","last_peered":"2023-06-10T21:33:56.520271+0000","last_clean":"2023-06-10T21:33:56.520271+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:33:56.520271+0000","last_undegraded":"2023-06-10T21:33:56.520271+0000","last_fullsized":"2023-06-10T21:33:56.520271+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":282,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.488333+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:33:44.488333+0000","last_peered":"2023-06-10T21:33:44.488333+0000","last_clean":"2023-06-10T21:33:44.488333+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:33:44.488333+0000","last_undegraded":"2023-06-10T21:33:44.488333+0000","last_fullsized":"2023-06-10T21:33:44.488333+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":286,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:46.513819+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:33:46.513819+0000","last_peered":"2023-06-10T21:33:46.513819+0000","last_clean":"2023-06-10T21:33:46.513819+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:33:46.513819+0000","last_undegraded":"2023-06-10T21:33:46.513819+0000","last_fullsized":"2023-06-10T21:33:46.513819+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":187,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:44.513154+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:33:44.513154+0000","last_peered":"2023-06-10T21:33:44.513154+0000","last_clean":"2023-06-10T21:33:44.513154+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:33:44.513154+0000","last_undegraded":"2023-06-10T21:33:44.513154+0000","last_fullsized":"2023-06-10T21:33:44.513154+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":32,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-10T21:33:58.511920+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:33:58.511920+0000","last_peered":"2023-06-10T21:33:58.511920+0000","last_clean":"2023-06-10T21:33:58.511920+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:33:58.511920+0000","last_undegraded":"2023-06-10T21:33:58.511920+0000","last_fullsized":"2023-06-10T21:33:58.511920+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3086483480,"num_objects":47097,"num_object_clones":0,"num_object_copies":141291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5908,"num_objects_misplaced":2901,"num_objects_unfound":0,"num_objects_dirty":47097,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121445,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1786,"num_bytes_recovered":116981760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9547296768,"data_stored":9547284552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106445,"ondisk_log_size":106445,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430644,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1326600,"kb_used_data":750016,"kb_used_omap":0,"kb_used_meta":576576,"kb_avail":92418552,"statfs":{"total":95995035648,"available":94636597248,"internally_reserved":0,"allocated":768016384,"data_stored":767452672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":590413824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":5.6710000000000003,"15min":8.3620000000000001},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":52.100000000000001,"5min":184.83699999999999,"15min":184.83699999999999},"last":2.008},{"interface":"front","average":{"1min":3.677,"5min":5.641,"15min":8.5800000000000001},"min":{"1min":0.31900000000000001,"5min":0.223,"15min":0.223},"max":{"1min":51.146999999999998,"5min":185.58000000000001,"15min":185.58000000000001},"last":2.0539999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":3.4220000000000002,"5min":2.5800000000000001,"15min":3.0350000000000001},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.21299999999999999},"max":{"1min":49.732999999999997,"5min":87.417000000000002,"15min":130.19399999999999},"last":4.7009999999999996},{"interface":"front","average":{"1min":4.883,"5min":2.7210000000000001,"15min":3.573},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":50.060000000000002,"5min":87.438000000000002,"15min":212.20599999999999},"last":2.3300000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:33:23 2023","interfaces":[{"interface":"back","average":{"1min":11.731999999999999,"5min":5.4450000000000003,"15min":3.7770000000000001},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":172.33600000000001,"5min":172.33600000000001,"15min":172.33600000000001},"last":1.7789999999999999},{"interface":"front","average":{"1min":11.782999999999999,"5min":5.3540000000000001,"15min":4.4139999999999997},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":172.185,"5min":172.185,"15min":212.13399999999999},"last":6.266}]},{"osd":3,"last update":"Sat Jun 10 21:33:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":2.8660000000000001,"15min":2.6219999999999999},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.9489999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":4.5350000000000001},{"interface":"front","average":{"1min":1.393,"5min":2.4969999999999999,"15min":2.0539999999999998},"min":{"1min":0.318,"5min":0.31,"15min":0.31},"max":{"1min":5.7869999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":2.198}]},{"osd":4,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.45700000000000002,"15min":0.72199999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.5049999999999999,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":1.0369999999999999},"min":{"1min":0.29599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.0310000000000001,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.54500000000000004,"15min":1.827},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1829999999999998,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47399999999999998,"15min":1.6220000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.92100000000000004,"5min":1.7,"15min":212.23699999999999},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57199999999999995,"15min":1.083},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":3.2559999999999998,"5min":3.2559999999999998,"15min":13.073},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.94399999999999995},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":1.1879999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.58599999999999997}]}]},{"osd":6,"up_from":0,"seq":180388626872,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566315,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3155820,"kb_used_data":2556964,"kb_used_omap":4,"kb_used_meta":598715,"kb_avail":90589332,"statfs":{"total":95995035648,"available":92763475968,"internally_reserved":0,"allocated":2618331136,"data_stored":2617382555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":613085070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3789999999999996,"5min":7.5979999999999999,"15min":11.686},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":70.254000000000005,"5min":154.31399999999999,"15min":154.31399999999999},"last":4.0030000000000001},{"interface":"front","average":{"1min":5.7859999999999996,"5min":7.4790000000000001,"15min":11.372999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":70.100999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":4.0739999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:33:35 2023","interfaces":[{"interface":"back","average":{"1min":15.648999999999999,"5min":10.077,"15min":4.8390000000000004},"min":{"1min":0.317,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":191.101,"5min":191.101,"15min":191.101},"last":0.441},{"interface":"front","average":{"1min":16.170000000000002,"5min":10.291,"15min":4.5869999999999997},"min":{"1min":0.40500000000000003,"5min":0.249,"15min":0.218},"max":{"1min":190.87899999999999,"5min":190.87899999999999,"15min":190.87899999999999},"last":7.6269999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:33:07 2023","interfaces":[{"interface":"back","average":{"1min":7.6360000000000001,"5min":3.6960000000000002,"15min":2.5489999999999999},"min":{"1min":0.37,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":80.584999999999994,"5min":80.584999999999994,"15min":80.584999999999994},"last":2.9590000000000001},{"interface":"front","average":{"1min":7.5880000000000001,"5min":4.8230000000000004,"15min":3.7290000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":80.873999999999995,"5min":80.873999999999995,"15min":80.873999999999995},"last":4.1029999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.4650000000000001,"15min":1.518},"min":{"1min":0.38,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.4260000000000002,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.536,"5min":1.4490000000000001,"15min":1.4039999999999999},"min":{"1min":0.28799999999999998,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":8.2739999999999991,"5min":8.2739999999999991,"15min":14.460000000000001},"last":4.0389999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5750000000000002,"5min":1.802,"15min":4.1050000000000004},"min":{"1min":0.35599999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5499999999999998,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.472,"15min":3.7949999999999999},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.4820000000000002,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":1.893,"15min":2.012},"min":{"1min":0.38300000000000001,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":6.5750000000000002,"5min":8.8580000000000005,"15min":14.858000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.655,"5min":1.4850000000000001,"15min":1.698},"min":{"1min":0.47599999999999998,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.5140000000000002,"5min":11.476000000000001,"15min":13.48},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:33:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":1.5980000000000001,"15min":2.4199999999999999},"min":{"1min":0.41099999999999998,"5min":0.36699999999999999,"15min":0.32600000000000001},"max":{"1min":5.3769999999999998,"5min":16.113,"15min":162.399},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.9319999999999999,"5min":1.6819999999999999,"15min":2.1960000000000002},"min":{"1min":0.34899999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.7629999999999999,"5min":9.7629999999999999,"15min":162.27199999999999},"last":4.5369999999999999}]}]},{"osd":0,"up_from":244,"seq":1047972020291,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821884,"kb_used_data":1578164,"kb_used_omap":10,"kb_used_meta":243701,"kb_avail":91923268,"statfs":{"total":95995035648,"available":94129426432,"internally_reserved":0,"allocated":1616039936,"data_stored":1615015890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":249549876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":5.056,"15min":4.625},"min":{"1min":0.215,"5min":0.191,"15min":0.191},"max":{"1min":8.6750000000000007,"5min":139.68000000000001,"15min":139.68000000000001},"last":2.8860000000000001},{"interface":"front","average":{"1min":3.2909999999999999,"5min":4.9800000000000004,"15min":5.7009999999999996},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.409000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.629}]},{"osd":2,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":15.375,"5min":7.8479999999999999,"15min":6.5839999999999996},"min":{"1min":0.28499999999999998,"5min":0.216,"15min":0.216},"max":{"1min":132.245,"5min":205.10900000000001,"15min":205.10900000000001},"last":6.5270000000000001},{"interface":"front","average":{"1min":15.032,"5min":7.9210000000000003,"15min":6.0439999999999996},"min":{"1min":0.46600000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":132.26599999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":1.208}]},{"osd":3,"last update":"Sat Jun 10 21:33:06 2023","interfaces":[{"interface":"back","average":{"1min":6.7779999999999996,"5min":5.0220000000000002,"15min":5.8650000000000002},"min":{"1min":0.245,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":111.745,"5min":111.745,"15min":111.745},"last":6.4729999999999999},{"interface":"front","average":{"1min":9.2940000000000005,"5min":6.0640000000000001,"15min":6.8769999999999998},"min":{"1min":0.316,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":114.205,"5min":114.205,"15min":114.205},"last":0.76200000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.617,"15min":4.2699999999999996},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.744999999999999,"5min":29.709,"15min":29.709},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.2669999999999999,"15min":3.605},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.9630000000000001,"5min":13.872,"15min":13.872},"last":6.5640000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.302,"15min":3.6909999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0270000000000001,"5min":14.105,"15min":14.105},"last":6.6239999999999997},{"interface":"front","average":{"1min":2.294,"5min":2.4079999999999999,"15min":3.2120000000000002},"min":{"1min":0.441,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.532,"5min":19.846,"15min":19.846},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.7400000000000002,"15min":4.4580000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.771000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.175,"5min":2.3300000000000001,"15min":3.7250000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.0650000000000004,"5min":13.82,"15min":13.82},"last":6.5979999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.21,"5min":2.6949999999999998,"15min":3.6230000000000002},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":17.523,"5min":19.866,"15min":19.866},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.6720000000000002,"5min":2.899,"15min":4.5090000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":17.792999999999999,"5min":29.77,"15min":29.77},"last":0.82499999999999996}]}]},{"osd":2,"up_from":120,"seq":515396075703,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2395428,"kb_used_data":1913500,"kb_used_omap":7,"kb_used_meta":481592,"kb_avail":91349724,"statfs":{"total":95995035648,"available":93542117376,"internally_reserved":0,"allocated":1959424000,"data_stored":1958483099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":493150289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":4.0060000000000002,"15min":4.4710000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":18.210999999999999,"5min":66.784000000000006,"15min":66.784000000000006},"last":13.622999999999999},{"interface":"front","average":{"1min":2.7639999999999998,"5min":4.3840000000000003,"15min":5.056},"min":{"1min":0.317,"5min":0.252,"15min":0.252},"max":{"1min":7.6109999999999998,"5min":67.251999999999995,"15min":67.251999999999995},"last":14.172000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":5.1619999999999999,"15min":2.8380000000000001},"min":{"1min":0.309,"5min":0.218,"15min":0.19},"max":{"1min":44.625,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.254},{"interface":"front","average":{"1min":2.0950000000000002,"5min":4.2089999999999996,"15min":2.4049999999999998},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":9.7729999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:33:01 2023","interfaces":[{"interface":"back","average":{"1min":10.353999999999999,"5min":4.8970000000000002,"15min":5.1260000000000003},"min":{"1min":0.32400000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":138.03299999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.626},{"interface":"front","average":{"1min":10.279,"5min":5.444,"15min":6.0099999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":138.095,"5min":138.095,"15min":138.095},"last":6.3550000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":3.1970000000000001,"15min":2.778},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":6.1689999999999996,"5min":110.337,"15min":224.00999999999999},"last":4.5599999999999996},{"interface":"front","average":{"1min":4.2080000000000002,"5min":3.972,"15min":3.105},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.486999999999998,"5min":110.2,"15min":223.28399999999999},"last":0.29999999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.671,"15min":1.4379999999999999},"min":{"1min":0.44700000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":9.7460000000000004,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.968,"15min":1.6950000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.242,"5min":16.125,"15min":16.125},"last":4.5860000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.0590000000000002,"15min":1.7669999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":6.1920000000000002,"5min":16.183,"15min":16.183},"last":4.6130000000000004},{"interface":"front","average":{"1min":4.3940000000000001,"5min":2.782,"15min":2.113},"min":{"1min":0.48299999999999998,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":29.538,"5min":29.538,"15min":29.538},"last":0.33000000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":7.9089999999999998,"5min":4.1289999999999996,"15min":2.2309999999999999},"min":{"1min":0.41199999999999998,"5min":0.34799999999999998,"15min":0.33400000000000002},"max":{"1min":73.445999999999998,"5min":113.913,"15min":113.913},"last":0.435},{"interface":"front","average":{"1min":7.8849999999999998,"5min":4.4939999999999998,"15min":2.577},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":73.475999999999999,"5min":113.937,"15min":113.937},"last":4.5199999999999996}]}]},{"osd":3,"up_from":250,"seq":1073741824063,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2058928,"kb_used_data":1838888,"kb_used_omap":7,"kb_used_meta":219896,"kb_avail":91686224,"statfs":{"total":95995035648,"available":93886693376,"internally_reserved":0,"allocated":1883021312,"data_stored":1882008772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":225173523},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5680000000000001,"5min":4.2039999999999997,"15min":5.0999999999999996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":14.522,"5min":93.992999999999995,"15min":93.992999999999995},"last":1.681},{"interface":"front","average":{"1min":1.387,"5min":3.891,"15min":4.8520000000000003},"min":{"1min":0.246,"5min":0.216,"15min":0.216},"max":{"1min":5.7729999999999997,"5min":95.623999999999995,"15min":95.623999999999995},"last":1.1830000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":3.7759999999999998,"15min":3.6840000000000002},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":17.422999999999998,"5min":19.859000000000002,"15min":19.859000000000002},"last":5.9710000000000001},{"interface":"front","average":{"1min":5.2939999999999996,"5min":3.028,"15min":3.3319999999999999},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":17.384,"5min":17.384,"15min":17.384},"last":1.71}]},{"osd":2,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":6.6130000000000004,"15min":10.819000000000001},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.9580000000000002,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":4.806,"5min":8.3859999999999992,"15min":12.356999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.170999999999999,"5min":80.566999999999993,"15min":80.566999999999993},"last":1.53}]},{"osd":4,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.4430000000000001,"15min":1.397},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":3.6070000000000002,"5min":9.2330000000000005,"15min":9.2330000000000005},"last":1.157},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.7570000000000001,"15min":2.7450000000000001},"min":{"1min":0.40100000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.384,"5min":19.728000000000002,"15min":19.728000000000002},"last":1.5780000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.514,"5min":1.2010000000000001,"15min":1.329},"min":{"1min":0.39400000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":1.5600000000000001},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.4610000000000001,"15min":1.4399999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.6419999999999999,"5min":9.2599999999999998,"15min":9.2599999999999998},"last":1.1240000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0670000000000002,"5min":2.7810000000000001,"15min":2.7690000000000001},"min":{"1min":0.438,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":14.119,"5min":19.805,"15min":19.805},"last":1.716},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.2689999999999999,"15min":1.383},"min":{"1min":0.47999999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9809999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":1.645}]},{"osd":7,"last update":"Sat Jun 10 21:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":2.7919999999999998,"15min":2.782},"min":{"1min":0.503,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":14.422000000000001,"5min":19.777000000000001,"15min":19.777000000000001},"last":1.6180000000000001},{"interface":"front","average":{"1min":1.524,"5min":1.2450000000000001,"15min":1.357},"min":{"1min":0.44900000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9379999999999997,"5min":6.9379999999999997,"15min":6.9379999999999997},"last":1.595}]}]},{"osd":4,"up_from":30,"seq":128849019328,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353924,"kb_used_data":754300,"kb_used_omap":0,"kb_used_meta":599616,"kb_avail":92391228,"statfs":{"total":95995035648,"available":94608617472,"internally_reserved":0,"allocated":772403200,"data_stored":771842056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":614006784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:37 2023","interfaces":[{"interface":"back","average":{"1min":10.459,"5min":5.0209999999999999,"15min":8.7919999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":234.203,"5min":234.203,"15min":234.203},"last":1.119},{"interface":"front","average":{"1min":11.193,"5min":5.8780000000000001,"15min":8.5860000000000003},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":234.19,"5min":234.19,"15min":234.19},"last":0.85299999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:33:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8969999999999998,"5min":1.9470000000000001,"15min":2.9390000000000001},"min":{"1min":0.314,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":13.461,"5min":13.461,"15min":182.035},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.833,"5min":1.7549999999999999,"15min":2.6800000000000002},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.255000000000001,"5min":10.255000000000001,"15min":181.73099999999999},"last":10.151}]},{"osd":2,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3380000000000001,"5min":2.4700000000000002,"15min":1.8080000000000001},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":40.311,"5min":40.311,"15min":40.311},"last":0.99299999999999999},{"interface":"front","average":{"1min":3.3690000000000002,"5min":2.3300000000000001,"15min":1.714},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":40.268999999999998,"5min":40.268999999999998,"15min":40.268999999999998},"last":2.4849999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":3.024,"5min":2.125,"15min":2.3029999999999999},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":29.856999999999999,"5min":29.856999999999999,"15min":29.856999999999999},"last":2.9009999999999998},{"interface":"front","average":{"1min":10.365,"5min":3.5270000000000001,"15min":2.5379999999999998},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":160.06200000000001,"5min":160.06200000000001,"15min":160.06200000000001},"last":3.0049999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.58999999999999997,"15min":1.022},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.23999999999999999},"max":{"1min":0.55200000000000005,"5min":6.742,"15min":13.941000000000001},"last":0.37},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.61399999999999999,"15min":1.0469999999999999},"min":{"1min":0.32400000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":5.25,"5min":5.4000000000000004,"15min":11.904999999999999},"last":0.33100000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50800000000000001,"15min":1.0309999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25600000000000001},"max":{"1min":1.7210000000000001,"5min":3.077,"15min":8.2599999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.63500000000000001,"15min":1.1000000000000001},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":1.71,"5min":6.8300000000000001,"15min":13.845000000000001},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.64900000000000002,"15min":1.2549999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.51100000000000001,"5min":6.798,"15min":16.265000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.64700000000000002,"15min":1.175},"min":{"1min":0.316,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.627,"5min":8.8960000000000008,"15min":9.5060000000000002},"last":0.39600000000000002}]}]},{"osd":5,"up_from":0,"seq":154618823100,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1595867136,"data_stored":1595867136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2610565120,"data_stored":2610561048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1947471872,"data_stored":1947467800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1865482240,"data_stored":1865482240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":764874752,"data_stored":764870680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":763035648,"data_stored":763035648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:34:12.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:34:12.389 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:12.418 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:12.466 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:12.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:12.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:12.602 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:12.682 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:12.828 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:12.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:12.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:12.968 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:13.017 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:13.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:13.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:13.131 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:12] "GET /metrics HTTP/1.1" 200 224073 "" "Prometheus/2.33.4" 2023-06-10T21:34:13.208 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:13.250 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:13.386 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:13 smithi064 ceph-mon[180297]: pgmap v1084: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 5917/133041 objects degraded (4.448%); 2716/133041 objects misplaced (2.041%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:13.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:13.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:13.654 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:13 smithi152 ceph-mon[156819]: pgmap v1084: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 5917/133041 objects degraded (4.448%); 2716/133041 objects misplaced (2.041%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:13.657 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:13.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:13 smithi064 ceph-mon[186706]: pgmap v1084: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 534 op/s; 5917/133041 objects degraded (4.448%); 2716/133041 objects misplaced (2.041%); 1.7 MiB/s, 26 objects/s recovering 2023-06-10T21:34:13.706 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:13.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:13.814 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:13.913 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:14.424 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:34:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1390557108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1390557108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:14.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1390557108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:15.135 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:15.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:15.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:15.273 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:15.295 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:15.402 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:15.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: pgmap v1085: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 5917/129339 objects degraded (4.575%); 2532/129339 objects misplaced (1.958%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: osdmap e300: 8 total, 8 up, 6 in 2023-06-10T21:34:15.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[186706]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: pgmap v1085: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 5917/129339 objects degraded (4.575%); 2532/129339 objects misplaced (1.958%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: osdmap e300: 8 total, 8 up, 6 in 2023-06-10T21:34:15.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:15 smithi064 ceph-mon[180297]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:15.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: pgmap v1085: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 5917/129339 objects degraded (4.575%); 2532/129339 objects misplaced (1.958%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:15.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:15.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:15.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: osdmap e300: 8 total, 8 up, 6 in 2023-06-10T21:34:15.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:15 smithi152 ceph-mon[156819]: failed to encode map e300 with expected crc 2023-06-10T21:34:15.536 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:15.585 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:15.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:15.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:15.716 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:15.758 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:15.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:15.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:15.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:15.896 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:15.947 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:15.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:16.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:16.151 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:16.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/129339 objects degraded (4.575%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:16.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:16.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/129339 objects degraded (4.575%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/129339 objects degraded (4.575%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:16.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:16.816 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:17.396 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.25280.0:49 2.2 2:4e99cc3e:::rbd_mirror_snapshot_schedule:head [omap-get-vals] snapc 0=[] RETRY=2 ondisk+retry+read+known_if_redirected+supports_pool_eio e244)", 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:34:17.397 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.90965127200002, 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0624303289999999, 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.25280", 2023-06-10T21:34:17.398 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2947718283", 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 49 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479548+0000", 2023-06-10T21:34:17.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479556+0000", 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3620000000000006e-06 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479566+0000", 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5880000000000004e-06 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479573+0000", 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.046e-06 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487169+0000", 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075960059999999998 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487178+0000", 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7660000000000004e-06 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541881+0000", 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.054703094 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541980+0000", 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8992000000000004e-05 2023-06-10T21:34:17.406 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111488 10.4 10:2ebf3782:::benchmark_data_smithi064_312713_object20732:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e244)", 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.90706315, 2023-06-10T21:34:17.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058806785, 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111488 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.408 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482137+0000", 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:17.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482142+0000", 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482152+0000", 2023-06-10T21:34:17.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:34:17.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482162+0000", 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0344e-05 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482209+0000", 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7237000000000002e-05 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482218+0000", 2023-06-10T21:34:17.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2040000000000003e-06 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525337+0000", 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.043118188 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525473+0000", 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136673 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525610+0000", 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136231 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532648+0000", 2023-06-10T21:34:17.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070388239999999999 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540822+0000", 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008173441 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540860+0000", 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7815999999999998e-05 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540944+0000", 2023-06-10T21:34:17.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4839999999999994e-05 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.420 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111524 10.4 10:203e818a:::benchmark_data_smithi064_312713_object20768:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.90702578100002, 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0589736080000001, 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.421 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111524 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:34:17.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482178+0000", 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2090000000000001e-06 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482184+0000", 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6080000000000001e-06 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482192+0000", 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:34:17.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5090000000000003e-05 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6470000000000004e-06 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532673+0000", 2023-06-10T21:34:17.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0504169059999999 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532795+0000", 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122521 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532918+0000", 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012313 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538256+0000", 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053373800000000001 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541023+0000", 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002766967 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541067+0000", 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4270000000000001e-05 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541149+0000", 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1736000000000005e-05 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111526 10.4 10:21f5efce:::benchmark_data_smithi064_312713_object20770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.90700251999999, 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0590798290000001, 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.434 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111526 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:34:17.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482201+0000", 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6919999999999998e-06 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482207+0000", 2023-06-10T21:34:17.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1460000000000001e-06 2023-06-10T21:34:17.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482221+0000", 2023-06-10T21:34:17.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3876999999999999e-05 2023-06-10T21:34:17.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:34:17.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7775999999999997e-05 2023-06-10T21:34:17.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482285+0000", 2023-06-10T21:34:17.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1879999999999999e-06 2023-06-10T21:34:17.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535227+0000", 2023-06-10T21:34:17.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0529423769999999 2023-06-10T21:34:17.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535306+0000", 2023-06-10T21:34:17.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8954000000000003e-05 2023-06-10T21:34:17.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:34:17.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535404+0000", 2023-06-10T21:34:17.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8250000000000003e-05 2023-06-10T21:34:17.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538295+0000", 2023-06-10T21:34:17.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028906679999999999 2023-06-10T21:34:17.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541164+0000", 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028689599999999998 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541202+0000", 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8213999999999997e-05 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541278+0000", 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6106999999999995e-05 2023-06-10T21:34:17.449 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.450 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.450 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.450 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111533 10.4 10:20aecf0c:::benchmark_data_smithi064_312713_object20777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:34:17.451 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:34:17.451 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.906956353, 2023-06-10T21:34:17.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0641654469999999, 2023-06-10T21:34:17.452 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.452 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.452 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.453 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111533 2023-06-10T21:34:17.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.453 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:34:17.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:34:17.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:34:17.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.315e-06 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:34:17.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-06-10T21:34:17.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482263+0000", 2023-06-10T21:34:17.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.063e-06 2023-06-10T21:34:17.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482314+0000", 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1063999999999998e-05 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482321+0000", 2023-06-10T21:34:17.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8589999999999999e-06 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538345+0000", 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.056023894 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538495+0000", 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149498 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538638+0000", 2023-06-10T21:34:17.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014375100000000001 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541316+0000", 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026775509999999998 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546267+0000", 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004951324 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546317+0000", 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9876999999999997e-05 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546410+0000", 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2786000000000002e-05 2023-06-10T21:34:17.466 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111555 10.4 10:24e69e32:::benchmark_data_smithi064_312713_object20799:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:34:17.467 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 343.90692314, 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0654074570000001, 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.468 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111555 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482277+0000", 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482280+0000", 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5069999999999999e-06 2023-06-10T21:34:17.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482286+0000", 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4180000000000002e-06 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482292+0000", 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3399999999999997e-06 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483443+0000", 2023-06-10T21:34:17.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151383 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483452+0000", 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1309999999999995e-06 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541485+0000", 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058032919 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541643+0000", 2023-06-10T21:34:17.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158212 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541783+0000", 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140111 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.544945+0000", 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031613840000000002 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547575+0000", 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026302230000000001 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547622+0000", 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7429999999999998e-05 2023-06-10T21:34:17.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547685+0000", 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2838999999999999e-05 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111621 10.a 10:56721995:::benchmark_data_smithi064_312713_object20865:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.83192437100001, 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96906832399999998, 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.480 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111621 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557279+0000", 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.503e-06 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557289+0000", 2023-06-10T21:34:17.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0631999999999999e-05 2023-06-10T21:34:17.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557294+0000", 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7770000000000002e-06 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557348+0000", 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3779999999999997e-05 2023-06-10T21:34:17.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557364+0000", 2023-06-10T21:34:17.488 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6325e-05 2023-06-10T21:34:17.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.516904+0000", 2023-06-10T21:34:17.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95953967500000004 2023-06-10T21:34:17.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517083+0000", 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017900899999999999 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517231+0000", 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526113+0000", 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088829010000000003 2023-06-10T21:34:17.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526152+0000", 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8665999999999999e-05 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526194+0000", 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2048000000000001e-05 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526241+0000", 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7145999999999999e-05 2023-06-10T21:34:17.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526345+0000", 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103491 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.494 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111622 10.a 10:5c194bad:::benchmark_data_smithi064_312713_object20866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.82906143600002, 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97161865400000003, 2023-06-10T21:34:17.495 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111622 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.496 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560138+0000", 2023-06-10T21:34:17.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560142+0000", 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4659999999999999e-06 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560149+0000", 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0509999999999999e-06 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560155+0000", 2023-06-10T21:34:17.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8510000000000001e-06 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560208+0000", 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2911000000000002e-05 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560219+0000", 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0859999999999999e-05 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520194+0000", 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95997470299999998 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520350+0000", 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015635700000000001 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520521+0000", 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017028799999999999 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526160+0000", 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056392309999999998 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531566+0000", 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054064539999999998 2023-06-10T21:34:17.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531604+0000", 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7854e-05 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531651+0000", 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7215000000000001e-05 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531758+0000", 2023-06-10T21:34:17.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010678100000000001 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111629 10.a 10:56a08505:::benchmark_data_smithi064_312713_object20873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:34:17.508 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.82168211800001, 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97554071600000003, 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.509 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111629 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.511 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567517+0000", 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567525+0000", 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.994e-06 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567537+0000", 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1922999999999999e-05 2023-06-10T21:34:17.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567545+0000", 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5900000000000002e-06 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567602+0000", 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7065999999999999e-05 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567613+0000", 2023-06-10T21:34:17.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523367+0000", 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95575399000000005 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523517+0000", 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014977799999999999 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523672+0000", 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015482899999999999 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538459+0000", 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0147876 2023-06-10T21:34:17.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540585+0000", 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021253919999999998 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542925+0000", 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023401250000000002 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542957+0000", 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2283999999999998e-05 2023-06-10T21:34:17.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543059+0000", 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102527 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.521 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111634 10.a 10:5fd06ea2:::benchmark_data_smithi064_312713_object20878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.81248301599999, 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.969888847, 2023-06-10T21:34:17.522 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111634 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576717+0000", 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576720+0000", 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6340000000000001e-06 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576727+0000", 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.216e-06 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576733+0000", 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4319999999999998e-06 2023-06-10T21:34:17.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576789+0000", 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6227999999999998e-05 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576800+0000", 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0572e-05 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526379+0000", 2023-06-10T21:34:17.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94957964100000003 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526493+0000", 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113999 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526618+0000", 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495299999999999 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540594+0000", 2023-06-10T21:34:17.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013976018999999999 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546331+0000", 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005736568 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546401+0000", 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0585000000000003e-05 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.533 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546466+0000", 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4929000000000001e-05 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546607+0000", 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014047500000000001 2023-06-10T21:34:17.534 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111642 10.a 10:5fdb9826:::benchmark_data_smithi064_312713_object20886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:34:17.535 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.79973238999997, 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96564841499999998, 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111642 2023-06-10T21:34:17.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589467+0000", 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589472+0000", 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0490000000000003e-06 2023-06-10T21:34:17.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589483+0000", 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0729e-05 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589490+0000", 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589573+0000", 2023-06-10T21:34:17.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3007e-05 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589585+0000", 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1836e-05 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528718+0000", 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93913300899999996 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528872+0000", 2023-06-10T21:34:17.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154108 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.529036+0000", 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164017 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540601+0000", 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011564677000000001 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546679+0000", 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060784560000000003 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555011+0000", 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083312960000000002 2023-06-10T21:34:17.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555038+0000", 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7032000000000001e-05 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555117+0000", 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9177000000000004e-05 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111646 10.a 10:58b447f4:::benchmark_data_smithi064_312713_object20890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.79644997899999, 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96249102799999997, 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.549 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111646 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592750+0000", 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592755+0000", 2023-06-10T21:34:17.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6600000000000003e-06 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592764+0000", 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4059999999999994e-06 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592769+0000", 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2590000000000002e-06 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592816+0000", 2023-06-10T21:34:17.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7280000000000001e-05 2023-06-10T21:34:17.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592823+0000", 2023-06-10T21:34:17.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1199999999999996e-06 2023-06-10T21:34:17.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531787+0000", 2023-06-10T21:34:17.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93896378800000002 2023-06-10T21:34:17.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531899+0000", 2023-06-10T21:34:17.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111659 2023-06-10T21:34:17.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.532018+0000", 2023-06-10T21:34:17.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119275 2023-06-10T21:34:17.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540608+0000", 2023-06-10T21:34:17.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085904399999999995 2023-06-10T21:34:17.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546763+0000", 2023-06-10T21:34:17.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061551050000000001 2023-06-10T21:34:17.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555159+0000", 2023-06-10T21:34:17.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008395563 2023-06-10T21:34:17.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555184+0000", 2023-06-10T21:34:17.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5528e-05 2023-06-10T21:34:17.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555242+0000", 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7454000000000003e-05 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.568 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111648 10.a 10:574e8e24:::benchmark_data_smithi064_312713_object20892:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.79190435999999, 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.958220356, 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.569 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111648 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:34:17.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597295+0000", 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597300+0000", 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0309999999999998e-06 2023-06-10T21:34:17.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597309+0000", 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8089999999999999e-06 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597316+0000", 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6630000000000004e-06 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597417+0000", 2023-06-10T21:34:17.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100995 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597432+0000", 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5517000000000001e-05 2023-06-10T21:34:17.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534105+0000", 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93667285600000005 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534213+0000", 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010740599999999999 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534345+0000", 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132714 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540617+0000", 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062720099999999997 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.552679+0000", 2023-06-10T21:34:17.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012062154 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555339+0000", 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026591420000000002 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555401+0000", 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2757999999999998e-05 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555517+0000", 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115498 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.583 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111681 10.a 10:560529bb:::benchmark_data_smithi064_312713_object20925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:34:17.583 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:34:17.583 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 342.74580339300002, 2023-06-10T21:34:17.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91238480600000005, 2023-06-10T21:34:17.584 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111681 2023-06-10T21:34:17.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:34:17.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643395+0000", 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643408+0000", 2023-06-10T21:34:17.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2588e-05 2023-06-10T21:34:17.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643431+0000", 2023-06-10T21:34:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3011999999999999e-05 2023-06-10T21:34:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643444+0000", 2023-06-10T21:34:17.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3047000000000001e-05 2023-06-10T21:34:17.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643726+0000", 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028253799999999998 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643738+0000", 2023-06-10T21:34:17.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1908e-05 2023-06-10T21:34:17.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536216+0000", 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89247793399999997 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536328+0000", 2023-06-10T21:34:17.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112285 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536430+0000", 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101981 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546177+0000", 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097468780000000005 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555141+0000", 2023-06-10T21:34:17.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089643450000000003 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555638+0000", 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049625199999999998 2023-06-10T21:34:17.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555676+0000", 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8074000000000002e-05 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555782+0000", 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106461 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85152 11.5 11:a61fdc6c:::benchmark_data_smithi064_385725_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879969095999996, 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.711236145, 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.601 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.602 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.602 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.602 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85152 2023-06-10T21:34:17.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.603 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:34:17.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509230+0000", 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:17.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509235+0000", 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4939999999999997e-06 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509247+0000", 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2203e-05 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509256+0000", 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0589999999999992e-06 2023-06-10T21:34:17.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509326+0000", 2023-06-10T21:34:17.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9915999999999999e-05 2023-06-10T21:34:17.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509339+0000", 2023-06-10T21:34:17.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527999999999999e-05 2023-06-10T21:34:17.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.195228+0000", 2023-06-10T21:34:17.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6858897430000002 2023-06-10T21:34:17.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.196887+0000", 2023-06-10T21:34:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016585160000000001 2023-06-10T21:34:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.197074+0000", 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018745799999999999 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205769+0000", 2023-06-10T21:34:17.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086945730000000006 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205810+0000", 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1607000000000003e-05 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217677+0000", 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011866642 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217739+0000", 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2154000000000002e-05 2023-06-10T21:34:17.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220468+0000", 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027286659999999998 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.616 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85191 11.5 11:ad36ddde:::benchmark_data_smithi064_385725_object918:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879914528, 2023-06-10T21:34:17.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7144825799999999, 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85191 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:34:17.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509285+0000", 2023-06-10T21:34:17.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509290+0000", 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.203e-06 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509304+0000", 2023-06-10T21:34:17.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3966e-05 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509314+0000", 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5650000000000007e-06 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509379+0000", 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5468000000000004e-05 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509401+0000", 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1651e-05 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202573+0000", 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.693172154 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202760+0000", 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018716300000000001 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202937+0000", 2023-06-10T21:34:17.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017642100000000001 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217663+0000", 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014726461999999999 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220485+0000", 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028217699999999999 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220695+0000", 2023-06-10T21:34:17.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000209547 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220756+0000", 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0946999999999998e-05 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223769+0000", 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030133080000000001 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.631 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85250 11.5 11:ad3b70c3:::benchmark_data_smithi064_385725_object977:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879880507999999, 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.718711994, 2023-06-10T21:34:17.632 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85250 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.633 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509323+0000", 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.83e-06 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509330+0000", 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.951e-06 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509336+0000", 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1190000000000002e-06 2023-06-10T21:34:17.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509513+0000", 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017721800000000001 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509522+0000", 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.969e-06 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205836+0000", 2023-06-10T21:34:17.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6963141390000001 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206003+0000", 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016704 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206123+0000", 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011994600000000001 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220498+0000", 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014375188000000001 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223827+0000", 2023-06-10T21:34:17.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033290490000000002 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223968+0000", 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014019100000000001 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.224024+0000", 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6718000000000001e-05 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228032+0000", 2023-06-10T21:34:17.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040080999999999997 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85263 11.5 11:a183d882:::benchmark_data_smithi064_385725_object990:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:34:17.645 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879733504000001, 2023-06-10T21:34:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7214953290000001, 2023-06-10T21:34:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.646 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85263 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:34:17.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:17.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509471+0000", 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6689999999999997e-06 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509482+0000", 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0746e-05 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509490+0000", 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4659999999999999e-06 2023-06-10T21:34:17.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509846+0000", 2023-06-10T21:34:17.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035595000000000001 2023-06-10T21:34:17.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509855+0000", 2023-06-10T21:34:17.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-06-10T21:34:17.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209100+0000", 2023-06-10T21:34:17.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6992444710000001 2023-06-10T21:34:17.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209286+0000", 2023-06-10T21:34:17.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018623000000000001 2023-06-10T21:34:17.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209449+0000", 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162953 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220508+0000", 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011058678000000001 2023-06-10T21:34:17.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223930+0000", 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003422671 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228127+0000", 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041968999999999999 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228171+0000", 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3764999999999999e-05 2023-06-10T21:34:17.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.230963+0000", 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002791689 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.661 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85264 11.5 11:a2a1b725:::benchmark_data_smithi064_385725_object991:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879438012999998, 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.724233897, 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.662 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85264 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:34:17.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509762+0000", 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509767+0000", 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.135e-06 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509778+0000", 2023-06-10T21:34:17.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058e-05 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509787+0000", 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8769999999999999e-06 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509887+0000", 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010014900000000001 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509893+0000", 2023-06-10T21:34:17.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:34:17.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214185+0000", 2023-06-10T21:34:17.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.704292031 2023-06-10T21:34:17.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214314+0000", 2023-06-10T21:34:17.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012892599999999999 2023-06-10T21:34:17.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214446+0000", 2023-06-10T21:34:17.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013181 2023-06-10T21:34:17.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220634+0000", 2023-06-10T21:34:17.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061883290000000002 2023-06-10T21:34:17.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228085+0000", 2023-06-10T21:34:17.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074512379999999998 2023-06-10T21:34:17.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231052+0000", 2023-06-10T21:34:17.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029661269999999998 2023-06-10T21:34:17.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231110+0000", 2023-06-10T21:34:17.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8733000000000002e-05 2023-06-10T21:34:17.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.233997+0000", 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028864709999999998 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85269 11.5 11:a8f773e1:::benchmark_data_smithi064_385725_object996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 81.879408498999993, 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7386240339999999, 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:17.681 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85269 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:17.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:17.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:17.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509795+0000", 2023-06-10T21:34:17.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0850000000000001e-06 2023-06-10T21:34:17.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:17.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509802+0000", 2023-06-10T21:34:17.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:34:17.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509810+0000", 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3529999999999995e-06 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510067+0000", 2023-06-10T21:34:17.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000256896 2023-06-10T21:34:17.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:34:17.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510076+0000", 2023-06-10T21:34:17.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0270000000000001e-06 2023-06-10T21:34:17.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234066+0000", 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7239896319999999 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234235+0000", 2023-06-10T21:34:17.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016923200000000001 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234406+0000", 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017128799999999999 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240469+0000", 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006062217 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245113+0000", 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046439960000000001 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:17.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245151+0000", 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8617e-05 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245210+0000", 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9166999999999998e-05 2023-06-10T21:34:17.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.248416+0000", 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032060209999999999 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:17.696 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:34:17.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:17 smithi152 ceph-mon[156819]: pgmap v1087: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 5917/127899 objects degraded (4.626%); 2532/127899 objects misplaced (1.980%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:17.697 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:17 smithi064 ceph-mon[186706]: pgmap v1087: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 5917/127899 objects degraded (4.626%); 2532/127899 objects misplaced (1.980%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:17.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:17 smithi064 ceph-mon[180297]: pgmap v1087: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.6 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 5917/127899 objects degraded (4.626%); 2532/127899 objects misplaced (1.980%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:17.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:17.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:18.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:18.186 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:18.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:34:18.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:18.630 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:19 smithi152 ceph-mon[156819]: pgmap v1088: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 5917/125124 objects degraded (4.729%); 2532/125124 objects misplaced (2.024%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:19.819 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:19 smithi064 ceph-mon[180297]: pgmap v1088: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 5917/125124 objects degraded (4.729%); 2532/125124 objects misplaced (2.024%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:19.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:19 smithi064 ceph-mon[186706]: pgmap v1088: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 540 op/s; 5917/125124 objects degraded (4.729%); 2532/125124 objects misplaced (2.024%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:19.854 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:34:19.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:19.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:20.010 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:20.038 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:20.084 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:20.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:20.194 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:20.221 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:20.267 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:20.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:20.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:20.451 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:20.602 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:20.607 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:20.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:20.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:20.746 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:20.747 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:34:20.791 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:20.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:20.796 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:34:20.850 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:34:20.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:20.946 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:34:20.970 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:34:21.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:34:21.043 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:21.070 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:34:21.124 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:34:21.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.153 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:34:21.192 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:21.195 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:21.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:21.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:21.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:21.475 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:34:21.475 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5917/120282 objects degraded (4.919%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1534,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":300,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":40094,"data_bytes":2626557862,"bytes_used":11147272192,"bytes_avail":564822941696,"bytes_total":575970213888,"degraded_objects":5917,"degraded_total":120282,"degraded_ratio":0.049192730416853729,"misplaced_objects":2356,"misplaced_total":120282,"misplaced_ratio":0.01958730317088176,"recovering_objects_per_sec":35,"recovering_bytes_per_sec":2357589,"recovering_keys_per_sec":0,"num_objects_recovered":360,"num_bytes_recovered":23592960,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":532},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":24,"modified":"2023-06-10T21:29:57.688996+0000","services":{}},"progress_events":{"08ad03ac-ce84-41c1-8854-b18550dc6087":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:34:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[180297]: pgmap v1089: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 5917/120282 objects degraded (4.919%); 2356/120282 objects misplaced (1.959%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2108002340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/120282 objects degraded (4.919%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[186706]: pgmap v1089: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 5917/120282 objects degraded (4.919%); 2356/120282 objects misplaced (1.959%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2108002340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:21.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/120282 objects degraded (4.919%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:21 smithi152 ceph-mon[156819]: pgmap v1089: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 532 op/s; 5917/120282 objects degraded (4.919%); 2356/120282 objects misplaced (1.959%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2108002340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/120282 objects degraded (4.919%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:21.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:21.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:22.015 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:22.141 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:22.450 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1035783404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:22.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1035783404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:22.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1035783404' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:23.062 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:22] "GET /metrics HTTP/1.1" 200 224074 "" "Prometheus/2.33.4" 2023-06-10T21:34:23.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:23.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:23.604 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:23.630 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:23.685 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:23.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:23.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:23.793 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:23.838 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:23 smithi152 ceph-mon[156819]: pgmap v1090: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5917/117621 objects degraded (5.031%); 2356/117621 objects misplaced (2.003%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:23.841 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:23.883 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:23.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:23.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:23 smithi064 ceph-mon[180297]: pgmap v1090: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5917/117621 objects degraded (5.031%); 2356/117621 objects misplaced (2.003%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:23 smithi064 ceph-mon[186706]: pgmap v1090: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 513 op/s; 5917/117621 objects degraded (5.031%); 2356/117621 objects misplaced (2.003%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:34:23.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:24.013 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:24.065 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:24.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:24.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:24.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:24.227 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:25.718 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:34:25.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:34:25.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:25 smithi064 ceph-mon[180297]: pgmap v1091: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 5917/116271 objects degraded (5.089%); 2356/116271 objects misplaced (2.026%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:34:25.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:25 smithi064 ceph-mon[186706]: pgmap v1091: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 5917/116271 objects degraded (5.089%); 2356/116271 objects misplaced (2.026%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:34:25.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:25.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:25.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:25.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:25 smithi152 ceph-mon[156819]: pgmap v1091: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 5917/116271 objects degraded (5.089%); 2356/116271 objects misplaced (2.026%); 1.1 MiB/s, 17 objects/s recovering 2023-06-10T21:34:26.110 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:26.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:26.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:26.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:26.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:34:26.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:26.449 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:26.495 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:26.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:26.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:26.630 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:26.665 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/116271 objects degraded (5.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:26.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/116271 objects degraded (5.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:26.680 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:26.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:26.787 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:26.812 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:34:26.813 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:34:26.886 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:26.890 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:26.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/116271 objects degraded (5.089%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:27.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:27.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:27.268 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:27.297 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:27.348 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:27.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:27.457 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:27.485 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:34:27.488 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:27.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:27.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:27.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:27 smithi152 ceph-mon[156819]: pgmap v1092: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5917/113046 objects degraded (5.234%); 2161/113046 objects misplaced (1.912%); 2.0 MiB/s, 32 objects/s recovering 2023-06-10T21:34:27.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2648097986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:27 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:27.706 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:27.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:27.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:27.841 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:27.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:27.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:27.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:27.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[180297]: pgmap v1092: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5917/113046 objects degraded (5.234%); 2161/113046 objects misplaced (1.912%); 2.0 MiB/s, 32 objects/s recovering 2023-06-10T21:34:27.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2648097986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[186706]: pgmap v1092: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 474 op/s; 5917/113046 objects degraded (5.234%); 2161/113046 objects misplaced (1.912%); 2.0 MiB/s, 32 objects/s recovering 2023-06-10T21:34:27.912 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2648097986' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.913 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:27 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:34:27.997 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:28.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:28.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:34:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[186706]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[186706]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:34:28.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[186706]: osdmap e301: 8 total, 8 up, 6 in 2023-06-10T21:34:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[180297]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[180297]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:34:28.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:28 smithi064 ceph-mon[180297]: osdmap e301: 8 total, 8 up, 6 in 2023-06-10T21:34:28.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:28 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:28 smithi152 ceph-mon[156819]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:28 smithi152 ceph-mon[156819]: failed to encode map e301 with expected crc 2023-06-10T21:34:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:28 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:34:28.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:28 smithi152 ceph-mon[156819]: osdmap e301: 8 total, 8 up, 6 in 2023-06-10T21:34:29.422 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:29.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:29.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:29.572 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:29.620 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:29.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:29.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:29.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:29 smithi064 ceph-mon[186706]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/110382 objects degraded (5.360%); 2161/110382 objects misplaced (1.958%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:34:29.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:29 smithi064 ceph-mon[180297]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/110382 objects degraded (5.360%); 2161/110382 objects misplaced (1.958%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:34:29.726 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:29.728 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:29.752 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:29 smithi152 ceph-mon[156819]: pgmap v1094: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/110382 objects degraded (5.360%); 2161/110382 objects misplaced (1.958%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:34:29.754 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:29.803 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:29.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:29.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:29.908 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:30.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:30.647 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:30.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:30.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:30.809 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:30.843 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:31.034 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:31.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:31.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:31.259 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:31.458 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:34:31.528 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:31.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:31.648 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:31.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:31 smithi064 ceph-mon[186706]: pgmap v1095: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 5917/105993 objects degraded (5.582%); 1987/105993 objects misplaced (1.875%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:31.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/105993 objects degraded (5.582%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:31.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:31 smithi064 ceph-mon[180297]: pgmap v1095: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 5917/105993 objects degraded (5.582%); 1987/105993 objects misplaced (1.875%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:31.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/105993 objects degraded (5.582%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:31.819 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:31.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:31 smithi152 ceph-mon[156819]: pgmap v1095: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 5917/105993 objects degraded (5.582%); 1987/105993 objects misplaced (1.875%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/105993 objects degraded (5.582%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:32.781 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:32.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:32.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:32.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:33.108 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:34:33.123 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:34:33.149 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:32] "GET /metrics HTTP/1.1" 200 224074 "" "Prometheus/2.33.4" 2023-06-10T21:34:33.714 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:33.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:33.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[186706]: pgmap v1096: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 5917/103209 objects degraded (5.733%); 1987/103209 objects misplaced (1.925%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[186706]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[180297]: pgmap v1096: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 5917/103209 objects degraded (5.733%); 1987/103209 objects misplaced (1.925%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[180297]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:33.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:33 smithi152 ceph-mon[156819]: pgmap v1096: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 5917/103209 objects degraded (5.733%); 1987/103209 objects misplaced (1.925%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:33 smithi152 ceph-mon[156819]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:34.172 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:34.616 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:34.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:34.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:34.704 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1096,"stamp":"2023-06-10T21:34:31.856330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2253592486,"num_objects":34403,"num_object_clones":0,"num_object_copies":103209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5917,"num_objects_misplaced":1987,"num_objects_unfound":0,"num_objects_dirty":34403,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":134424,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2732,"num_bytes_recovered":178069202,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110158,"ondisk_log_size":110158,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9812176,"kb_used_data":7065568,"kb_used_omap":30,"kb_used_meta":2746209,"kb_avail":552658736,"statfs":{"total":575970213888,"available":565922545664,"internally_reserved":0,"allocated":7235141632,"data_stored":7230088062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2812118054},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314834944,"num_objects":-4804,"num_object_clones":0,"num_object_copies":-14412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-369,"num_objects_unfound":0,"num_objects_dirty":-4804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4804,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":369,"num_bytes_recovered":24182784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.020111"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":294,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421898+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:34:14.421898+0000","last_peered":"2023-06-10T21:34:14.421898+0000","last_clean":"2023-06-10T21:34:14.421898+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:34:14.421898+0000","last_undegraded":"2023-06-10T21:34:14.421898+0000","last_fullsized":"2023-06-10T21:34:14.421898+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":235,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625083+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:34:14.625083+0000","last_peered":"2023-06-10T21:34:14.625083+0000","last_clean":"2023-06-10T21:34:14.625083+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:34:14.625083+0000","last_undegraded":"2023-06-10T21:34:14.625083+0000","last_fullsized":"2023-06-10T21:34:14.625083+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":210,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625103+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:34:14.625103+0000","last_peered":"2023-06-10T21:34:14.625103+0000","last_clean":"2023-06-10T21:34:14.625103+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:34:14.625103+0000","last_undegraded":"2023-06-10T21:34:14.625103+0000","last_fullsized":"2023-06-10T21:34:14.625103+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":156,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257491+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:34:00.257491+0000","last_peered":"2023-06-10T21:34:00.257491+0000","last_clean":"2023-06-10T21:34:00.257491+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:34:00.257491+0000","last_undegraded":"2023-06-10T21:34:00.257491+0000","last_fullsized":"2023-06-10T21:34:00.257491+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":275,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439647+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:34:14.439647+0000","last_peered":"2023-06-10T21:34:14.439647+0000","last_clean":"2023-06-10T21:34:14.439647+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:34:14.439647+0000","last_undegraded":"2023-06-10T21:34:14.439647+0000","last_fullsized":"2023-06-10T21:34:14.439647+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":266,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.260284+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:34:00.260284+0000","last_peered":"2023-06-10T21:34:00.260284+0000","last_clean":"2023-06-10T21:34:00.260284+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:34:00.260284+0000","last_undegraded":"2023-06-10T21:34:00.260284+0000","last_fullsized":"2023-06-10T21:34:00.260284+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":156,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.260380+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:34:00.260380+0000","last_peered":"2023-06-10T21:34:00.260380+0000","last_clean":"2023-06-10T21:34:00.260380+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:34:00.260380+0000","last_undegraded":"2023-06-10T21:34:00.260380+0000","last_fullsized":"2023-06-10T21:34:00.260380+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":249,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439205+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:34:14.439205+0000","last_peered":"2023-06-10T21:34:14.439205+0000","last_clean":"2023-06-10T21:34:14.439205+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:34:14.439205+0000","last_undegraded":"2023-06-10T21:34:14.439205+0000","last_fullsized":"2023-06-10T21:34:14.439205+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420628+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:34:14.420628+0000","last_peered":"2023-06-10T21:34:14.420628+0000","last_clean":"2023-06-10T21:34:14.420628+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:34:14.420628+0000","last_undegraded":"2023-06-10T21:34:14.420628+0000","last_fullsized":"2023-06-10T21:34:14.420628+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439479+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:34:14.439479+0000","last_peered":"2023-06-10T21:34:14.439479+0000","last_clean":"2023-06-10T21:34:14.439479+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:34:14.439479+0000","last_undegraded":"2023-06-10T21:34:14.439479+0000","last_fullsized":"2023-06-10T21:34:14.439479+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":28,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418776+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:34:14.418776+0000","last_peered":"2023-06-10T21:34:14.418776+0000","last_clean":"2023-06-10T21:34:14.418776+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:34:14.418776+0000","last_undegraded":"2023-06-10T21:34:14.418776+0000","last_fullsized":"2023-06-10T21:34:14.418776+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421802+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:34:14.421802+0000","last_peered":"2023-06-10T21:34:14.421802+0000","last_clean":"2023-06-10T21:34:14.421802+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:34:14.421802+0000","last_undegraded":"2023-06-10T21:34:14.421802+0000","last_fullsized":"2023-06-10T21:34:14.421802+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":171,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421482+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:34:14.421482+0000","last_peered":"2023-06-10T21:34:14.421482+0000","last_clean":"2023-06-10T21:34:14.421482+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:34:14.421482+0000","last_undegraded":"2023-06-10T21:34:14.421482+0000","last_fullsized":"2023-06-10T21:34:14.421482+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":32,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420701+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:34:14.420701+0000","last_peered":"2023-06-10T21:34:14.420701+0000","last_clean":"2023-06-10T21:34:14.420701+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:34:14.420701+0000","last_undegraded":"2023-06-10T21:34:14.420701+0000","last_fullsized":"2023-06-10T21:34:14.420701+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":300,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420752+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:34:14.420752+0000","last_peered":"2023-06-10T21:34:14.420752+0000","last_clean":"2023-06-10T21:34:14.420752+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:34:14.420752+0000","last_undegraded":"2023-06-10T21:34:14.420752+0000","last_fullsized":"2023-06-10T21:34:14.420752+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439452+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:34:14.439452+0000","last_peered":"2023-06-10T21:34:14.439452+0000","last_clean":"2023-06-10T21:34:14.439452+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:34:14.439452+0000","last_undegraded":"2023-06-10T21:34:14.439452+0000","last_fullsized":"2023-06-10T21:34:14.439452+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":156,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256514+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:34:00.256514+0000","last_peered":"2023-06-10T21:34:00.256514+0000","last_clean":"2023-06-10T21:34:00.256514+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:34:00.256514+0000","last_undegraded":"2023-06-10T21:34:00.256514+0000","last_fullsized":"2023-06-10T21:34:00.256514+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.419126+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:34:14.419126+0000","last_peered":"2023-06-10T21:34:14.419126+0000","last_clean":"2023-06-10T21:34:14.419126+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:34:14.419126+0000","last_undegraded":"2023-06-10T21:34:14.419126+0000","last_fullsized":"2023-06-10T21:34:14.419126+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418624+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:34:14.418624+0000","last_peered":"2023-06-10T21:34:14.418624+0000","last_clean":"2023-06-10T21:34:14.418624+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:34:14.418624+0000","last_undegraded":"2023-06-10T21:34:14.418624+0000","last_fullsized":"2023-06-10T21:34:14.418624+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":156,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257322+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:34:00.257322+0000","last_peered":"2023-06-10T21:34:00.257322+0000","last_clean":"2023-06-10T21:34:00.257322+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:34:00.257322+0000","last_undegraded":"2023-06-10T21:34:00.257322+0000","last_fullsized":"2023-06-10T21:34:00.257322+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":275,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256292+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:34:00.256292+0000","last_peered":"2023-06-10T21:34:00.256292+0000","last_clean":"2023-06-10T21:34:00.256292+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:34:00.256292+0000","last_undegraded":"2023-06-10T21:34:00.256292+0000","last_fullsized":"2023-06-10T21:34:00.256292+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439543+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:34:14.439543+0000","last_peered":"2023-06-10T21:34:14.439543+0000","last_clean":"2023-06-10T21:34:14.439543+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:34:14.439543+0000","last_undegraded":"2023-06-10T21:34:14.439543+0000","last_fullsized":"2023-06-10T21:34:14.439543+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"300'8312","reported_seq":8353,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:27.812464+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:34:27.812464+0000","last_peered":"2023-06-10T21:34:27.812464+0000","last_clean":"2023-06-10T21:34:27.812464+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:34:27.812464+0000","last_undegraded":"2023-06-10T21:34:27.812464+0000","last_fullsized":"2023-06-10T21:34:27.812464+0000","mapping_epoch":269,"log_start":"280'1900","ondisk_log_start":"280'1900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6412,"log_dups_size":0,"ondisk_log_size":6412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146276352,"num_objects":2232,"num_object_clones":0,"num_object_copies":6696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8312,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418658+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:34:14.418658+0000","last_peered":"2023-06-10T21:34:14.418658+0000","last_clean":"2023-06-10T21:34:14.418658+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:34:14.418658+0000","last_undegraded":"2023-06-10T21:34:14.418658+0000","last_fullsized":"2023-06-10T21:34:14.418658+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418253+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:34:14.418253+0000","last_peered":"2023-06-10T21:34:14.418253+0000","last_clean":"2023-06-10T21:34:14.418253+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:34:14.418253+0000","last_undegraded":"2023-06-10T21:34:14.418253+0000","last_fullsized":"2023-06-10T21:34:14.418253+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"300'8414","reported_seq":8455,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:27.862654+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:34:27.862654+0000","last_peered":"2023-06-10T21:34:27.862654+0000","last_clean":"2023-06-10T21:34:27.862654+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:34:27.862654+0000","last_undegraded":"2023-06-10T21:34:27.862654+0000","last_fullsized":"2023-06-10T21:34:27.862654+0000","mapping_epoch":269,"log_start":"280'2000","ondisk_log_start":"280'2000","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142999552,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420717+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:34:14.420717+0000","last_peered":"2023-06-10T21:34:14.420717+0000","last_clean":"2023-06-10T21:34:14.420717+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:34:14.420717+0000","last_undegraded":"2023-06-10T21:34:14.420717+0000","last_fullsized":"2023-06-10T21:34:14.420717+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":174,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625105+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:34:14.625105+0000","last_peered":"2023-06-10T21:34:14.625105+0000","last_clean":"2023-06-10T21:34:14.625105+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:34:14.625105+0000","last_undegraded":"2023-06-10T21:34:14.625105+0000","last_fullsized":"2023-06-10T21:34:14.625105+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"300'8570","reported_seq":8614,"reported_epoch":300,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:27.920099+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:34:27.920099+0000","last_peered":"2023-06-10T21:34:27.920099+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:34:27.920099+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"280'2100","ondisk_log_start":"280'2100","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153092096,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8570,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":2336},{"shards":"0,3","objects":-3058}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":308,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418922+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:34:14.418922+0000","last_peered":"2023-06-10T21:34:14.418922+0000","last_clean":"2023-06-10T21:34:14.418922+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:34:14.418922+0000","last_undegraded":"2023-06-10T21:34:14.418922+0000","last_fullsized":"2023-06-10T21:34:14.418922+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418533+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:34:14.418533+0000","last_peered":"2023-06-10T21:34:14.418533+0000","last_clean":"2023-06-10T21:34:14.418533+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:34:14.418533+0000","last_undegraded":"2023-06-10T21:34:14.418533+0000","last_fullsized":"2023-06-10T21:34:14.418533+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"301'8314","reported_seq":8352,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:29.642739+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:34:29.642739+0000","last_peered":"2023-06-10T21:34:29.642739+0000","last_clean":"2023-06-10T21:34:29.642739+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:34:29.642739+0000","last_undegraded":"2023-06-10T21:34:29.642739+0000","last_fullsized":"2023-06-10T21:34:29.642739+0000","mapping_epoch":269,"log_start":"277'900","ondisk_log_start":"277'900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7414,"log_dups_size":0,"ondisk_log_size":7414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142213120,"num_objects":2170,"num_object_clones":0,"num_object_copies":6510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8314,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439560+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:34:14.439560+0000","last_peered":"2023-06-10T21:34:14.439560+0000","last_clean":"2023-06-10T21:34:14.439560+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:34:14.439560+0000","last_undegraded":"2023-06-10T21:34:14.439560+0000","last_fullsized":"2023-06-10T21:34:14.439560+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256660+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:34:00.256660+0000","last_peered":"2023-06-10T21:34:00.256660+0000","last_clean":"2023-06-10T21:34:00.256660+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:34:00.256660+0000","last_undegraded":"2023-06-10T21:34:00.256660+0000","last_fullsized":"2023-06-10T21:34:00.256660+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"301'8281","reported_seq":13711,"reported_epoch":301,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:29.072429+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:34:29.072429+0000","last_peered":"2023-06-10T21:34:29.072429+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:34:29.072429+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8281,"log_dups_size":0,"ondisk_log_size":8281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138084352,"num_objects":2107,"num_object_clones":0,"num_object_copies":6321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":464,"num_objects_misplaced":1987,"num_objects_unfound":0,"num_objects_dirty":2107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8281,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-344},{"shards":"1,2,3","objects":1987}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":252,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624912+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:34:14.624912+0000","last_peered":"2023-06-10T21:34:14.624912+0000","last_clean":"2023-06-10T21:34:14.624912+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:34:14.624912+0000","last_undegraded":"2023-06-10T21:34:14.624912+0000","last_fullsized":"2023-06-10T21:34:14.624912+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439636+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:34:14.439636+0000","last_peered":"2023-06-10T21:34:14.439636+0000","last_clean":"2023-06-10T21:34:14.439636+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:34:14.439636+0000","last_undegraded":"2023-06-10T21:34:14.439636+0000","last_fullsized":"2023-06-10T21:34:14.439636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"301'8520","reported_seq":8562,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.988987+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:34:28.988987+0000","last_peered":"2023-06-10T21:34:28.988987+0000","last_clean":"2023-06-10T21:34:28.988987+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:34:28.988987+0000","last_undegraded":"2023-06-10T21:34:28.988987+0000","last_fullsized":"2023-06-10T21:34:28.988987+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8520,"log_dups_size":0,"ondisk_log_size":8520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8520,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624747+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:34:14.624747+0000","last_peered":"2023-06-10T21:34:14.624747+0000","last_clean":"2023-06-10T21:34:14.624747+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:34:14.624747+0000","last_undegraded":"2023-06-10T21:34:14.624747+0000","last_fullsized":"2023-06-10T21:34:14.624747+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":292,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.422059+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:34:14.422059+0000","last_peered":"2023-06-10T21:34:14.422059+0000","last_clean":"2023-06-10T21:34:14.422059+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:34:14.422059+0000","last_undegraded":"2023-06-10T21:34:14.422059+0000","last_fullsized":"2023-06-10T21:34:14.422059+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"301'8437","reported_seq":8479,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.458450+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:34:31.458450+0000","last_peered":"2023-06-10T21:34:31.458450+0000","last_clean":"2023-06-10T21:34:31.458450+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:34:31.458450+0000","last_undegraded":"2023-06-10T21:34:31.458450+0000","last_fullsized":"2023-06-10T21:34:31.458450+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5837,"log_dups_size":0,"ondisk_log_size":5837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137691136,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8437,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421883+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:34:14.421883+0000","last_peered":"2023-06-10T21:34:14.421883+0000","last_clean":"2023-06-10T21:34:14.421883+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:34:14.421883+0000","last_undegraded":"2023-06-10T21:34:14.421883+0000","last_fullsized":"2023-06-10T21:34:14.421883+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624990+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:34:14.624990+0000","last_peered":"2023-06-10T21:34:14.624990+0000","last_clean":"2023-06-10T21:34:14.624990+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:34:14.624990+0000","last_undegraded":"2023-06-10T21:34:14.624990+0000","last_fullsized":"2023-06-10T21:34:14.624990+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"301'8448","reported_seq":8490,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.510293+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:34:31.510293+0000","last_peered":"2023-06-10T21:34:31.510293+0000","last_clean":"2023-06-10T21:34:31.510293+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:34:31.510293+0000","last_undegraded":"2023-06-10T21:34:31.510293+0000","last_fullsized":"2023-06-10T21:34:31.510293+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137232384,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8448,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439473+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:34:14.439473+0000","last_peered":"2023-06-10T21:34:14.439473+0000","last_clean":"2023-06-10T21:34:14.439473+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:34:14.439473+0000","last_undegraded":"2023-06-10T21:34:14.439473+0000","last_fullsized":"2023-06-10T21:34:14.439473+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":211,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624747+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:34:14.624747+0000","last_peered":"2023-06-10T21:34:14.624747+0000","last_clean":"2023-06-10T21:34:14.624747+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:34:14.624747+0000","last_undegraded":"2023-06-10T21:34:14.624747+0000","last_fullsized":"2023-06-10T21:34:14.624747+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"301'8333","reported_seq":8376,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:29.044955+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:34:29.044955+0000","last_peered":"2023-06-10T21:34:29.044955+0000","last_clean":"2023-06-10T21:34:29.044955+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:34:29.044955+0000","last_undegraded":"2023-06-10T21:34:29.044955+0000","last_fullsized":"2023-06-10T21:34:29.044955+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8333,"log_dups_size":0,"ondisk_log_size":8333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136970264,"num_objects":2091,"num_object_clones":0,"num_object_copies":6273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2091,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8333,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":123,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.258825+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:34:00.258825+0000","last_peered":"2023-06-10T21:34:00.258825+0000","last_clean":"2023-06-10T21:34:00.258825+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:34:00.258825+0000","last_undegraded":"2023-06-10T21:34:00.258825+0000","last_fullsized":"2023-06-10T21:34:00.258825+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625127+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:34:14.625127+0000","last_peered":"2023-06-10T21:34:14.625127+0000","last_clean":"2023-06-10T21:34:14.625127+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:34:14.625127+0000","last_undegraded":"2023-06-10T21:34:14.625127+0000","last_fullsized":"2023-06-10T21:34:14.625127+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":143,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256366+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:34:00.256366+0000","last_peered":"2023-06-10T21:34:00.256366+0000","last_clean":"2023-06-10T21:34:00.256366+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:34:00.256366+0000","last_undegraded":"2023-06-10T21:34:00.256366+0000","last_fullsized":"2023-06-10T21:34:00.256366+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"301'8410","reported_seq":8452,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.502343+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:34:31.502343+0000","last_peered":"2023-06-10T21:34:31.502343+0000","last_clean":"2023-06-10T21:34:31.502343+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:34:31.502343+0000","last_undegraded":"2023-06-10T21:34:31.502343+0000","last_fullsized":"2023-06-10T21:34:31.502343+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5810,"log_dups_size":0,"ondisk_log_size":5810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140115968,"num_objects":2138,"num_object_clones":0,"num_object_copies":6414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8410,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":169,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256440+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:34:00.256440+0000","last_peered":"2023-06-10T21:34:00.256440+0000","last_clean":"2023-06-10T21:34:00.256440+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:34:00.256440+0000","last_undegraded":"2023-06-10T21:34:00.256440+0000","last_fullsized":"2023-06-10T21:34:00.256440+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":243,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256790+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:34:00.256790+0000","last_peered":"2023-06-10T21:34:00.256790+0000","last_clean":"2023-06-10T21:34:00.256790+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:34:00.256790+0000","last_undegraded":"2023-06-10T21:34:00.256790+0000","last_fullsized":"2023-06-10T21:34:00.256790+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425733+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:34:14.425733+0000","last_peered":"2023-06-10T21:34:14.425733+0000","last_clean":"2023-06-10T21:34:14.425733+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:34:14.425733+0000","last_undegraded":"2023-06-10T21:34:14.425733+0000","last_fullsized":"2023-06-10T21:34:14.425733+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"301'8467","reported_seq":8509,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.353923+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:34:31.353923+0000","last_peered":"2023-06-10T21:34:31.353923+0000","last_clean":"2023-06-10T21:34:31.353923+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:34:31.353923+0000","last_undegraded":"2023-06-10T21:34:31.353923+0000","last_fullsized":"2023-06-10T21:34:31.353923+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5867,"log_dups_size":0,"ondisk_log_size":5867,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8467,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":145,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421634+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:34:14.421634+0000","last_peered":"2023-06-10T21:34:14.421634+0000","last_clean":"2023-06-10T21:34:14.421634+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:34:14.421634+0000","last_undegraded":"2023-06-10T21:34:14.421634+0000","last_fullsized":"2023-06-10T21:34:14.421634+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421741+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:34:14.421741+0000","last_peered":"2023-06-10T21:34:14.421741+0000","last_clean":"2023-06-10T21:34:14.421741+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:34:14.421741+0000","last_undegraded":"2023-06-10T21:34:14.421741+0000","last_fullsized":"2023-06-10T21:34:14.421741+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.260458+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:34:00.260458+0000","last_peered":"2023-06-10T21:34:00.260458+0000","last_clean":"2023-06-10T21:34:00.260458+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:34:00.260458+0000","last_undegraded":"2023-06-10T21:34:00.260458+0000","last_fullsized":"2023-06-10T21:34:00.260458+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"301'8314","reported_seq":8356,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.421956+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:34:31.421956+0000","last_peered":"2023-06-10T21:34:31.421956+0000","last_clean":"2023-06-10T21:34:31.421956+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:34:31.421956+0000","last_undegraded":"2023-06-10T21:34:31.421956+0000","last_fullsized":"2023-06-10T21:34:31.421956+0000","mapping_epoch":269,"log_start":"281'2500","ondisk_log_start":"281'2500","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134217728,"num_objects":2048,"num_object_clones":0,"num_object_copies":6144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8314,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":325,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.429876+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:34:14.429876+0000","last_peered":"2023-06-10T21:34:14.429876+0000","last_clean":"2023-06-10T21:34:14.429876+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:34:14.429876+0000","last_undegraded":"2023-06-10T21:34:14.429876+0000","last_fullsized":"2023-06-10T21:34:14.429876+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":464,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418308+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:34:14.418308+0000","last_peered":"2023-06-10T21:34:14.418308+0000","last_clean":"2023-06-10T21:34:14.418308+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:34:14.418308+0000","last_undegraded":"2023-06-10T21:34:14.418308+0000","last_fullsized":"2023-06-10T21:34:14.418308+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":238,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624777+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:34:14.624777+0000","last_peered":"2023-06-10T21:34:14.624777+0000","last_clean":"2023-06-10T21:34:14.624777+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:34:14.624777+0000","last_undegraded":"2023-06-10T21:34:14.624777+0000","last_fullsized":"2023-06-10T21:34:14.624777+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.419062+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:34:14.419062+0000","last_peered":"2023-06-10T21:34:14.419062+0000","last_clean":"2023-06-10T21:34:14.419062+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:34:14.419062+0000","last_undegraded":"2023-06-10T21:34:14.419062+0000","last_fullsized":"2023-06-10T21:34:14.419062+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"301'8424","reported_seq":8466,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:31.491723+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:34:31.491723+0000","last_peered":"2023-06-10T21:34:31.491723+0000","last_clean":"2023-06-10T21:34:31.491723+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:34:31.491723+0000","last_undegraded":"2023-06-10T21:34:31.491723+0000","last_fullsized":"2023-06-10T21:34:31.491723+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145227776,"num_objects":2216,"num_object_clones":0,"num_object_copies":6648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2216,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8424,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":320,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418330+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:34:14.418330+0000","last_peered":"2023-06-10T21:34:14.418330+0000","last_clean":"2023-06-10T21:34:14.418330+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:34:14.418330+0000","last_undegraded":"2023-06-10T21:34:14.418330+0000","last_fullsized":"2023-06-10T21:34:14.418330+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":138,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257515+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:34:00.257515+0000","last_peered":"2023-06-10T21:34:00.257515+0000","last_clean":"2023-06-10T21:34:00.257515+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:34:00.257515+0000","last_undegraded":"2023-06-10T21:34:00.257515+0000","last_fullsized":"2023-06-10T21:34:00.257515+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":123,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257238+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:34:00.257238+0000","last_peered":"2023-06-10T21:34:00.257238+0000","last_clean":"2023-06-10T21:34:00.257238+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:34:00.257238+0000","last_undegraded":"2023-06-10T21:34:00.257238+0000","last_fullsized":"2023-06-10T21:34:00.257238+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"301'8336","reported_seq":8378,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:29.049497+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:34:29.049497+0000","last_peered":"2023-06-10T21:34:29.049497+0000","last_clean":"2023-06-10T21:34:29.049497+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:34:29.049497+0000","last_undegraded":"2023-06-10T21:34:29.049497+0000","last_fullsized":"2023-06-10T21:34:29.049497+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144179200,"num_objects":2200,"num_object_clones":0,"num_object_copies":6600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":305,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418995+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:34:14.418995+0000","last_peered":"2023-06-10T21:34:14.418995+0000","last_clean":"2023-06-10T21:34:14.418995+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:34:14.418995+0000","last_undegraded":"2023-06-10T21:34:14.418995+0000","last_fullsized":"2023-06-10T21:34:14.418995+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425825+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:34:14.425825+0000","last_peered":"2023-06-10T21:34:14.425825+0000","last_clean":"2023-06-10T21:34:14.425825+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:34:14.425825+0000","last_undegraded":"2023-06-10T21:34:14.425825+0000","last_fullsized":"2023-06-10T21:34:14.425825+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":119,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256912+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:34:00.256912+0000","last_peered":"2023-06-10T21:34:00.256912+0000","last_clean":"2023-06-10T21:34:00.256912+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:34:00.256912+0000","last_undegraded":"2023-06-10T21:34:00.256912+0000","last_fullsized":"2023-06-10T21:34:00.256912+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"301'8280","reported_seq":8320,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:29.659068+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:34:29.659068+0000","last_peered":"2023-06-10T21:34:29.659068+0000","last_clean":"2023-06-10T21:34:29.659068+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:34:29.659068+0000","last_undegraded":"2023-06-10T21:34:29.659068+0000","last_fullsized":"2023-06-10T21:34:29.659068+0000","mapping_epoch":269,"log_start":"277'900","ondisk_log_start":"277'900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7380,"log_dups_size":0,"ondisk_log_size":7380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138543104,"num_objects":2114,"num_object_clones":0,"num_object_copies":6342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8280,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439642+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:34:14.439642+0000","last_peered":"2023-06-10T21:34:14.439642+0000","last_clean":"2023-06-10T21:34:14.439642+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:34:14.439642+0000","last_undegraded":"2023-06-10T21:34:14.439642+0000","last_fullsized":"2023-06-10T21:34:14.439642+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624894+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:34:14.624894+0000","last_peered":"2023-06-10T21:34:14.624894+0000","last_clean":"2023-06-10T21:34:14.624894+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:34:14.624894+0000","last_undegraded":"2023-06-10T21:34:14.624894+0000","last_fullsized":"2023-06-10T21:34:14.624894+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439381+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:34:14.439381+0000","last_peered":"2023-06-10T21:34:14.439381+0000","last_clean":"2023-06-10T21:34:14.439381+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:34:14.439381+0000","last_undegraded":"2023-06-10T21:34:14.439381+0000","last_fullsized":"2023-06-10T21:34:14.439381+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"301'8303","reported_seq":8343,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:29.619567+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:34:29.619567+0000","last_peered":"2023-06-10T21:34:29.619567+0000","last_clean":"2023-06-10T21:34:29.619567+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:34:29.619567+0000","last_undegraded":"2023-06-10T21:34:29.619567+0000","last_fullsized":"2023-06-10T21:34:29.619567+0000","mapping_epoch":269,"log_start":"277'900","ondisk_log_start":"277'900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134545408,"num_objects":2053,"num_object_clones":0,"num_object_copies":6159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8303,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418609+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:34:14.418609+0000","last_peered":"2023-06-10T21:34:14.418609+0000","last_clean":"2023-06-10T21:34:14.418609+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:34:14.418609+0000","last_undegraded":"2023-06-10T21:34:14.418609+0000","last_fullsized":"2023-06-10T21:34:14.418609+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":127,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257415+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:34:00.257415+0000","last_peered":"2023-06-10T21:34:00.257415+0000","last_clean":"2023-06-10T21:34:00.257415+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:34:00.257415+0000","last_undegraded":"2023-06-10T21:34:00.257415+0000","last_fullsized":"2023-06-10T21:34:00.257415+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":255,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.257259+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:34:00.257259+0000","last_peered":"2023-06-10T21:34:00.257259+0000","last_clean":"2023-06-10T21:34:00.257259+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:34:00.257259+0000","last_undegraded":"2023-06-10T21:34:00.257259+0000","last_fullsized":"2023-06-10T21:34:00.257259+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":123,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.256990+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:34:00.256990+0000","last_peered":"2023-06-10T21:34:00.256990+0000","last_clean":"2023-06-10T21:34:00.256990+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:34:00.256990+0000","last_undegraded":"2023-06-10T21:34:00.256990+0000","last_fullsized":"2023-06-10T21:34:00.256990+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":142,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425938+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:34:14.425938+0000","last_peered":"2023-06-10T21:34:14.425938+0000","last_clean":"2023-06-10T21:34:14.425938+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:34:14.425938+0000","last_undegraded":"2023-06-10T21:34:14.425938+0000","last_fullsized":"2023-06-10T21:34:14.425938+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418423+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:34:14.418423+0000","last_peered":"2023-06-10T21:34:14.418423+0000","last_clean":"2023-06-10T21:34:14.418423+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:34:14.418423+0000","last_undegraded":"2023-06-10T21:34:14.418423+0000","last_fullsized":"2023-06-10T21:34:14.418423+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624750+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:34:14.624750+0000","last_peered":"2023-06-10T21:34:14.624750+0000","last_clean":"2023-06-10T21:34:14.624750+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:34:14.624750+0000","last_undegraded":"2023-06-10T21:34:14.624750+0000","last_fullsized":"2023-06-10T21:34:14.624750+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":79,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624851+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:34:14.624851+0000","last_peered":"2023-06-10T21:34:14.624851+0000","last_clean":"2023-06-10T21:34:14.624851+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:34:14.624851+0000","last_undegraded":"2023-06-10T21:34:14.624851+0000","last_fullsized":"2023-06-10T21:34:14.624851+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624667+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:34:14.624667+0000","last_peered":"2023-06-10T21:34:14.624667+0000","last_clean":"2023-06-10T21:34:14.624667+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:34:14.624667+0000","last_undegraded":"2023-06-10T21:34:14.624667+0000","last_fullsized":"2023-06-10T21:34:14.624667+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":284,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624636+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:34:14.624636+0000","last_peered":"2023-06-10T21:34:14.624636+0000","last_clean":"2023-06-10T21:34:14.624636+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:34:14.624636+0000","last_undegraded":"2023-06-10T21:34:14.624636+0000","last_fullsized":"2023-06-10T21:34:14.624636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624719+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:34:14.624719+0000","last_peered":"2023-06-10T21:34:14.624719+0000","last_clean":"2023-06-10T21:34:14.624719+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:34:14.624719+0000","last_undegraded":"2023-06-10T21:34:14.624719+0000","last_fullsized":"2023-06-10T21:34:14.624719+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":188,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-06-10T21:34:00.260549+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:34:00.260549+0000","last_peered":"2023-06-10T21:34:00.260549+0000","last_clean":"2023-06-10T21:34:00.260549+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:34:00.260549+0000","last_undegraded":"2023-06-10T21:34:00.260549+0000","last_fullsized":"2023-06-10T21:34:00.260549+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421392+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:34:14.421392+0000","last_peered":"2023-06-10T21:34:14.421392+0000","last_clean":"2023-06-10T21:34:14.421392+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:34:14.421392+0000","last_undegraded":"2023-06-10T21:34:14.421392+0000","last_fullsized":"2023-06-10T21:34:14.421392+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2252996632,"num_objects":34379,"num_object_clones":0,"num_object_copies":103137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5917,"num_objects_misplaced":1987,"num_objects_unfound":0,"num_objects_dirty":34379,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134163,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2691,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7161327616,"data_stored":7161315400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109963,"ondisk_log_size":109963,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430649,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1132632,"kb_used_data":558352,"kb_used_omap":0,"kb_used_meta":574272,"kb_avail":92612520,"statfs":{"total":95995035648,"available":94835220480,"internally_reserved":0,"allocated":571752448,"data_stored":571188148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":588054528},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":5.6710000000000003,"15min":8.3620000000000001},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":52.100000000000001,"5min":184.83699999999999,"15min":184.83699999999999},"last":1.75},{"interface":"front","average":{"1min":3.677,"5min":5.641,"15min":8.5800000000000001},"min":{"1min":0.31900000000000001,"5min":0.223,"15min":0.223},"max":{"1min":51.146999999999998,"5min":185.58000000000001,"15min":185.58000000000001},"last":5.4980000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:34:13 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":3.3130000000000002,"15min":3.2519999999999998},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.21299999999999999},"max":{"1min":59.268000000000001,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":4.4749999999999996,"5min":3.4769999999999999,"15min":3.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":59.256999999999998,"5min":87.438000000000002,"15min":212.20599999999999},"last":1.367}]},{"osd":2,"last update":"Sat Jun 10 21:34:26 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":6.4400000000000004,"15min":4.1849999999999996},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":89.055999999999997,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.86299999999999999},{"interface":"front","average":{"1min":7.0510000000000002,"5min":6.3639999999999999,"15min":4.8490000000000002},"min":{"1min":0.38300000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":88.787000000000006,"5min":172.185,"15min":212.13399999999999},"last":1.972}]},{"osd":3,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":8.5359999999999996,"5min":4.0730000000000004,"15min":3.024},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":70.841999999999999,"5min":99.266000000000005,"15min":99.266000000000005},"last":4.9530000000000003},{"interface":"front","average":{"1min":7.9180000000000001,"5min":3.714,"15min":2.46},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":69.796999999999997,"5min":99.138000000000005,"15min":99.138000000000005},"last":0.52800000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.45700000000000002,"15min":0.72199999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.5049999999999999,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":1.0369999999999999},"min":{"1min":0.29599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.0310000000000001,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.54500000000000004,"15min":1.827},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1829999999999998,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47399999999999998,"15min":1.6220000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.92100000000000004,"5min":1.7,"15min":212.23699999999999},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57199999999999995,"15min":1.083},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":3.2559999999999998,"5min":3.2559999999999998,"15min":13.073},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.94399999999999995},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":1.1879999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.498}]}]},{"osd":6,"up_from":0,"seq":180388626877,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566320,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2583036,"kb_used_data":1953908,"kb_used_omap":4,"kb_used_meta":629051,"kb_avail":91162116,"statfs":{"total":95995035648,"available":93350006784,"internally_reserved":0,"allocated":2000801792,"data_stored":1999852623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":644149134},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3789999999999996,"5min":7.5979999999999999,"15min":11.686},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":70.254000000000005,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":5.7859999999999996,"5min":7.4790000000000001,"15min":11.372999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":70.100999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.23499999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:33:35 2023","interfaces":[{"interface":"back","average":{"1min":15.648999999999999,"5min":10.077,"15min":4.8390000000000004},"min":{"1min":0.317,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":191.101,"5min":191.101,"15min":191.101},"last":0.93899999999999995},{"interface":"front","average":{"1min":16.170000000000002,"5min":10.291,"15min":4.5869999999999997},"min":{"1min":0.40500000000000003,"5min":0.249,"15min":0.218},"max":{"1min":190.87899999999999,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.66300000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.2290000000000001,"5min":4.3470000000000004,"15min":2.766},"min":{"1min":0.32300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.694000000000003,"5min":80.584999999999994,"15min":80.584999999999994},"last":0.85399999999999998},{"interface":"front","average":{"1min":5.3959999999999999,"5min":5.266,"15min":3.8759999999999999},"min":{"1min":0.40400000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":42.734999999999999,"5min":80.873999999999995,"15min":80.873999999999995},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.4430000000000001,"15min":1.4810000000000001},"min":{"1min":0.28699999999999998,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.032,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.024,"5min":1.504,"15min":1.3540000000000001},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":4.8200000000000003,"5min":8.2739999999999991,"15min":14.460000000000001},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.0339999999999998,"15min":4.1349999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7089999999999996,"5min":8.9009999999999998,"15min":383.83100000000002},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.46,"15min":3.7400000000000002},"min":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.0679999999999996,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.129,"15min":2.0310000000000001},"min":{"1min":0.34799999999999998,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":8.7550000000000008,"5min":8.8580000000000005,"15min":14.858000000000001},"last":0.875},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5129999999999999,"15min":1.643},"min":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.1360000000000001,"5min":11.476000000000001,"15min":13.48},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3970000000000002,"5min":2.1709999999999998,"15min":2.5979999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32600000000000001},"max":{"1min":47.844000000000001,"5min":47.844000000000001,"15min":162.399},"last":0.60999999999999999},{"interface":"front","average":{"1min":4.4699999999999998,"5min":2.407,"15min":2.3759999999999999},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":47.753999999999998,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.52500000000000002}]}]},{"osd":0,"up_from":244,"seq":1047972020296,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1410052,"kb_used_data":1162172,"kb_used_omap":10,"kb_used_meta":247797,"kb_avail":92335100,"statfs":{"total":95995035648,"available":94551142400,"internally_reserved":0,"allocated":1190064128,"data_stored":1189039788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":253744180},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":5.056,"15min":4.625},"min":{"1min":0.215,"5min":0.191,"15min":0.191},"max":{"1min":8.6750000000000007,"5min":139.68000000000001,"15min":139.68000000000001},"last":8.1129999999999995},{"interface":"front","average":{"1min":3.2909999999999999,"5min":4.9800000000000004,"15min":5.7009999999999996},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.409000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":15.375,"5min":7.8479999999999999,"15min":6.5839999999999996},"min":{"1min":0.28499999999999998,"5min":0.216,"15min":0.216},"max":{"1min":132.245,"5min":205.10900000000001,"15min":205.10900000000001},"last":4.0709999999999997},{"interface":"front","average":{"1min":15.032,"5min":7.9210000000000003,"15min":6.0439999999999996},"min":{"1min":0.46600000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":132.26599999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":8.1370000000000005}]},{"osd":3,"last update":"Sat Jun 10 21:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.871,"5min":4.5389999999999997,"15min":5.7039999999999997},"min":{"1min":0.23999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":26.167999999999999,"5min":111.745,"15min":111.745},"last":6.3639999999999999},{"interface":"front","average":{"1min":4.7510000000000003,"5min":5.5579999999999998,"15min":6.7080000000000002},"min":{"1min":0.30399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":26.201000000000001,"5min":114.205,"15min":114.205},"last":8.0860000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.617,"15min":4.2699999999999996},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.744999999999999,"5min":29.709,"15min":29.709},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.2669999999999999,"15min":3.605},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.9630000000000001,"5min":13.872,"15min":13.872},"last":3.706}]},{"osd":5,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.302,"15min":3.6909999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0270000000000001,"5min":14.105,"15min":14.105},"last":3.6160000000000001},{"interface":"front","average":{"1min":2.294,"5min":2.4079999999999999,"15min":3.2120000000000002},"min":{"1min":0.441,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.532,"5min":19.846,"15min":19.846},"last":8.0079999999999991}]},{"osd":6,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.7400000000000002,"15min":4.4580000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.771000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.175,"5min":2.3300000000000001,"15min":3.7250000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.0650000000000004,"5min":13.82,"15min":13.82},"last":3.6499999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.21,"5min":2.6949999999999998,"15min":3.6230000000000002},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":17.523,"5min":19.866,"15min":19.866},"last":7.9669999999999996},{"interface":"front","average":{"1min":3.6720000000000002,"5min":2.899,"15min":4.5090000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":17.792999999999999,"5min":29.77,"15min":29.77},"last":0.376}]}]},{"osd":2,"up_from":120,"seq":515396075708,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1876972,"kb_used_data":1403364,"kb_used_omap":7,"kb_used_meta":473400,"kb_avail":91868180,"statfs":{"total":95995035648,"available":94073016320,"internally_reserved":0,"allocated":1437044736,"data_stored":1436103541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":484761681},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":4.0060000000000002,"15min":4.4710000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":18.210999999999999,"5min":66.784000000000006,"15min":66.784000000000006},"last":0.307},{"interface":"front","average":{"1min":2.7639999999999998,"5min":4.3840000000000003,"15min":5.056},"min":{"1min":0.317,"5min":0.252,"15min":0.252},"max":{"1min":7.6109999999999998,"5min":67.251999999999995,"15min":67.251999999999995},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":5.359,"5min":5.1619999999999999,"15min":2.8380000000000001},"min":{"1min":0.309,"5min":0.218,"15min":0.19},"max":{"1min":44.625,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.0950000000000002,"5min":4.2089999999999996,"15min":2.4049999999999998},"min":{"1min":0.27300000000000002,"5min":0.214,"15min":0.214},"max":{"1min":9.7729999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.498}]},{"osd":3,"last update":"Sat Jun 10 21:34:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":4.5810000000000004,"15min":5.0209999999999999},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":29.719999999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":61.655000000000001},{"interface":"front","average":{"1min":3.0630000000000002,"5min":4.798,"15min":5.7949999999999999},"min":{"1min":0.40999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.406000000000001,"5min":138.095,"15min":138.095},"last":57.744}]},{"osd":4,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0070000000000001,"5min":3.1970000000000001,"15min":2.778},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":6.1689999999999996,"5min":110.337,"15min":224.00999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":4.2080000000000002,"5min":3.972,"15min":3.105},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":29.486999999999998,"5min":110.2,"15min":223.28399999999999},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":1.671,"15min":1.4379999999999999},"min":{"1min":0.44700000000000001,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":9.7460000000000004,"5min":14.180999999999999,"15min":14.180999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.0329999999999999,"5min":1.968,"15min":1.6950000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":6.242,"5min":16.125,"15min":16.125},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":2.0590000000000002,"15min":1.7669999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":6.1920000000000002,"5min":16.183,"15min":16.183},"last":0.53200000000000003},{"interface":"front","average":{"1min":4.3940000000000001,"5min":2.782,"15min":2.113},"min":{"1min":0.48299999999999998,"5min":0.35999999999999999,"15min":0.33600000000000002},"max":{"1min":29.538,"5min":29.538,"15min":29.538},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:33:30 2023","interfaces":[{"interface":"back","average":{"1min":7.9089999999999998,"5min":4.1289999999999996,"15min":2.2309999999999999},"min":{"1min":0.41199999999999998,"5min":0.34799999999999998,"15min":0.33400000000000002},"max":{"1min":73.445999999999998,"5min":113.913,"15min":113.913},"last":0.58999999999999997},{"interface":"front","average":{"1min":7.8849999999999998,"5min":4.4939999999999998,"15min":2.577},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":73.475999999999999,"5min":113.937,"15min":113.937},"last":0.43099999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824068,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1658240,"kb_used_data":1434232,"kb_used_omap":7,"kb_used_meta":223992,"kb_avail":92086912,"statfs":{"total":95995035648,"available":94296997888,"internally_reserved":0,"allocated":1468653568,"data_stored":1467640440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":229367827},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":4.2629999999999999,"15min":5.1200000000000001},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":53.756999999999998,"5min":93.992999999999995,"15min":93.992999999999995},"last":2.5259999999999998},{"interface":"front","average":{"1min":5.5069999999999997,"5min":3.9260000000000002,"15min":4.8639999999999999},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.216},"max":{"1min":48.247,"5min":95.623999999999995,"15min":95.623999999999995},"last":0.90900000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.3230000000000004,"5min":4.1130000000000004,"15min":3.7970000000000002},"min":{"1min":0.30399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":20.574999999999999,"5min":20.574999999999999,"15min":20.574999999999999},"last":2.0649999999999999},{"interface":"front","average":{"1min":4.6429999999999998,"5min":3.2599999999999998,"15min":3.4100000000000001},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.545,"5min":17.384,"15min":17.384},"last":4.9370000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2240000000000002,"5min":4.6740000000000004,"15min":10.172000000000001},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.529999999999999,"5min":87.555999999999997,"15min":87.555999999999997},"last":2.5760000000000001},{"interface":"front","average":{"1min":5.1399999999999997,"5min":6.5449999999999999,"15min":11.743},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.541,"5min":80.566999999999993,"15min":80.566999999999993},"last":2.6110000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":1.621,"15min":1.456},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.417,"5min":13.417,"15min":13.417},"last":0.52200000000000002},{"interface":"front","average":{"1min":3.133,"5min":2.835,"15min":2.7709999999999999},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.311},"max":{"1min":9.2300000000000004,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":1.383,"15min":1.389},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.8810000000000002,"5min":6.891,"15min":6.891},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.931,"5min":1.762,"15min":1.54},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":2.8580000000000001,"15min":2.7949999999999999},"min":{"1min":0.505,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":9.2579999999999991,"5min":19.805,"15min":19.805},"last":0.64100000000000001},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.4370000000000001,"15min":1.4390000000000001},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9039999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":0.433}]},{"osd":7,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.54,"15min":3.032},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":79.984999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":5.7229999999999999,"5min":2.1070000000000002,"15min":1.645},"min":{"1min":0.48199999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":80.272999999999996,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.47399999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019333,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1151244,"kb_used_data":553540,"kb_used_omap":0,"kb_used_meta":597696,"kb_avail":92593908,"statfs":{"total":95995035648,"available":94816161792,"internally_reserved":0,"allocated":566824960,"data_stored":566263522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612040704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:37 2023","interfaces":[{"interface":"back","average":{"1min":10.459,"5min":5.0209999999999999,"15min":8.7919999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":234.203,"5min":234.203,"15min":234.203},"last":8.298},{"interface":"front","average":{"1min":11.193,"5min":5.8780000000000001,"15min":8.5860000000000003},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":234.19,"5min":234.19,"15min":234.19},"last":3.847}]},{"osd":1,"last update":"Sat Jun 10 21:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.782,"15min":2.9159999999999999},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.7170000000000001,"5min":13.461,"15min":182.035},"last":0.434},{"interface":"front","average":{"1min":3.5049999999999999,"5min":2.1360000000000001,"15min":2.8359999999999999},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.151,"5min":10.255000000000001,"15min":181.73099999999999},"last":4.4429999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":4.3380000000000001,"5min":2.4700000000000002,"15min":1.8080000000000001},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":40.311,"5min":40.311,"15min":40.311},"last":2.1640000000000001},{"interface":"front","average":{"1min":3.3690000000000002,"5min":2.3300000000000001,"15min":1.714},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":40.268999999999998,"5min":40.268999999999998,"15min":40.268999999999998},"last":15.148999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":14.234,"5min":4.4939999999999998,"15min":3.0920000000000001},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":234.91900000000001,"5min":234.91900000000001,"15min":234.91900000000001},"last":4.2629999999999999},{"interface":"front","average":{"1min":15.222,"5min":6.1619999999999999,"15min":3.4159999999999999},"min":{"1min":0.38200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":235.05099999999999,"5min":235.05099999999999,"15min":235.05099999999999},"last":8.5609999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.58999999999999997,"15min":1.022},"min":{"1min":0.28699999999999998,"5min":0.26000000000000001,"15min":0.23999999999999999},"max":{"1min":0.55200000000000005,"5min":6.742,"15min":13.941000000000001},"last":0.497},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.61399999999999999,"15min":1.0469999999999999},"min":{"1min":0.32400000000000001,"5min":0.29899999999999999,"15min":0.247},"max":{"1min":5.25,"5min":5.4000000000000004,"15min":11.904999999999999},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50800000000000001,"15min":1.0309999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25600000000000001},"max":{"1min":1.7210000000000001,"5min":3.077,"15min":8.2599999999999998},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.63500000000000001,"15min":1.1000000000000001},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":1.71,"5min":6.8300000000000001,"15min":13.845000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":0.80500000000000005,"15min":1.2889999999999999},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":39.713999999999999,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.629},{"interface":"front","average":{"1min":2.2050000000000001,"5min":0.82699999999999996,"15min":1.2310000000000001},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":37.429000000000002,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.57199999999999995}]}]},{"osd":5,"up_from":0,"seq":154618823105,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1178075136,"data_stored":1178075136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1990987776,"data_stored":1990983704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1415450624,"data_stored":1415446552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1452670976,"data_stored":1452670976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":558108672,"data_stored":558104600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":566034432,"data_stored":566034432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:34:34.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:34.836 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:35.388 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:35.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:35.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:35.521 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:35.548 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:35.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:34:35.579 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:34:35.579 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:34:35.595 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:35.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:35.619 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:34:35.651 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:34:35.704 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:35.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:34:35.730 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:35.743 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:34:35.775 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:35.779 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:34:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:35.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:35.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:34:35.824 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:35 smithi064 ceph-mon[180297]: pgmap v1097: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 5917/101583 objects degraded (5.825%); 1987/101583 objects misplaced (1.956%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:35.825 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1070265638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:35.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:35 smithi064 ceph-mon[186706]: pgmap v1097: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 5917/101583 objects degraded (5.825%); 1987/101583 objects misplaced (1.956%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:35.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1070265638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:35.865 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:34:35.871 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.872 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.873 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:34:35.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:35 smithi152 ceph-mon[156819]: pgmap v1097: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 5917/101583 objects degraded (5.825%); 1987/101583 objects misplaced (1.956%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1070265638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:36.004 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:36.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/101583 objects degraded (5.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:36.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/101583 objects degraded (5.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:36.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/101583 objects degraded (5.825%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:37.712 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:34:37.737 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:37 smithi064 ceph-mon[180297]: pgmap v1098: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/98319 objects degraded (6.018%); 1761/98319 objects misplaced (1.791%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:34:37.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:37 smithi064 ceph-mon[186706]: pgmap v1098: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/98319 objects degraded (6.018%); 1761/98319 objects misplaced (1.791%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:34:37.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:37.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:37.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:37.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:34:37.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:37 smithi152 ceph-mon[156819]: pgmap v1098: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 2.0 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 490 op/s; 5917/98319 objects degraded (6.018%); 1761/98319 objects misplaced (1.791%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:34:37.976 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:38.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:38.485 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:38.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:38.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:38.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:38.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:38.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:38.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:38 smithi064 ceph-mon[180297]: pgmap v1099: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/95316 objects degraded (6.208%); 1761/95316 objects misplaced (1.848%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:34:38.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:38 smithi064 ceph-mon[186706]: pgmap v1099: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/95316 objects degraded (6.208%); 1761/95316 objects misplaced (1.848%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:34:38.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:38.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:38 smithi152 ceph-mon[156819]: pgmap v1099: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.9 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/95316 objects degraded (6.208%); 1761/95316 objects misplaced (1.848%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:34:38.971 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:39.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:39.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:39.121 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:39.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:39.199 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:39.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:39.304 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:39.333 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:39.386 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:39.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:39.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:39.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:39.522 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:39.578 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:39.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:39.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:39.693 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:39.794 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:39.804 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:40.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:40.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:40.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:34:40.199 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:39 smithi152 ceph-mon[156819]: pgmap v1100: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 5917/90648 objects degraded (6.527%); 1553/90648 objects misplaced (1.713%); 3.2 MiB/s, 50 objects/s recovering 2023-06-10T21:34:40.200 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:40.249 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:34:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:40.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:34:40.266 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:39 smithi064 ceph-mon[180297]: pgmap v1100: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 5917/90648 objects degraded (6.527%); 1553/90648 objects misplaced (1.713%); 3.2 MiB/s, 50 objects/s recovering 2023-06-10T21:34:40.267 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:39 smithi064 ceph-mon[186706]: pgmap v1100: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 5917/90648 objects degraded (6.527%); 1553/90648 objects misplaced (1.713%); 3.2 MiB/s, 50 objects/s recovering 2023-06-10T21:34:40.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:34:40.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:40.386 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:40.433 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:40.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:40.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:40.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:40.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:40.906 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:40.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:41.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:41.099 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:40 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/90648 objects degraded (6.527%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:41.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/90648 objects degraded (6.527%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:41.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:40 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/90648 objects degraded (6.527%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:41.993 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:34:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4005171352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[180297]: pgmap v1101: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/88101 objects degraded (6.716%); 1553/88101 objects misplaced (1.763%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4005171352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:41 smithi064 ceph-mon[186706]: pgmap v1101: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/88101 objects degraded (6.716%); 1553/88101 objects misplaced (1.763%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:42.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4005171352' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:41 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:34:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:41 smithi152 ceph-mon[156819]: pgmap v1101: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/88101 objects degraded (6.716%); 1553/88101 objects misplaced (1.763%); 2.3 MiB/s, 36 objects/s recovering 2023-06-10T21:34:42.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:42.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:42.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:42.709 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:42.737 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:42.787 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:42.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:42.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:42.925 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:43.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:34:43.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[180297]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[180297]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:43.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[180297]: osdmap e302: 8 total, 8 up, 6 in 2023-06-10T21:34:43.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:42] "GET /metrics HTTP/1.1" 200 224086 "" "Prometheus/2.33.4" 2023-06-10T21:34:43.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:34:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[186706]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[186706]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:43.163 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:42 smithi064 ceph-mon[186706]: osdmap e302: 8 total, 8 up, 6 in 2023-06-10T21:34:43.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:42 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:34:43.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:42 smithi152 ceph-mon[156819]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:42 smithi152 ceph-mon[156819]: failed to encode map e302 with expected crc 2023-06-10T21:34:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:42 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:34:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:42 smithi152 ceph-mon[156819]: osdmap e302: 8 total, 8 up, 6 in 2023-06-10T21:34:43.620 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:43.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:43.734 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:43.762 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:43.804 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:43.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:43.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:43.960 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:44.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:43 smithi152 ceph-mon[156819]: pgmap v1103: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 5917/86661 objects degraded (6.828%); 1553/86661 objects misplaced (1.792%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:44.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:44.273 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:43 smithi064 ceph-mon[180297]: pgmap v1103: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 5917/86661 objects degraded (6.828%); 1553/86661 objects misplaced (1.792%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:44.273 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:43 smithi064 ceph-mon[186706]: pgmap v1103: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.8 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 5917/86661 objects degraded (6.828%); 1553/86661 objects misplaced (1.792%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:34:44.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:44.374 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:44.476 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:44.849 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:44.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:44.957 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:44.985 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:45.034 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:45.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:45.132 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:45.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:45.252 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:45.321 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:34:45.321 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:34:45.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:34:45.321 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24681 11.5 11:adf48b79:::benchmark_data_smithi064_385725_object24680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.19902588299999, 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28889557700000001, 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.322 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24681 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:34:45.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115708+0000", 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115733+0000", 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5386000000000001e-05 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115742+0000", 2023-06-10T21:34:45.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0780000000000002e-06 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115749+0000", 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4110000000000001e-06 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115922+0000", 2023-06-10T21:34:45.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173318 2023-06-10T21:34:45.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116070+0000", 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014733500000000001 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116228+0000", 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158376 2023-06-10T21:34:45.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120319+0000", 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040913800000000004 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324910+0000", 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.204591211 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404530+0000", 2023-06-10T21:34:45.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079619326000000004 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404570+0000", 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9895999999999999e-05 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404605+0000", 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5114999999999998e-05 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24682 11.a 11:59387607:::benchmark_data_smithi064_385725_object24681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.19898071700001, 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28861932000000001, 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.334 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24682 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:34:45.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:34:45.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115775+0000", 2023-06-10T21:34:45.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1480000000000001e-05 2023-06-10T21:34:45.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115782+0000", 2023-06-10T21:34:45.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5980000000000002e-06 2023-06-10T21:34:45.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115787+0000", 2023-06-10T21:34:45.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2730000000000002e-06 2023-06-10T21:34:45.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115977+0000", 2023-06-10T21:34:45.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019004300000000001 2023-06-10T21:34:45.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116096+0000", 2023-06-10T21:34:45.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119048 2023-06-10T21:34:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:34:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116239+0000", 2023-06-10T21:34:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014270900000000001 2023-06-10T21:34:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117796+0000", 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001557014 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.121220+0000", 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034239399999999999 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404279+0000", 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.283058645 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404326+0000", 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7493000000000001e-05 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404374+0000", 2023-06-10T21:34:45.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7534000000000003e-05 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24686 11.5 11:aad12278:::benchmark_data_smithi064_385725_object24685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.351 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.198187512, 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.288163171, 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24686 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:34:45.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116547+0000", 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:45.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116571+0000", 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4745000000000001e-05 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116584+0000", 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2991e-05 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116593+0000", 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2369999999999992e-06 2023-06-10T21:34:45.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117598+0000", 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010048850000000001 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117717+0000", 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119888 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117865+0000", 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014747999999999999 2023-06-10T21:34:45.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120325+0000", 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002459934 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325488+0000", 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20516282499999999 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404638+0000", 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079150140999999993 2023-06-10T21:34:45.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404675+0000", 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7067000000000003e-05 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404711+0000", 2023-06-10T21:34:45.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5892000000000001e-05 2023-06-10T21:34:45.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.363 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.364 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.364 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24687 11.b 11:d00b5318:::benchmark_data_smithi064_385725_object24686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.364 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.19813400999999, 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28619599200000001, 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24687 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:34:45.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116622+0000", 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1503999999999999e-05 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116631+0000", 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6300000000000004e-06 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116637+0000", 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4819999999999999e-06 2023-06-10T21:34:45.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116750+0000", 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011303600000000001 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116884+0000", 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013408300000000001 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117022+0000", 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013734700000000001 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119935+0000", 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029137130000000001 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324876+0000", 2023-06-10T21:34:45.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20494073099999999 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402697+0000", 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077820749999999994 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402754+0000", 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6580999999999999e-05 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402797+0000", 2023-06-10T21:34:45.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3612999999999997e-05 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24688 11.5 11:aaadba38:::benchmark_data_smithi064_385725_object24687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:34:45.376 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.19598167999999, 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.286064453, 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.377 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24688 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118752+0000", 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118776+0000", 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4070000000000002e-05 2023-06-10T21:34:45.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118794+0000", 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7810000000000001e-05 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118804+0000", 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0053e-05 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118981+0000", 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000177503 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119111+0000", 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129488 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119301+0000", 2023-06-10T21:34:45.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018976 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120860+0000", 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015592309999999999 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325669+0000", 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20480867 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404750+0000", 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079081311000000001 2023-06-10T21:34:45.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404787+0000", 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7070999999999998e-05 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404818+0000", 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0985000000000002e-05 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.387 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24693 11.9 11:93e9483b:::benchmark_data_smithi064_385725_object24692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.194360141, 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.282496517, 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.388 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24693 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120374+0000", 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120401+0000", 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7317e-05 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120419+0000", 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7456999999999999e-05 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120427+0000", 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7839999999999992e-06 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120485+0000", 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7590999999999998e-05 2023-06-10T21:34:45.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120609+0000", 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012447700000000001 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120750+0000", 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140925 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.122064+0000", 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001313374 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325118+0000", 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.203053815 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402795+0000", 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077677874999999993 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402838+0000", 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2296000000000003e-05 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402872+0000", 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3819999999999998e-05 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24695 11.b 11:d780a7ce:::benchmark_data_smithi064_385725_object24694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 188.19429243100001, 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28245634200000003, 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.399 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24695 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120442+0000", 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120465+0000", 2023-06-10T21:34:45.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3102e-05 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120474+0000", 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3240000000000003e-06 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120480+0000", 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6449999999999999e-06 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120523+0000", 2023-06-10T21:34:45.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2395000000000003e-05 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120636+0000", 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112953 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120751+0000", 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011537200000000001 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.121926+0000", 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001175155 2023-06-10T21:34:45.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324953+0000", 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20302672799999999 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402834+0000", 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077880854999999999 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402870+0000", 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6094999999999999e-05 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402899+0000", 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9294000000000001e-05 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99515 11.1 11:8ea0c8ed:::benchmark_data_smithi064_385725_object15242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.111272043, 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30862035399999999, 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.410 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99515 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203461+0000", 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203469+0000", 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203482+0000", 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3318e-05 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203491+0000", 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7749999999999998e-06 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203533+0000", 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2716999999999998e-05 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203662+0000", 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012818000000000001 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203771+0000", 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109725 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.213580+0000", 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098082800000000008 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.218941+0000", 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053615360000000001 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.511973+0000", 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29303174399999998 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512034+0000", 2023-06-10T21:34:45.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1147000000000003e-05 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512083+0000", 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9351000000000002e-05 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.420 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99523 11.1 11:88f160b1:::benchmark_data_smithi064_385725_object15250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.099483862, 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.311404917, 2023-06-10T21:34:45.421 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99523 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:34:45.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215250+0000", 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215255+0000", 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0019999999999997e-06 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215263+0000", 2023-06-10T21:34:45.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1470000000000001e-06 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215270+0000", 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4099999999999996e-06 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215321+0000", 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1413999999999999e-05 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215475+0000", 2023-06-10T21:34:45.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153478 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215610+0000", 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135716 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.219102+0000", 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034921179999999998 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.220787+0000", 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001685099 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526574+0000", 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30578659800000002 2023-06-10T21:34:45.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526618+0000", 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4140000000000001e-05 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526656+0000", 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8003999999999997e-05 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.432 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99530 11.1 11:816041a6:::benchmark_data_smithi064_385725_object15257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 83.089012064000002, 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30104990300000001, 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99530 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:34:45.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225722+0000", 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225728+0000", 2023-06-10T21:34:45.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9259999999999997e-06 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225741+0000", 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2894999999999999e-05 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225749+0000", 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9209999999999992e-06 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225790+0000", 2023-06-10T21:34:45.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1196999999999999e-05 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225895+0000", 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010484699999999999 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.226005+0000", 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110655 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229259+0000", 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032538469999999998 2023-06-10T21:34:45.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229476+0000", 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021646299999999999 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526694+0000", 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29721868499999998 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526739+0000", 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4938999999999998e-05 2023-06-10T21:34:45.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526773+0000", 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3716e-05 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101022 11.b 11:d995299d:::benchmark_data_smithi064_385725_object16749:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.074389304999997, 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29097264099999998, 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.444 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101022 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240344+0000", 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240352+0000", 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5129999999999999e-06 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240369+0000", 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7329e-05 2023-06-10T21:34:45.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240378+0000", 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8529999999999998e-06 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240590+0000", 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000211561 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240700+0000", 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110667 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240808+0000", 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010772000000000001 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.251610+0000", 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010801638000000001 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.255480+0000", 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038706489999999999 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531216+0000", 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27573550899999999 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.453 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531274+0000", 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8594000000000002e-05 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531318+0000", 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4014999999999998e-05 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.454 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101024 11.9 11:99976c21:::benchmark_data_smithi064_385725_object16751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.067856759999998, 2023-06-10T21:34:45.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.284438739, 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101024 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.456 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246877+0000", 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:34:45.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246883+0000", 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4069999999999998e-06 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246896+0000", 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459e-05 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246903+0000", 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4580000000000004e-06 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.248757+0000", 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00185384 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.248899+0000", 2023-06-10T21:34:45.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141674 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.249016+0000", 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117834 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.255313+0000", 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062968360000000001 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.261888+0000", 2023-06-10T21:34:45.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065752049999999998 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531237+0000", 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26934889699999998 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531279+0000", 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1313999999999998e-05 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531317+0000", 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8510000000000002e-05 2023-06-10T21:34:45.465 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101027 11.1 11:84832f10:::benchmark_data_smithi064_385725_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.067818152000001, 2023-06-10T21:34:45.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29481494899999999, 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101027 2023-06-10T21:34:45.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.468 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246919+0000", 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.565e-06 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246925+0000", 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7769999999999999e-06 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246930+0000", 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6949999999999999e-06 2023-06-10T21:34:45.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252174+0000", 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052442779999999998 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252278+0000", 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010410499999999999 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252344+0000", 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6168999999999996e-05 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.256536+0000", 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041921479999999997 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531254+0000", 2023-06-10T21:34:45.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27471757699999999 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541651+0000", 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010397027 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541697+0000", 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5889000000000001e-05 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541732+0000", 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5098999999999997e-05 2023-06-10T21:34:45.476 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101036 11.9 11:91c4f911:::benchmark_data_smithi064_385725_object16763:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:34:45.477 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 80.051952383, 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27844923399999999, 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101036 2023-06-10T21:34:45.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262781+0000", 2023-06-10T21:34:45.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262788+0000", 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4790000000000001e-06 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262804+0000", 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6654999999999999e-05 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262812+0000", 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6799999999999993e-06 2023-06-10T21:34:45.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262862+0000", 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9917000000000003e-05 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.263004+0000", 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014181600000000001 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.263145+0000", 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140937 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.266594+0000", 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003449662 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.266716+0000", 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012186800000000001 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541154+0000", 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27443805900000001 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541196+0000", 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2249999999999997e-05 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541232+0000", 2023-06-10T21:34:45.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5506000000000002e-05 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126745 11.1 11:8766de5a:::benchmark_data_smithi064_385725_object42472:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:34:45.488 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.891149069000001, 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33496997699999997, 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126745 2023-06-10T21:34:45.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423584+0000", 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:34:45.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423595+0000", 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1586000000000001e-05 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423618+0000", 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2609999999999999e-05 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423630+0000", 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2449000000000001e-05 2023-06-10T21:34:45.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423686+0000", 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5452999999999999e-05 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423832+0000", 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146155 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423960+0000", 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012785100000000001 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.434684+0000", 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01072394 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625142+0000", 2023-06-10T21:34:45.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19045854700000001 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758437+0000", 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13329444800000001 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758501+0000", 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3959000000000002e-05 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758556+0000", 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5461000000000003e-05 2023-06-10T21:34:45.498 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126747 11.1 11:8706db2c:::benchmark_data_smithi064_385725_object42474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:34:45.499 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.891090035000001, 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33505084800000001, 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126747 2023-06-10T21:34:45.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.502 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423648+0000", 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1719999999999999e-06 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.503 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423654+0000", 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8340000000000004e-06 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423660+0000", 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7059999999999998e-06 2023-06-10T21:34:45.504 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437557+0000", 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013896382000000001 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.505 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437673+0000", 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116501 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437787+0000", 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113354 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.506 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625022+0000", 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18723535299999999 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625168+0000", 2023-06-10T21:34:45.507 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146494 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758602+0000", 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13343400399999999 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.508 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758650+0000", 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7052000000000003e-05 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758696+0000", 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6424000000000001e-05 2023-06-10T21:34:45.509 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126748 11.8 11:1c2ac95f:::benchmark_data_smithi064_385725_object42475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:34:45.510 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.891052067, 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32430128899999999, 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126748 2023-06-10T21:34:45.511 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.512 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423682+0000", 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:34:45.513 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423688+0000", 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6160000000000001e-06 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.514 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423699+0000", 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1623e-05 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423707+0000", 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8830000000000006e-06 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.515 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423759+0000", 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1573000000000002e-05 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.516 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423869+0000", 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109951 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423973+0000", 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104144 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.517 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437147+0000", 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013174289 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.624707+0000", 2023-06-10T21:34:45.518 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.187559904 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747900+0000", 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12319322200000001 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.519 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747947+0000", 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6390000000000001e-05 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747984+0000", 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7703999999999997e-05 2023-06-10T21:34:45.520 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126751 11.1 11:8ace7e90:::benchmark_data_smithi064_385725_object42478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.875609351000001, 2023-06-10T21:34:45.521 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31970830500000003, 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126751 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.522 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.523 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439125+0000", 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:34:45.524 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439130+0000", 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5249999999999996e-06 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.525 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439144+0000", 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3719e-05 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439152+0000", 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2449999999999998e-06 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.526 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625233+0000", 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.186081301 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.527 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625322+0000", 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8226999999999994e-05 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625417+0000", 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5429999999999999e-05 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.528 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631580+0000", 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061630039999999997 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.529 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631643+0000", 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3408e-05 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758744+0000", 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.127100451 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.530 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758794+0000", 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0526999999999999e-05 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758834+0000", 2023-06-10T21:34:45.531 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9748e-05 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126756 11.1 11:81432d08:::benchmark_data_smithi064_385725_object42483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:34:45.532 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 30.875571437000001, 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.328752612, 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.533 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126756 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.534 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439163+0000", 2023-06-10T21:34:45.535 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439166+0000", 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1990000000000002e-06 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.536 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439174+0000", 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439180+0000", 2023-06-10T21:34:45.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8000000000000004e-06 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627884+0000", 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.188704802 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.538 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627970+0000", 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5963000000000003e-05 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.628045+0000", 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4239000000000005e-05 2023-06-10T21:34:45.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631687+0000", 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036425799999999999 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631697+0000", 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0166999999999999e-05 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767840+0000", 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13614217200000001 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767882+0000", 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2262000000000003e-05 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767916+0000", 2023-06-10T21:34:45.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4548000000000002e-05 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:138782 11.8 11:10cd1588:::benchmark_data_smithi064_385725_object54509:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-06-10T21:34:45.543 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.3855951199999996, 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.288210731, 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:34:45.544 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 138782 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:34:45.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929142+0000", 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.667e-06 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929148+0000", 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4040000000000003e-06 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929153+0000", 2023-06-10T21:34:45.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0119999999999996e-06 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934464+0000", 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053112979999999999 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934571+0000", 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106839 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934679+0000", 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108268 2023-06-10T21:34:45.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.941381+0000", 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067020270000000002 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.189309+0000", 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24792794200000001 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217251+0000", 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027941506000000001 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:34:45.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217304+0000", 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3751000000000001e-05 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217351+0000", 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6397000000000003e-05 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.554 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:34:45.555 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:34:45.555 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:34:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:45.556 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:45.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:45.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:45.695 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:34:45.711 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:34:45.753 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:46.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:45 smithi064 ceph-mon[186706]: from='client.56209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:46.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:45 smithi064 ceph-mon[186706]: pgmap v1104: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/83403 objects degraded (7.094%); 1316/83403 objects misplaced (1.578%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:46.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:45 smithi064 ceph-mon[180297]: from='client.56209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:46.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:45 smithi064 ceph-mon[180297]: pgmap v1104: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/83403 objects degraded (7.094%); 1316/83403 objects misplaced (1.578%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:46.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:34:46.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:45 smithi152 ceph-mon[156819]: from='client.56209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:34:46.455 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:45 smithi152 ceph-mon[156819]: pgmap v1104: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.7 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/83403 objects degraded (7.094%); 1316/83403 objects misplaced (1.578%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:46.954 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:46.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:47.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:47.145 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:47.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/86661 objects degraded (6.828%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:47.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/86661 objects degraded (6.828%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:47.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/86661 objects degraded (6.828%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:47.863 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1103,"stamp":"2023-06-10T21:34:43.863081+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1892095910,"num_objects":28887,"num_object_clones":0,"num_object_copies":86661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5917,"num_objects_misplaced":1553,"num_objects_unfound":0,"num_objects_dirty":28887,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":139940,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3166,"num_bytes_recovered":206511826,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111374,"ondisk_log_size":111374,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8827760,"kb_used_data":6055552,"kb_used_omap":30,"kb_used_meta":2771809,"kb_avail":553643152,"statfs":{"total":575970213888,"available":566930587648,"internally_reserved":0,"allocated":6200885248,"data_stored":6195830502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2838332454},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,5,2,0,0,9],"upper_bound":256},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-325976064,"num_objects":-4974,"num_object_clones":0,"num_object_copies":-14922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-434,"num_objects_unfound":0,"num_objects_dirty":-4974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4974,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":434,"num_bytes_recovered":28442624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006142"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":294,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421898+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:34:14.421898+0000","last_peered":"2023-06-10T21:34:14.421898+0000","last_clean":"2023-06-10T21:34:14.421898+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:34:14.421898+0000","last_undegraded":"2023-06-10T21:34:14.421898+0000","last_fullsized":"2023-06-10T21:34:14.421898+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":235,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625083+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:34:14.625083+0000","last_peered":"2023-06-10T21:34:14.625083+0000","last_clean":"2023-06-10T21:34:14.625083+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:34:14.625083+0000","last_undegraded":"2023-06-10T21:34:14.625083+0000","last_fullsized":"2023-06-10T21:34:14.625083+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":210,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625103+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:34:14.625103+0000","last_peered":"2023-06-10T21:34:14.625103+0000","last_clean":"2023-06-10T21:34:14.625103+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:34:14.625103+0000","last_undegraded":"2023-06-10T21:34:14.625103+0000","last_fullsized":"2023-06-10T21:34:14.625103+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":159,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:33.086139+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:34:33.086139+0000","last_peered":"2023-06-10T21:34:33.086139+0000","last_clean":"2023-06-10T21:34:33.086139+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:34:33.086139+0000","last_undegraded":"2023-06-10T21:34:33.086139+0000","last_fullsized":"2023-06-10T21:34:33.086139+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":275,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439647+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:34:14.439647+0000","last_peered":"2023-06-10T21:34:14.439647+0000","last_clean":"2023-06-10T21:34:14.439647+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:34:14.439647+0000","last_undegraded":"2023-06-10T21:34:14.439647+0000","last_fullsized":"2023-06-10T21:34:14.439647+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":268,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868487+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:34:28.868487+0000","last_peered":"2023-06-10T21:34:28.868487+0000","last_clean":"2023-06-10T21:34:28.868487+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:34:28.868487+0000","last_undegraded":"2023-06-10T21:34:28.868487+0000","last_fullsized":"2023-06-10T21:34:28.868487+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868509+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:34:28.868509+0000","last_peered":"2023-06-10T21:34:28.868509+0000","last_clean":"2023-06-10T21:34:28.868509+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:34:28.868509+0000","last_undegraded":"2023-06-10T21:34:28.868509+0000","last_fullsized":"2023-06-10T21:34:28.868509+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":249,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439205+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:34:14.439205+0000","last_peered":"2023-06-10T21:34:14.439205+0000","last_clean":"2023-06-10T21:34:14.439205+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:34:14.439205+0000","last_undegraded":"2023-06-10T21:34:14.439205+0000","last_fullsized":"2023-06-10T21:34:14.439205+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420628+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:34:14.420628+0000","last_peered":"2023-06-10T21:34:14.420628+0000","last_clean":"2023-06-10T21:34:14.420628+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:34:14.420628+0000","last_undegraded":"2023-06-10T21:34:14.420628+0000","last_fullsized":"2023-06-10T21:34:14.420628+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439479+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:34:14.439479+0000","last_peered":"2023-06-10T21:34:14.439479+0000","last_clean":"2023-06-10T21:34:14.439479+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:34:14.439479+0000","last_undegraded":"2023-06-10T21:34:14.439479+0000","last_fullsized":"2023-06-10T21:34:14.439479+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":28,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418776+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:34:14.418776+0000","last_peered":"2023-06-10T21:34:14.418776+0000","last_clean":"2023-06-10T21:34:14.418776+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:34:14.418776+0000","last_undegraded":"2023-06-10T21:34:14.418776+0000","last_fullsized":"2023-06-10T21:34:14.418776+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":263,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421802+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:34:14.421802+0000","last_peered":"2023-06-10T21:34:14.421802+0000","last_clean":"2023-06-10T21:34:14.421802+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:34:14.421802+0000","last_undegraded":"2023-06-10T21:34:14.421802+0000","last_fullsized":"2023-06-10T21:34:14.421802+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":171,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421482+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:34:14.421482+0000","last_peered":"2023-06-10T21:34:14.421482+0000","last_clean":"2023-06-10T21:34:14.421482+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:34:14.421482+0000","last_undegraded":"2023-06-10T21:34:14.421482+0000","last_fullsized":"2023-06-10T21:34:14.421482+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":32,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420701+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:34:14.420701+0000","last_peered":"2023-06-10T21:34:14.420701+0000","last_clean":"2023-06-10T21:34:14.420701+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:34:14.420701+0000","last_undegraded":"2023-06-10T21:34:14.420701+0000","last_fullsized":"2023-06-10T21:34:14.420701+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":300,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420752+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:34:14.420752+0000","last_peered":"2023-06-10T21:34:14.420752+0000","last_clean":"2023-06-10T21:34:14.420752+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:34:14.420752+0000","last_undegraded":"2023-06-10T21:34:14.420752+0000","last_fullsized":"2023-06-10T21:34:14.420752+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":235,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439452+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:34:14.439452+0000","last_peered":"2023-06-10T21:34:14.439452+0000","last_clean":"2023-06-10T21:34:14.439452+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:34:14.439452+0000","last_undegraded":"2023-06-10T21:34:14.439452+0000","last_fullsized":"2023-06-10T21:34:14.439452+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":158,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868005+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:34:28.868005+0000","last_peered":"2023-06-10T21:34:28.868005+0000","last_clean":"2023-06-10T21:34:28.868005+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:34:28.868005+0000","last_undegraded":"2023-06-10T21:34:28.868005+0000","last_fullsized":"2023-06-10T21:34:28.868005+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":229,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.419126+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:34:14.419126+0000","last_peered":"2023-06-10T21:34:14.419126+0000","last_clean":"2023-06-10T21:34:14.419126+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:34:14.419126+0000","last_undegraded":"2023-06-10T21:34:14.419126+0000","last_fullsized":"2023-06-10T21:34:14.419126+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":277,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418624+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:34:14.418624+0000","last_peered":"2023-06-10T21:34:14.418624+0000","last_clean":"2023-06-10T21:34:14.418624+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:34:14.418624+0000","last_undegraded":"2023-06-10T21:34:14.418624+0000","last_fullsized":"2023-06-10T21:34:14.418624+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":158,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868442+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:34:28.868442+0000","last_peered":"2023-06-10T21:34:28.868442+0000","last_clean":"2023-06-10T21:34:28.868442+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:34:28.868442+0000","last_undegraded":"2023-06-10T21:34:28.868442+0000","last_fullsized":"2023-06-10T21:34:28.868442+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":277,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867677+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:34:28.867677+0000","last_peered":"2023-06-10T21:34:28.867677+0000","last_clean":"2023-06-10T21:34:28.867677+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:34:28.867677+0000","last_undegraded":"2023-06-10T21:34:28.867677+0000","last_fullsized":"2023-06-10T21:34:28.867677+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":253,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439543+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:34:14.439543+0000","last_peered":"2023-06-10T21:34:14.439543+0000","last_clean":"2023-06-10T21:34:14.439543+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:34:14.439543+0000","last_undegraded":"2023-06-10T21:34:14.439543+0000","last_fullsized":"2023-06-10T21:34:14.439543+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"302'8807","reported_seq":8850,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:43.063081+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:34:43.063081+0000","last_peered":"2023-06-10T21:34:43.063081+0000","last_clean":"2023-06-10T21:34:43.063081+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:34:43.063081+0000","last_undegraded":"2023-06-10T21:34:43.063081+0000","last_fullsized":"2023-06-10T21:34:43.063081+0000","mapping_epoch":269,"log_start":"281'2400","ondisk_log_start":"281'2400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8807,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418658+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:34:14.418658+0000","last_peered":"2023-06-10T21:34:14.418658+0000","last_clean":"2023-06-10T21:34:14.418658+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:34:14.418658+0000","last_undegraded":"2023-06-10T21:34:14.418658+0000","last_fullsized":"2023-06-10T21:34:14.418658+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418253+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:34:14.418253+0000","last_peered":"2023-06-10T21:34:14.418253+0000","last_clean":"2023-06-10T21:34:14.418253+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:34:14.418253+0000","last_undegraded":"2023-06-10T21:34:14.418253+0000","last_fullsized":"2023-06-10T21:34:14.418253+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"302'8864","reported_seq":8907,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:43.053506+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:34:43.053506+0000","last_peered":"2023-06-10T21:34:43.053506+0000","last_clean":"2023-06-10T21:34:43.053506+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:34:43.053506+0000","last_undegraded":"2023-06-10T21:34:43.053506+0000","last_fullsized":"2023-06-10T21:34:43.053506+0000","mapping_epoch":269,"log_start":"281'2400","ondisk_log_start":"281'2400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8864,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.420717+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:34:14.420717+0000","last_peered":"2023-06-10T21:34:14.420717+0000","last_clean":"2023-06-10T21:34:14.420717+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:34:14.420717+0000","last_undegraded":"2023-06-10T21:34:14.420717+0000","last_fullsized":"2023-06-10T21:34:14.420717+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":174,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625105+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:34:14.625105+0000","last_peered":"2023-06-10T21:34:14.625105+0000","last_clean":"2023-06-10T21:34:14.625105+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:34:14.625105+0000","last_undegraded":"2023-06-10T21:34:14.625105+0000","last_fullsized":"2023-06-10T21:34:14.625105+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"302'9109","reported_seq":9155,"reported_epoch":302,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:43.083008+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:34:43.083008+0000","last_peered":"2023-06-10T21:34:43.083008+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:34:43.083008+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"281'2700","ondisk_log_start":"281'2700","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117768192,"num_objects":1797,"num_object_clones":0,"num_object_copies":5391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9109,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":1797},{"shards":"0,3","objects":-3597}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":308,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418922+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:34:14.418922+0000","last_peered":"2023-06-10T21:34:14.418922+0000","last_clean":"2023-06-10T21:34:14.418922+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:34:14.418922+0000","last_undegraded":"2023-06-10T21:34:14.418922+0000","last_fullsized":"2023-06-10T21:34:14.418922+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418533+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:34:14.418533+0000","last_peered":"2023-06-10T21:34:14.418533+0000","last_clean":"2023-06-10T21:34:14.418533+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:34:14.418533+0000","last_undegraded":"2023-06-10T21:34:14.418533+0000","last_fullsized":"2023-06-10T21:34:14.418533+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"301'8616","reported_seq":8656,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:39.561441+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:34:39.561441+0000","last_peered":"2023-06-10T21:34:39.561441+0000","last_clean":"2023-06-10T21:34:39.561441+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:34:39.561441+0000","last_undegraded":"2023-06-10T21:34:39.561441+0000","last_fullsized":"2023-06-10T21:34:39.561441+0000","mapping_epoch":269,"log_start":"278'1200","ondisk_log_start":"278'1200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7416,"log_dups_size":0,"ondisk_log_size":7416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122421248,"num_objects":1868,"num_object_clones":0,"num_object_copies":5604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8616,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439560+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:34:14.439560+0000","last_peered":"2023-06-10T21:34:14.439560+0000","last_clean":"2023-06-10T21:34:14.439560+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:34:14.439560+0000","last_undegraded":"2023-06-10T21:34:14.439560+0000","last_fullsized":"2023-06-10T21:34:14.439560+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867716+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:34:28.867716+0000","last_peered":"2023-06-10T21:34:28.867716+0000","last_clean":"2023-06-10T21:34:28.867716+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:34:28.867716+0000","last_undegraded":"2023-06-10T21:34:28.867716+0000","last_fullsized":"2023-06-10T21:34:28.867716+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"301'8575","reported_seq":14873,"reported_epoch":301,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:34:38.941625+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:34:38.941625+0000","last_peered":"2023-06-10T21:34:38.941625+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:34:38.941625+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8575,"log_dups_size":0,"ondisk_log_size":8575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118816768,"num_objects":1813,"num_object_clones":0,"num_object_copies":5439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":464,"num_objects_misplaced":1553,"num_objects_unfound":0,"num_objects_dirty":1813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8575,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3125,"num_bytes_recovered":204734464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":-204},{"shards":"1,2,3","objects":1553}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":252,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624912+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:34:14.624912+0000","last_peered":"2023-06-10T21:34:14.624912+0000","last_clean":"2023-06-10T21:34:14.624912+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:34:14.624912+0000","last_undegraded":"2023-06-10T21:34:14.624912+0000","last_fullsized":"2023-06-10T21:34:14.624912+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439636+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:34:14.439636+0000","last_peered":"2023-06-10T21:34:14.439636+0000","last_clean":"2023-06-10T21:34:14.439636+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:34:14.439636+0000","last_undegraded":"2023-06-10T21:34:14.439636+0000","last_fullsized":"2023-06-10T21:34:14.439636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"301'8816","reported_seq":8855,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:38.923031+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:34:38.923031+0000","last_peered":"2023-06-10T21:34:38.923031+0000","last_clean":"2023-06-10T21:34:38.923031+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:34:38.923031+0000","last_undegraded":"2023-06-10T21:34:38.923031+0000","last_fullsized":"2023-06-10T21:34:38.923031+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8816,"log_dups_size":0,"ondisk_log_size":8816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127008768,"num_objects":1938,"num_object_clones":0,"num_object_copies":5814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8816,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624747+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:34:14.624747+0000","last_peered":"2023-06-10T21:34:14.624747+0000","last_clean":"2023-06-10T21:34:14.624747+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:34:14.624747+0000","last_undegraded":"2023-06-10T21:34:14.624747+0000","last_fullsized":"2023-06-10T21:34:14.624747+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":292,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.422059+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:34:14.422059+0000","last_peered":"2023-06-10T21:34:14.422059+0000","last_clean":"2023-06-10T21:34:14.422059+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:34:14.422059+0000","last_undegraded":"2023-06-10T21:34:14.422059+0000","last_fullsized":"2023-06-10T21:34:14.422059+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"301'8754","reported_seq":8796,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.545231+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:34:41.545231+0000","last_peered":"2023-06-10T21:34:41.545231+0000","last_clean":"2023-06-10T21:34:41.545231+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:34:41.545231+0000","last_undegraded":"2023-06-10T21:34:41.545231+0000","last_fullsized":"2023-06-10T21:34:41.545231+0000","mapping_epoch":269,"log_start":"282'2900","ondisk_log_start":"282'2900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8754,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421883+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:34:14.421883+0000","last_peered":"2023-06-10T21:34:14.421883+0000","last_clean":"2023-06-10T21:34:14.421883+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:34:14.421883+0000","last_undegraded":"2023-06-10T21:34:14.421883+0000","last_fullsized":"2023-06-10T21:34:14.421883+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624990+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:34:14.624990+0000","last_peered":"2023-06-10T21:34:14.624990+0000","last_clean":"2023-06-10T21:34:14.624990+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:34:14.624990+0000","last_undegraded":"2023-06-10T21:34:14.624990+0000","last_fullsized":"2023-06-10T21:34:14.624990+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"301'8749","reported_seq":8791,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.333717+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:34:41.333717+0000","last_peered":"2023-06-10T21:34:41.333717+0000","last_clean":"2023-06-10T21:34:41.333717+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:34:41.333717+0000","last_undegraded":"2023-06-10T21:34:41.333717+0000","last_fullsized":"2023-06-10T21:34:41.333717+0000","mapping_epoch":269,"log_start":"282'2900","ondisk_log_start":"282'2900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117506048,"num_objects":1793,"num_object_clones":0,"num_object_copies":5379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8749,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":255,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439473+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:34:14.439473+0000","last_peered":"2023-06-10T21:34:14.439473+0000","last_clean":"2023-06-10T21:34:14.439473+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:34:14.439473+0000","last_undegraded":"2023-06-10T21:34:14.439473+0000","last_fullsized":"2023-06-10T21:34:14.439473+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":211,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624747+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:34:14.624747+0000","last_peered":"2023-06-10T21:34:14.624747+0000","last_clean":"2023-06-10T21:34:14.624747+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:34:14.624747+0000","last_undegraded":"2023-06-10T21:34:14.624747+0000","last_fullsized":"2023-06-10T21:34:14.624747+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"301'8672","reported_seq":8715,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:38.924155+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:34:38.924155+0000","last_peered":"2023-06-10T21:34:38.924155+0000","last_clean":"2023-06-10T21:34:38.924155+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:34:38.924155+0000","last_undegraded":"2023-06-10T21:34:38.924155+0000","last_fullsized":"2023-06-10T21:34:38.924155+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8672,"log_dups_size":0,"ondisk_log_size":8672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114753560,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8672,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868228+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:34:28.868228+0000","last_peered":"2023-06-10T21:34:28.868228+0000","last_clean":"2023-06-10T21:34:28.868228+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:34:28.868228+0000","last_undegraded":"2023-06-10T21:34:28.868228+0000","last_fullsized":"2023-06-10T21:34:28.868228+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.625127+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:34:14.625127+0000","last_peered":"2023-06-10T21:34:14.625127+0000","last_clean":"2023-06-10T21:34:14.625127+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:34:14.625127+0000","last_undegraded":"2023-06-10T21:34:14.625127+0000","last_fullsized":"2023-06-10T21:34:14.625127+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":145,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867844+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:34:28.867844+0000","last_peered":"2023-06-10T21:34:28.867844+0000","last_clean":"2023-06-10T21:34:28.867844+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:34:28.867844+0000","last_undegraded":"2023-06-10T21:34:28.867844+0000","last_fullsized":"2023-06-10T21:34:28.867844+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"301'8734","reported_seq":8776,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.515348+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:34:41.515348+0000","last_peered":"2023-06-10T21:34:41.515348+0000","last_clean":"2023-06-10T21:34:41.515348+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:34:41.515348+0000","last_undegraded":"2023-06-10T21:34:41.515348+0000","last_fullsized":"2023-06-10T21:34:41.515348+0000","mapping_epoch":269,"log_start":"282'2900","ondisk_log_start":"282'2900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118882304,"num_objects":1814,"num_object_clones":0,"num_object_copies":5442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8734,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":171,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867926+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:34:28.867926+0000","last_peered":"2023-06-10T21:34:28.867926+0000","last_clean":"2023-06-10T21:34:28.867926+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:34:28.867926+0000","last_undegraded":"2023-06-10T21:34:28.867926+0000","last_fullsized":"2023-06-10T21:34:28.867926+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":245,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867841+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:34:28.867841+0000","last_peered":"2023-06-10T21:34:28.867841+0000","last_clean":"2023-06-10T21:34:28.867841+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:34:28.867841+0000","last_undegraded":"2023-06-10T21:34:28.867841+0000","last_fullsized":"2023-06-10T21:34:28.867841+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":134,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425733+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:34:14.425733+0000","last_peered":"2023-06-10T21:34:14.425733+0000","last_clean":"2023-06-10T21:34:14.425733+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:34:14.425733+0000","last_undegraded":"2023-06-10T21:34:14.425733+0000","last_fullsized":"2023-06-10T21:34:14.425733+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"301'8758","reported_seq":8800,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.555590+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:34:41.555590+0000","last_peered":"2023-06-10T21:34:41.555590+0000","last_clean":"2023-06-10T21:34:41.555590+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:34:41.555590+0000","last_undegraded":"2023-06-10T21:34:41.555590+0000","last_fullsized":"2023-06-10T21:34:41.555590+0000","mapping_epoch":269,"log_start":"282'2900","ondisk_log_start":"282'2900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8758,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":145,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421634+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:34:14.421634+0000","last_peered":"2023-06-10T21:34:14.421634+0000","last_clean":"2023-06-10T21:34:14.421634+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:34:14.421634+0000","last_undegraded":"2023-06-10T21:34:14.421634+0000","last_fullsized":"2023-06-10T21:34:14.421634+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421741+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:34:14.421741+0000","last_peered":"2023-06-10T21:34:14.421741+0000","last_clean":"2023-06-10T21:34:14.421741+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:34:14.421741+0000","last_undegraded":"2023-06-10T21:34:14.421741+0000","last_fullsized":"2023-06-10T21:34:14.421741+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":194,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868536+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:34:28.868536+0000","last_peered":"2023-06-10T21:34:28.868536+0000","last_clean":"2023-06-10T21:34:28.868536+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:34:28.868536+0000","last_undegraded":"2023-06-10T21:34:28.868536+0000","last_fullsized":"2023-06-10T21:34:28.868536+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"301'8607","reported_seq":8648,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.412189+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:34:41.412189+0000","last_peered":"2023-06-10T21:34:41.412189+0000","last_clean":"2023-06-10T21:34:41.412189+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:34:41.412189+0000","last_undegraded":"2023-06-10T21:34:41.412189+0000","last_fullsized":"2023-06-10T21:34:41.412189+0000","mapping_epoch":269,"log_start":"282'2800","ondisk_log_start":"282'2800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5807,"log_dups_size":0,"ondisk_log_size":5807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115015680,"num_objects":1755,"num_object_clones":0,"num_object_copies":5265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8607,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":325,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.429876+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:34:14.429876+0000","last_peered":"2023-06-10T21:34:14.429876+0000","last_clean":"2023-06-10T21:34:14.429876+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:34:14.429876+0000","last_undegraded":"2023-06-10T21:34:14.429876+0000","last_fullsized":"2023-06-10T21:34:14.429876+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":464,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418308+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:34:14.418308+0000","last_peered":"2023-06-10T21:34:14.418308+0000","last_clean":"2023-06-10T21:34:14.418308+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:34:14.418308+0000","last_undegraded":"2023-06-10T21:34:14.418308+0000","last_fullsized":"2023-06-10T21:34:14.418308+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":238,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624777+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:34:14.624777+0000","last_peered":"2023-06-10T21:34:14.624777+0000","last_clean":"2023-06-10T21:34:14.624777+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:34:14.624777+0000","last_undegraded":"2023-06-10T21:34:14.624777+0000","last_fullsized":"2023-06-10T21:34:14.624777+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.419062+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:34:14.419062+0000","last_peered":"2023-06-10T21:34:14.419062+0000","last_clean":"2023-06-10T21:34:14.419062+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:34:14.419062+0000","last_undegraded":"2023-06-10T21:34:14.419062+0000","last_fullsized":"2023-06-10T21:34:14.419062+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"301'8748","reported_seq":8790,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:41.505499+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:34:41.505499+0000","last_peered":"2023-06-10T21:34:41.505499+0000","last_clean":"2023-06-10T21:34:41.505499+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:34:41.505499+0000","last_undegraded":"2023-06-10T21:34:41.505499+0000","last_fullsized":"2023-06-10T21:34:41.505499+0000","mapping_epoch":269,"log_start":"282'2900","ondisk_log_start":"282'2900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123994112,"num_objects":1892,"num_object_clones":0,"num_object_copies":5676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8748,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":320,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418330+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:34:14.418330+0000","last_peered":"2023-06-10T21:34:14.418330+0000","last_clean":"2023-06-10T21:34:14.418330+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:34:14.418330+0000","last_undegraded":"2023-06-10T21:34:14.418330+0000","last_fullsized":"2023-06-10T21:34:14.418330+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":140,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867799+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:34:28.867799+0000","last_peered":"2023-06-10T21:34:28.867799+0000","last_clean":"2023-06-10T21:34:28.867799+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:34:28.867799+0000","last_undegraded":"2023-06-10T21:34:28.867799+0000","last_fullsized":"2023-06-10T21:34:28.867799+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868330+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:34:28.868330+0000","last_peered":"2023-06-10T21:34:28.868330+0000","last_clean":"2023-06-10T21:34:28.868330+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:34:28.868330+0000","last_undegraded":"2023-06-10T21:34:28.868330+0000","last_fullsized":"2023-06-10T21:34:28.868330+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"301'8635","reported_seq":8677,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:38.859063+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:34:38.859063+0000","last_peered":"2023-06-10T21:34:38.859063+0000","last_clean":"2023-06-10T21:34:38.859063+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:34:38.859063+0000","last_undegraded":"2023-06-10T21:34:38.859063+0000","last_fullsized":"2023-06-10T21:34:38.859063+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8635,"log_dups_size":0,"ondisk_log_size":8635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124583936,"num_objects":1901,"num_object_clones":0,"num_object_copies":5703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8635,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":305,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418995+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:34:14.418995+0000","last_peered":"2023-06-10T21:34:14.418995+0000","last_clean":"2023-06-10T21:34:14.418995+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:34:14.418995+0000","last_undegraded":"2023-06-10T21:34:14.418995+0000","last_fullsized":"2023-06-10T21:34:14.418995+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425825+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:34:14.425825+0000","last_peered":"2023-06-10T21:34:14.425825+0000","last_clean":"2023-06-10T21:34:14.425825+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:34:14.425825+0000","last_undegraded":"2023-06-10T21:34:14.425825+0000","last_fullsized":"2023-06-10T21:34:14.425825+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":121,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868157+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:34:28.868157+0000","last_peered":"2023-06-10T21:34:28.868157+0000","last_clean":"2023-06-10T21:34:28.868157+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:34:28.868157+0000","last_undegraded":"2023-06-10T21:34:28.868157+0000","last_fullsized":"2023-06-10T21:34:28.868157+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"301'8604","reported_seq":8644,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:39.622116+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:34:39.622116+0000","last_peered":"2023-06-10T21:34:39.622116+0000","last_clean":"2023-06-10T21:34:39.622116+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:34:39.622116+0000","last_undegraded":"2023-06-10T21:34:39.622116+0000","last_fullsized":"2023-06-10T21:34:39.622116+0000","mapping_epoch":269,"log_start":"277'1200","ondisk_log_start":"277'1200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8604,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":278,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439642+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:34:14.439642+0000","last_peered":"2023-06-10T21:34:14.439642+0000","last_clean":"2023-06-10T21:34:14.439642+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:34:14.439642+0000","last_undegraded":"2023-06-10T21:34:14.439642+0000","last_fullsized":"2023-06-10T21:34:14.439642+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":75,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624894+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:34:14.624894+0000","last_peered":"2023-06-10T21:34:14.624894+0000","last_clean":"2023-06-10T21:34:14.624894+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:34:14.624894+0000","last_undegraded":"2023-06-10T21:34:14.624894+0000","last_fullsized":"2023-06-10T21:34:14.624894+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.439381+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:34:14.439381+0000","last_peered":"2023-06-10T21:34:14.439381+0000","last_clean":"2023-06-10T21:34:14.439381+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:34:14.439381+0000","last_undegraded":"2023-06-10T21:34:14.439381+0000","last_fullsized":"2023-06-10T21:34:14.439381+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"301'8631","reported_seq":8671,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:39.620350+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:34:39.620350+0000","last_peered":"2023-06-10T21:34:39.620350+0000","last_clean":"2023-06-10T21:34:39.620350+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:34:39.620350+0000","last_undegraded":"2023-06-10T21:34:39.620350+0000","last_fullsized":"2023-06-10T21:34:39.620350+0000","mapping_epoch":269,"log_start":"279'1300","ondisk_log_start":"279'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7331,"log_dups_size":0,"ondisk_log_size":7331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113049600,"num_objects":1725,"num_object_clones":0,"num_object_copies":5175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8631,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":22,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418609+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:34:14.418609+0000","last_peered":"2023-06-10T21:34:14.418609+0000","last_clean":"2023-06-10T21:34:14.418609+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:34:14.418609+0000","last_undegraded":"2023-06-10T21:34:14.418609+0000","last_fullsized":"2023-06-10T21:34:14.418609+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":129,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868474+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:34:28.868474+0000","last_peered":"2023-06-10T21:34:28.868474+0000","last_clean":"2023-06-10T21:34:28.868474+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:34:28.868474+0000","last_undegraded":"2023-06-10T21:34:28.868474+0000","last_fullsized":"2023-06-10T21:34:28.868474+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":257,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.867657+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:34:28.867657+0000","last_peered":"2023-06-10T21:34:28.867657+0000","last_clean":"2023-06-10T21:34:28.867657+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:34:28.867657+0000","last_undegraded":"2023-06-10T21:34:28.867657+0000","last_fullsized":"2023-06-10T21:34:28.867657+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868187+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:34:28.868187+0000","last_peered":"2023-06-10T21:34:28.868187+0000","last_clean":"2023-06-10T21:34:28.868187+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:34:28.868187+0000","last_undegraded":"2023-06-10T21:34:28.868187+0000","last_fullsized":"2023-06-10T21:34:28.868187+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":142,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.425938+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:34:14.425938+0000","last_peered":"2023-06-10T21:34:14.425938+0000","last_clean":"2023-06-10T21:34:14.425938+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:34:14.425938+0000","last_undegraded":"2023-06-10T21:34:14.425938+0000","last_fullsized":"2023-06-10T21:34:14.425938+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":26,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.418423+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:34:14.418423+0000","last_peered":"2023-06-10T21:34:14.418423+0000","last_clean":"2023-06-10T21:34:14.418423+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:34:14.418423+0000","last_undegraded":"2023-06-10T21:34:14.418423+0000","last_fullsized":"2023-06-10T21:34:14.418423+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624750+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:34:14.624750+0000","last_peered":"2023-06-10T21:34:14.624750+0000","last_clean":"2023-06-10T21:34:14.624750+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:34:14.624750+0000","last_undegraded":"2023-06-10T21:34:14.624750+0000","last_fullsized":"2023-06-10T21:34:14.624750+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":79,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624851+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:34:14.624851+0000","last_peered":"2023-06-10T21:34:14.624851+0000","last_clean":"2023-06-10T21:34:14.624851+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:34:14.624851+0000","last_undegraded":"2023-06-10T21:34:14.624851+0000","last_fullsized":"2023-06-10T21:34:14.624851+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624667+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:34:14.624667+0000","last_peered":"2023-06-10T21:34:14.624667+0000","last_clean":"2023-06-10T21:34:14.624667+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:34:14.624667+0000","last_undegraded":"2023-06-10T21:34:14.624667+0000","last_fullsized":"2023-06-10T21:34:14.624667+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":284,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624636+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:34:14.624636+0000","last_peered":"2023-06-10T21:34:14.624636+0000","last_clean":"2023-06-10T21:34:14.624636+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:34:14.624636+0000","last_undegraded":"2023-06-10T21:34:14.624636+0000","last_fullsized":"2023-06-10T21:34:14.624636+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.624719+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:34:14.624719+0000","last_peered":"2023-06-10T21:34:14.624719+0000","last_clean":"2023-06-10T21:34:14.624719+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:34:14.624719+0000","last_undegraded":"2023-06-10T21:34:14.624719+0000","last_fullsized":"2023-06-10T21:34:14.624719+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":190,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-06-10T21:34:28.868590+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:34:28.868590+0000","last_peered":"2023-06-10T21:34:28.868590+0000","last_clean":"2023-06-10T21:34:28.868590+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:34:28.868590+0000","last_undegraded":"2023-06-10T21:34:28.868590+0000","last_fullsized":"2023-06-10T21:34:28.868590+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":34,"reported_epoch":300,"state":"active+clean","last_fresh":"2023-06-10T21:34:14.421392+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:34:14.421392+0000","last_peered":"2023-06-10T21:34:14.421392+0000","last_clean":"2023-06-10T21:34:14.421392+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:34:14.421392+0000","last_undegraded":"2023-06-10T21:34:14.421392+0000","last_fullsized":"2023-06-10T21:34:14.421392+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1891500056,"num_objects":28863,"num_object_clones":0,"num_object_copies":86589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5917,"num_objects_misplaced":1553,"num_objects_unfound":0,"num_objects_dirty":28863,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139679,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3125,"num_bytes_recovered":204734464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6135164928,"data_stored":6135152712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111179,"ondisk_log_size":111179,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430651,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1071000,"kb_used_data":486032,"kb_used_omap":0,"kb_used_meta":584960,"kb_avail":92674152,"statfs":{"total":95995035648,"available":94898331648,"internally_reserved":0,"allocated":497696768,"data_stored":497132468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":598999040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.194,"5min":5.6710000000000003,"15min":8.3620000000000001},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":52.100000000000001,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.677,"5min":5.641,"15min":8.5800000000000001},"min":{"1min":0.31900000000000001,"5min":0.223,"15min":0.223},"max":{"1min":51.146999999999998,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.28599999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:34:13 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":3.3130000000000002,"15min":3.2519999999999998},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.21299999999999999},"max":{"1min":59.268000000000001,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":4.4749999999999996,"5min":3.4769999999999999,"15min":3.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":59.256999999999998,"5min":87.438000000000002,"15min":212.20599999999999},"last":3.9580000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:34:26 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":6.4400000000000004,"15min":4.1849999999999996},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":89.055999999999997,"5min":172.33600000000001,"15min":172.33600000000001},"last":3.0699999999999998},{"interface":"front","average":{"1min":7.0510000000000002,"5min":6.3639999999999999,"15min":4.8490000000000002},"min":{"1min":0.38300000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":88.787000000000006,"5min":172.185,"15min":212.13399999999999},"last":0.38900000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":8.5359999999999996,"5min":4.0730000000000004,"15min":3.024},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":70.841999999999999,"5min":99.266000000000005,"15min":99.266000000000005},"last":2.5920000000000001},{"interface":"front","average":{"1min":7.9180000000000001,"5min":3.714,"15min":2.46},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":69.796999999999997,"5min":99.138000000000005,"15min":99.138000000000005},"last":3.8460000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.45700000000000002,"15min":0.72199999999999998},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.5049999999999999,"5min":3.9199999999999999,"15min":8.1419999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":1.0369999999999999},"min":{"1min":0.29599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":3.0310000000000001,"5min":9.0470000000000006,"15min":14.657999999999999},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.54500000000000004,"15min":1.827},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":3.1829999999999998,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.47399999999999998,"15min":1.6220000000000001},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.92100000000000004,"5min":1.7,"15min":212.23699999999999},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.57199999999999995,"15min":1.083},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":3.2559999999999998,"5min":3.2559999999999998,"15min":13.073},"last":0.495},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.94399999999999995},"min":{"1min":0.36099999999999999,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":1.1879999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.60899999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626879,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566322,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2339260,"kb_used_data":1718388,"kb_used_omap":4,"kb_used_meta":620859,"kb_avail":91405892,"statfs":{"total":95995035648,"available":93599633408,"internally_reserved":0,"allocated":1759629312,"data_stored":1758680143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":635760526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:45 2023","interfaces":[{"interface":"back","average":{"1min":6.3789999999999996,"5min":7.5979999999999999,"15min":11.686},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":70.254000000000005,"5min":154.31399999999999,"15min":154.31399999999999},"last":2.9969999999999999},{"interface":"front","average":{"1min":5.7859999999999996,"5min":7.4790000000000001,"15min":11.372999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":70.100999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":3.0350000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":8.8239999999999998,"15min":4.9909999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":18.041,"5min":191.101,"15min":191.101},"last":4.8390000000000004},{"interface":"front","average":{"1min":3,"5min":9.1199999999999992,"15min":4.7560000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.218},"max":{"1min":11.964,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.495}]},{"osd":3,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.2290000000000001,"5min":4.3470000000000004,"15min":2.766},"min":{"1min":0.32300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.694000000000003,"5min":80.584999999999994,"15min":80.584999999999994},"last":4.7599999999999998},{"interface":"front","average":{"1min":5.3959999999999999,"5min":5.266,"15min":3.8759999999999999},"min":{"1min":0.40400000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":42.734999999999999,"5min":80.873999999999995,"15min":80.873999999999995},"last":10.961}]},{"osd":4,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.4430000000000001,"15min":1.4810000000000001},"min":{"1min":0.28699999999999998,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.032,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.024,"5min":1.504,"15min":1.3540000000000001},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":4.8200000000000003,"5min":8.2739999999999991,"15min":14.460000000000001},"last":3.0720000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.0339999999999998,"15min":4.1349999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7089999999999996,"5min":8.9009999999999998,"15min":383.83100000000002},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.46,"15min":3.7400000000000002},"min":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.0679999999999996,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.129,"15min":2.0310000000000001},"min":{"1min":0.34799999999999998,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":8.7550000000000008,"5min":8.8580000000000005,"15min":14.858000000000001},"last":1.115},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5129999999999999,"15min":1.643},"min":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.1360000000000001,"5min":11.476000000000001,"15min":13.48},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3970000000000002,"5min":2.1709999999999998,"15min":2.5979999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32600000000000001},"max":{"1min":47.844000000000001,"5min":47.844000000000001,"15min":162.399},"last":0.56899999999999995},{"interface":"front","average":{"1min":4.4699999999999998,"5min":2.407,"15min":2.3759999999999999},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":47.753999999999998,"5min":47.753999999999998,"15min":162.27199999999999},"last":3.1000000000000001}]}]},{"osd":0,"up_from":244,"seq":1047972020299,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166932,"kb_used_data":923148,"kb_used_omap":10,"kb_used_meta":243701,"kb_avail":92578220,"statfs":{"total":95995035648,"available":94800097280,"internally_reserved":0,"allocated":945303552,"data_stored":944278624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":249549876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.3239999999999998,"5min":5.056,"15min":4.625},"min":{"1min":0.215,"5min":0.191,"15min":0.191},"max":{"1min":8.6750000000000007,"5min":139.68000000000001,"15min":139.68000000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.2909999999999999,"5min":4.9800000000000004,"15min":5.7009999999999996},"min":{"1min":0.246,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":11.409000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.22800000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":15.375,"5min":7.8479999999999999,"15min":6.5839999999999996},"min":{"1min":0.28499999999999998,"5min":0.216,"15min":0.216},"max":{"1min":132.245,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":15.032,"5min":7.9210000000000003,"15min":6.0439999999999996},"min":{"1min":0.46600000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":132.26599999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.871,"5min":4.5389999999999997,"15min":5.7039999999999997},"min":{"1min":0.23999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":26.167999999999999,"5min":111.745,"15min":111.745},"last":0.33800000000000002},{"interface":"front","average":{"1min":4.7510000000000003,"5min":5.5579999999999998,"15min":6.7080000000000002},"min":{"1min":0.30399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":26.201000000000001,"5min":114.205,"15min":114.205},"last":0.37}]},{"osd":4,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":2.617,"15min":4.2699999999999996},"min":{"1min":0.32400000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":10.744999999999999,"5min":29.709,"15min":29.709},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.2669999999999999,"15min":3.605},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.9630000000000001,"5min":13.872,"15min":13.872},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":2.302,"15min":3.6909999999999998},"min":{"1min":0.34399999999999997,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0270000000000001,"5min":14.105,"15min":14.105},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.294,"5min":2.4079999999999999,"15min":3.2120000000000002},"min":{"1min":0.441,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":7.532,"5min":19.846,"15min":19.846},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":2.887,"5min":2.7400000000000002,"15min":4.4580000000000002},"min":{"1min":0.39600000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.771000000000001,"5min":29.806000000000001,"15min":29.806000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.175,"5min":2.3300000000000001,"15min":3.7250000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":6.0650000000000004,"5min":13.82,"15min":13.82},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:33:49 2023","interfaces":[{"interface":"back","average":{"1min":3.21,"5min":2.6949999999999998,"15min":3.6230000000000002},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":17.523,"5min":19.866,"15min":19.866},"last":0.499},{"interface":"front","average":{"1min":3.6720000000000002,"5min":2.899,"15min":4.5090000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":17.792999999999999,"5min":29.77,"15min":29.77},"last":0.42699999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075710,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1671284,"kb_used_data":1185324,"kb_used_omap":7,"kb_used_meta":485688,"kb_avail":92073868,"statfs":{"total":95995035648,"available":94283640832,"internally_reserved":0,"allocated":1213771776,"data_stored":1212830287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":497344593},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.6389999999999998,"5min":4.0060000000000002,"15min":4.4710000000000001},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":18.210999999999999,"5min":66.784000000000006,"15min":66.784000000000006},"last":1.8009999999999999},{"interface":"front","average":{"1min":2.7639999999999998,"5min":4.3840000000000003,"15min":5.056},"min":{"1min":0.317,"5min":0.252,"15min":0.252},"max":{"1min":7.6109999999999998,"5min":67.251999999999995,"15min":67.251999999999995},"last":1.821}]},{"osd":1,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":5.5350000000000001,"15min":3.1160000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.19},"max":{"1min":12.734,"5min":147.20099999999999,"15min":147.20099999999999},"last":3.7589999999999999},{"interface":"front","average":{"1min":5.4580000000000002,"5min":4.8010000000000002,"15min":2.742},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.214},"max":{"1min":16.503,"5min":147.81299999999999,"15min":147.81299999999999},"last":7.0449999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:34:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6629999999999998,"5min":4.5810000000000004,"15min":5.0209999999999999},"min":{"1min":0.30399999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":29.719999999999999,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.443},{"interface":"front","average":{"1min":3.0630000000000002,"5min":4.798,"15min":5.7949999999999999},"min":{"1min":0.40999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.406000000000001,"5min":138.095,"15min":138.095},"last":0.29899999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.744,"15min":2.8359999999999999},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":4.5599999999999996,"5min":8.0969999999999995,"15min":224.00999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.8620000000000001,"15min":3.2480000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":8.2739999999999991,"5min":29.486999999999998,"15min":223.28399999999999},"last":3.7090000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":1.819,"15min":1.5960000000000001},"min":{"1min":0.41199999999999998,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":10.554,"5min":10.554,"15min":14.180999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.548,"5min":1.7849999999999999,"15min":1.7529999999999999},"min":{"1min":0.38200000000000001,"5min":0.371,"15min":0.32900000000000001},"max":{"1min":4.5860000000000003,"5min":8.1989999999999998,"15min":16.125},"last":0.372}]},{"osd":6,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.8560000000000001,"15min":1.825},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":4.6130000000000004,"5min":8.1690000000000005,"15min":16.183},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.2559999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.3870000000000005,"5min":29.538,"15min":29.538},"last":3.7930000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":5.5590000000000002,"5min":3.544,"15min":2.5529999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":41.164999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.624},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4860000000000002,"15min":2.7999999999999998},"min":{"1min":0.43099999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":41.219999999999999,"5min":73.475999999999999,"15min":113.937},"last":0.39300000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824070,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1508928,"kb_used_data":1275064,"kb_used_omap":7,"kb_used_meta":233848,"kb_avail":92236224,"statfs":{"total":95995035648,"available":94449893376,"internally_reserved":0,"allocated":1305665536,"data_stored":1304652408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":239460371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,9],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":4.2629999999999999,"15min":5.1200000000000001},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":53.756999999999998,"5min":93.992999999999995,"15min":93.992999999999995},"last":1.8660000000000001},{"interface":"front","average":{"1min":5.5069999999999997,"5min":3.9260000000000002,"15min":4.8639999999999999},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.216},"max":{"1min":48.247,"5min":95.623999999999995,"15min":95.623999999999995},"last":2.0190000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.3230000000000004,"5min":4.1130000000000004,"15min":3.7970000000000002},"min":{"1min":0.30399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":20.574999999999999,"5min":20.574999999999999,"15min":20.574999999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":4.6429999999999998,"5min":3.2599999999999998,"15min":3.4100000000000001},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.545,"5min":17.384,"15min":17.384},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2240000000000002,"5min":4.6740000000000004,"15min":10.172000000000001},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.529999999999999,"5min":87.555999999999997,"15min":87.555999999999997},"last":0.33400000000000002},{"interface":"front","average":{"1min":5.1399999999999997,"5min":6.5449999999999999,"15min":11.743},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.541,"5min":80.566999999999993,"15min":80.566999999999993},"last":0.315}]},{"osd":4,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":1.621,"15min":1.456},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.417,"5min":13.417,"15min":13.417},"last":0.627},{"interface":"front","average":{"1min":3.133,"5min":2.835,"15min":2.7709999999999999},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.311},"max":{"1min":9.2300000000000004,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":1.383,"15min":1.389},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.8810000000000002,"5min":6.891,"15min":6.891},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.931,"5min":1.762,"15min":1.54},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":2.8580000000000001,"15min":2.7949999999999999},"min":{"1min":0.505,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":9.2579999999999991,"5min":19.805,"15min":19.805},"last":0.67600000000000005},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.4370000000000001,"15min":1.4390000000000001},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9039999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.54,"15min":3.032},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":79.984999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":5.7229999999999999,"5min":2.1070000000000002,"15min":1.645},"min":{"1min":0.48199999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":80.272999999999996,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.63900000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019335,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1070356,"kb_used_data":467596,"kb_used_omap":0,"kb_used_meta":602752,"kb_avail":92674796,"statfs":{"total":95995035648,"available":94898991104,"internally_reserved":0,"allocated":478818304,"data_stored":478256572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":617218048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:33:37 2023","interfaces":[{"interface":"back","average":{"1min":10.459,"5min":5.0209999999999999,"15min":8.7919999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":234.203,"5min":234.203,"15min":234.203},"last":0.49099999999999999},{"interface":"front","average":{"1min":11.193,"5min":5.8780000000000001,"15min":8.5860000000000003},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":234.19,"5min":234.19,"15min":234.19},"last":0.628}]},{"osd":1,"last update":"Sat Jun 10 21:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.782,"15min":2.9159999999999999},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.7170000000000001,"5min":13.461,"15min":182.035},"last":0.438},{"interface":"front","average":{"1min":3.5049999999999999,"5min":2.1360000000000001,"15min":2.8359999999999999},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.151,"5min":10.255000000000001,"15min":181.73099999999999},"last":0.80900000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":22.809999999999999,"5min":6.6989999999999998,"15min":3.294},"min":{"1min":0.495,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":255.386,"5min":255.386,"15min":255.386},"last":5.165},{"interface":"front","average":{"1min":24.347999999999999,"5min":6.8520000000000003,"15min":3.2949999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":255.29599999999999,"5min":255.29599999999999,"15min":255.29599999999999},"last":4.4080000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":14.234,"5min":4.4939999999999998,"15min":3.0920000000000001},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":234.91900000000001,"5min":234.91900000000001,"15min":234.91900000000001},"last":7.8979999999999997},{"interface":"front","average":{"1min":15.222,"5min":6.1619999999999999,"15min":3.4159999999999999},"min":{"1min":0.38200000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":235.05099999999999,"5min":235.05099999999999,"15min":235.05099999999999},"last":2.0449999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":1.0009999999999999},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.58699999999999997,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.46600000000000003,"15min":1.016},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.67900000000000005,"5min":5.25,"15min":11.904999999999999},"last":0.34899999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:33:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50800000000000001,"15min":1.0309999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.25600000000000001},"max":{"1min":1.7210000000000001,"5min":3.077,"15min":8.2599999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.63500000000000001,"15min":1.1000000000000001},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":1.71,"5min":6.8300000000000001,"15min":13.845000000000001},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":0.80500000000000005,"15min":1.2889999999999999},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":39.713999999999999,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.2050000000000001,"5min":0.82699999999999996,"15min":1.2310000000000001},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":37.429000000000002,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.42799999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823107,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":925171712,"data_stored":925171712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1749684224,"data_stored":1749680152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1209012224,"data_stored":1209008152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1287520256,"data_stored":1287520256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":471863296,"data_stored":471859224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":491913216,"data_stored":491913216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:34:47.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:34:47.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:34:48.085 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:48.088 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:47 smithi064 ceph-mon[180297]: pgmap v1105: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 5917/80502 objects degraded (7.350%); 1316/80502 objects misplaced (1.635%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:48.092 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:47 smithi064 ceph-mon[186706]: pgmap v1105: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 5917/80502 objects degraded (7.350%); 1316/80502 objects misplaced (1.635%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:48.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:48.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:48.222 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:48.267 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:48.273 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:47 smithi152 ceph-mon[156819]: pgmap v1105: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.6 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 493 op/s; 5917/80502 objects degraded (7.350%); 1316/80502 objects misplaced (1.635%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:34:48.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:48.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:48.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:48.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:48.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:48.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:48.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:49.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:49.102 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:49.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:49.187 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:49.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:49.288 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:34:49.430 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:34:49.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:49.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:49.496 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:34:49.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:49.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:34:49.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:34:49.707 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:49.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:34:49.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:49.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:34:50.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:50.171 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:50.244 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:34:50.295 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:34:50.328 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:34:50.403 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:34:50.445 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:34:50.498 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:34:50.537 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:34:50.599 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:34:50.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.602 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.603 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:34:50.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[186706]: pgmap v1106: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/75756 objects degraded (7.811%); 1055/75756 objects misplaced (1.393%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:50.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1973176921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[180297]: pgmap v1106: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/75756 objects degraded (7.811%); 1055/75756 objects misplaced (1.393%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:50.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1973176921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:50.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:50 smithi152 ceph-mon[156819]: pgmap v1106: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 496 op/s; 5917/75756 objects degraded (7.811%); 1055/75756 objects misplaced (1.393%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:50.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1973176921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:34:52.050 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/75756 objects degraded (7.811%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:52.051 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/75756 objects degraded (7.811%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:52.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/75756 objects degraded (7.811%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:52.480 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:52.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:52.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:52.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:52.737 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[180297]: pgmap v1107: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5917/72978 objects degraded (8.108%); 1055/72978 objects misplaced (1.446%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:52.737 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:52.737 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:52.983 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:52.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:52.998 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:34:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:34:52] "GET /metrics HTTP/1.1" 200 224076 "" "Prometheus/2.33.4" 2023-06-10T21:34:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[186706]: pgmap v1107: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5917/72978 objects degraded (8.108%); 1055/72978 objects misplaced (1.446%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:52.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:52 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:53.098 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:53.127 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:52 smithi152 ceph-mon[156819]: pgmap v1107: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 5917/72978 objects degraded (8.108%); 1055/72978 objects misplaced (1.446%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:34:53.128 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:53.128 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:52 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:34:53.131 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:53.181 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:53.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:53.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:53.319 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:53.371 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:53.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:53.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:53.505 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:53.552 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:53.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:53.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:53.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:53.695 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:53.743 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:53.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:53.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:53.878 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:53.925 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:53.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:54.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:54.056 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:54.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:54.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:53 smithi064 ceph-mon[180297]: pgmap v1108: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/71670 objects degraded (8.256%); 1055/71670 objects misplaced (1.472%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:54.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:53 smithi064 ceph-mon[186706]: pgmap v1108: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/71670 objects degraded (8.256%); 1055/71670 objects misplaced (1.472%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:54.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:53 smithi152 ceph-mon[156819]: pgmap v1108: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.5 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/71670 objects degraded (8.256%); 1055/71670 objects misplaced (1.472%); 2.6 MiB/s, 41 objects/s recovering 2023-06-10T21:34:54.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:34:54.231 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:54.282 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:34:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:54.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:34:54.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:54.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:54.727 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:54.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:54.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:54.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:34:54.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:55.190 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:34:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:55.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:34:55.306 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:34:55.334 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:55.377 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:34:55.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:55.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:34:55.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:55.487 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:34:55.487 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:34:55.627 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:55.773 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:55.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:55.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:55.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:56.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:55 smithi064 ceph-mon[180297]: pgmap v1109: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 5917/68541 objects degraded (8.633%); 790/68541 objects misplaced (1.153%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:34:56.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/71670 objects degraded (8.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:56.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:55 smithi064 ceph-mon[186706]: pgmap v1109: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 5917/68541 objects degraded (8.633%); 790/68541 objects misplaced (1.153%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:34:56.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/71670 objects degraded (8.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:56.197 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:34:56.197 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 5917/68541 objects degraded (8.633%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1569,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":302,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":87},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":22847,"data_bytes":1496258470,"bytes_used":7941742592,"bytes_avail":568028471296,"bytes_total":575970213888,"degraded_objects":5917,"degraded_total":68541,"degraded_ratio":0.086327891335113288,"misplaced_objects":790,"misplaced_total":68541,"misplaced_ratio":0.011525947972746239,"recovering_objects_per_sec":63,"recovering_bytes_per_sec":4164122,"recovering_keys_per_sec":0,"num_objects_recovered":763,"num_bytes_recovered":50003968,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":502},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":24,"modified":"2023-06-10T21:29:57.688996+0000","services":{}},"progress_events":{"08ad03ac-ce84-41c1-8854-b18550dc6087":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:34:56.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:34:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:55 smithi152 ceph-mon[156819]: pgmap v1109: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 502 op/s; 5917/68541 objects degraded (8.633%); 790/68541 objects misplaced (1.153%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:34:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/71670 objects degraded (8.256%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:34:56.954 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:56.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:56.974 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:34:57.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:34:57.100 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:57.142 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:34:57.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:57.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:34:57.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:57.336 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2539073407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1444040344' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:57.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:57.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:57.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[180297]: osdmap e303: 8 total, 8 up, 6 in 2023-06-10T21:34:57.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2539073407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1444040344' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:57.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:57.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:57.339 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:56 smithi064 ceph-mon[186706]: osdmap e303: 8 total, 8 up, 6 in 2023-06-10T21:34:57.340 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2539073407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:34:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1444040344' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:34:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:34:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:34:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:56 smithi152 ceph-mon[156819]: osdmap e303: 8 total, 8 up, 6 in 2023-06-10T21:34:57.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:34:57.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:57.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:57.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:34:57.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[180297]: failed to encode map e303 with expected crc 2023-06-10T21:34:57.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[180297]: failed to encode map e303 with expected crc 2023-06-10T21:34:57.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[180297]: failed to encode map e303 with expected crc 2023-06-10T21:34:57.983 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[180297]: pgmap v1111: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 5917/65877 objects degraded (8.982%); 790/65877 objects misplaced (1.199%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:34:58.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[186706]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[186706]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[186706]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:57 smithi064 ceph-mon[186706]: pgmap v1111: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 5917/65877 objects degraded (8.982%); 790/65877 objects misplaced (1.199%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:34:58.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:57 smithi152 ceph-mon[156819]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:57 smithi152 ceph-mon[156819]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:57 smithi152 ceph-mon[156819]: failed to encode map e303 with expected crc 2023-06-10T21:34:58.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:57 smithi152 ceph-mon[156819]: pgmap v1111: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.3 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 487 op/s; 5917/65877 objects degraded (8.982%); 790/65877 objects misplaced (1.199%); 3.3 MiB/s, 52 objects/s recovering 2023-06-10T21:34:58.825 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:34:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:58.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:34:58.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:59.013 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:59.169 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:34:59.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:34:59.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:59.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:59.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:59.470 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:34:59.520 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:59.577 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:34:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:34:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:34:59.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:34:59.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:34:59.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:00.139 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:34:59 smithi064 ceph-mon[180297]: pgmap v1112: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 5917/61419 objects degraded (9.634%); 488/61419 objects misplaced (0.795%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:00.143 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:34:59 smithi064 ceph-mon[186706]: pgmap v1112: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 5917/61419 objects degraded (9.634%); 488/61419 objects misplaced (0.795%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:00.149 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:00.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:00.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:00.377 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:34:59 smithi152 ceph-mon[156819]: pgmap v1112: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 477 op/s; 5917/61419 objects degraded (9.634%); 488/61419 objects misplaced (0.795%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:01.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:00 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/61419 objects degraded (9.634%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:01.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:00 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/61419 objects degraded (9.634%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:01.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:35:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:00 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/61419 objects degraded (9.634%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:02.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:01 smithi064 ceph-mon[180297]: pgmap v1113: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/58476 objects degraded (10.119%); 488/58476 objects misplaced (0.835%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:02.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:01 smithi064 ceph-mon[186706]: pgmap v1113: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/58476 objects degraded (10.119%); 488/58476 objects misplaced (0.835%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:02.434 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:02.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:02.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:02.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:01 smithi152 ceph-mon[156819]: pgmap v1113: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 5917/58476 objects degraded (10.119%); 488/58476 objects misplaced (0.835%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:02.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:02.619 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:02.771 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:02.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:02.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:02.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:03.006 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:02] "GET /metrics HTTP/1.1" 200 224076 "" "Prometheus/2.33.4" 2023-06-10T21:35:03.021 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:03.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:03.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:03.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:03.485 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:03.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:03.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:35:03.720 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:35:03.751 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:03.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:03.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:03.795 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:35:03.850 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:35:03.864 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:03.891 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:03.899 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:35:03.951 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:35:03.951 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:35:03.951 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:35:03.951 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:35:03.952 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:35:03.954 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:03.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:03.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:03.964 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:35:03.980 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:35:04.010 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:35:04.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.059 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:35:04.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:04.089 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:04.134 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:04.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:04.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:04.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:04.254 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/972383467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[180297]: pgmap v1114: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 5917/56928 objects degraded (10.394%); 488/56928 objects misplaced (0.857%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/972383467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:04.255 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:04 smithi064 ceph-mon[186706]: pgmap v1114: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 5917/56928 objects degraded (10.394%); 488/56928 objects misplaced (0.857%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:04.346 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:04.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/972383467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:04.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:04 smithi152 ceph-mon[156819]: pgmap v1114: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 1.2 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 5917/56928 objects degraded (10.394%); 488/56928 objects misplaced (0.857%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:35:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:05 smithi064 ceph-mon[180297]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 5917/53541 objects degraded (11.051%); 162/53541 objects misplaced (0.303%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:35:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:05 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5917/53541 objects degraded (11.051%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:06.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:05 smithi064 ceph-mon[186706]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 5917/53541 objects degraded (11.051%); 162/53541 objects misplaced (0.303%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:35:06.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:05 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5917/53541 objects degraded (11.051%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:05 smithi152 ceph-mon[156819]: pgmap v1115: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 5917/53541 objects degraded (11.051%); 162/53541 objects misplaced (0.303%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:35:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:05 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5917/53541 objects degraded (11.051%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:06.461 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:06.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:06.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:06.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:06.662 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:07.895 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:07.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:08.004 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:08.087 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:08.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:07 smithi064 ceph-mon[180297]: pgmap v1116: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/50793 objects degraded (11.649%); 162/50793 objects misplaced (0.319%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:35:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:07 smithi064 ceph-mon[186706]: pgmap v1116: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/50793 objects degraded (11.649%); 162/50793 objects misplaced (0.319%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:35:08.196 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:35:08.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:07 smithi152 ceph-mon[156819]: pgmap v1116: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 5917/50793 objects degraded (11.649%); 162/50793 objects misplaced (0.319%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:35:08.466 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:08.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:08.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:08.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:08.656 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:08.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:08.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:08.761 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:08.905 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:09.551 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:09.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:09.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:09.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:09.847 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:09.889 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:09.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:09.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:35:10.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:10.031 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:35:10.039 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:10.088 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:10.091 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:35:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:10.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:10.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:10.221 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:10.266 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:10.269 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:09 smithi152 ceph-mon[156819]: pgmap v1117: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 961 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 5654/46179 objects degraded (12.244%); 4.1 MiB/s, 91 objects/s recovering 2023-06-10T21:35:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:10.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:10.369 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:09 smithi064 ceph-mon[180297]: pgmap v1117: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 961 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 5654/46179 objects degraded (12.244%); 4.1 MiB/s, 91 objects/s recovering 2023-06-10T21:35:10.369 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:09 smithi064 ceph-mon[186706]: pgmap v1117: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 961 MiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 5654/46179 objects degraded (12.244%); 4.1 MiB/s, 91 objects/s recovering 2023-06-10T21:35:10.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:10.472 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:11.026 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:35:11.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[180297]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:11.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3393437696' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5654/46179 objects degraded (12.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:11.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[186706]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:11.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3393437696' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:10 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5654/46179 objects degraded (12.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:11.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:10 smithi152 ceph-mon[156819]: from='client.46454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3393437696' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:11.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:10 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5654/46179 objects degraded (12.244%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:12.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:12.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:12.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: osdmap e304: 8 total, 8 up, 6 in 2023-06-10T21:35:12.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.391 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[180297]: pgmap v1119: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 903 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5654/43404 objects degraded (13.026%); 3.0 MiB/s, 80 objects/s recovering 2023-06-10T21:35:12.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:12.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:12.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: osdmap e304: 8 total, 8 up, 6 in 2023-06-10T21:35:12.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.393 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:11 smithi064 ceph-mon[186706]: pgmap v1119: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 903 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5654/43404 objects degraded (13.026%); 3.0 MiB/s, 80 objects/s recovering 2023-06-10T21:35:12.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:12.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: osdmap e304: 8 total, 8 up, 6 in 2023-06-10T21:35:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: failed to encode map e304 with expected crc 2023-06-10T21:35:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:11 smithi152 ceph-mon[156819]: pgmap v1119: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 903 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 5654/43404 objects degraded (13.026%); 3.0 MiB/s, 80 objects/s recovering 2023-06-10T21:35:13.067 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:13.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:13.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:13.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: osdmap e305: 8 total, 8 up, 6 in 2023-06-10T21:35:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.161 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:12] "GET /metrics HTTP/1.1" 200 224062 "" "Prometheus/2.33.4" 2023-06-10T21:35:13.166 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: osdmap e305: 8 total, 8 up, 6 in 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.167 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:13.294 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: osdmap e305: 8 total, 8 up, 6 in 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: failed to encode map e305 with expected crc 2023-06-10T21:35:13.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:13.488 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1117,"stamp":"2023-06-10T21:35:09.891609+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1007753126,"num_objects":15393,"num_object_clones":0,"num_object_copies":46179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5654,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15393,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":153434,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5034,"num_bytes_recovered":308289234,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112068,"ondisk_log_size":112068,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":270,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6461524,"kb_used_data":3664612,"kb_used_omap":30,"kb_used_meta":2796577,"kb_avail":556009388,"statfs":{"total":575970213888,"available":569353613312,"internally_reserved":0,"allocated":3752562688,"data_stored":3747493587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2863694886},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,11,11,1,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-430309376,"num_objects":-6566,"num_object_clones":0,"num_object_copies":-19698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-263,"num_objects_misplaced":-790,"num_objects_unfound":0,"num_objects_dirty":-6566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1105,"num_bytes_recovered":51773440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.019554"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":296,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.004015+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:34:42.004015+0000","last_peered":"2023-06-10T21:34:42.004015+0000","last_clean":"2023-06-10T21:34:42.004015+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:34:42.004015+0000","last_undegraded":"2023-06-10T21:34:42.004015+0000","last_fullsized":"2023-06-10T21:34:42.004015+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":237,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193803+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:34:42.193803+0000","last_peered":"2023-06-10T21:34:42.193803+0000","last_clean":"2023-06-10T21:34:42.193803+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:34:42.193803+0000","last_undegraded":"2023-06-10T21:34:42.193803+0000","last_fullsized":"2023-06-10T21:34:42.193803+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":212,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.195865+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:34:42.195865+0000","last_peered":"2023-06-10T21:34:42.195865+0000","last_clean":"2023-06-10T21:34:42.195865+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:34:42.195865+0000","last_undegraded":"2023-06-10T21:34:42.195865+0000","last_fullsized":"2023-06-10T21:34:42.195865+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":160,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.007831+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:34:42.007831+0000","last_peered":"2023-06-10T21:34:42.007831+0000","last_clean":"2023-06-10T21:34:42.007831+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:34:42.007831+0000","last_undegraded":"2023-06-10T21:34:42.007831+0000","last_fullsized":"2023-06-10T21:34:42.007831+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.194074+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:34:42.194074+0000","last_peered":"2023-06-10T21:34:42.194074+0000","last_clean":"2023-06-10T21:34:42.194074+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:34:42.194074+0000","last_undegraded":"2023-06-10T21:34:42.194074+0000","last_fullsized":"2023-06-10T21:34:42.194074+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":270,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129718+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:34:48.129718+0000","last_peered":"2023-06-10T21:34:48.129718+0000","last_clean":"2023-06-10T21:34:48.129718+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:34:48.129718+0000","last_undegraded":"2023-06-10T21:34:48.129718+0000","last_fullsized":"2023-06-10T21:34:48.129718+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":159,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.006951+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:34:42.006951+0000","last_peered":"2023-06-10T21:34:42.006951+0000","last_clean":"2023-06-10T21:34:42.006951+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:34:42.006951+0000","last_undegraded":"2023-06-10T21:34:42.006951+0000","last_fullsized":"2023-06-10T21:34:42.006951+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":251,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.074079+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:34:42.074079+0000","last_peered":"2023-06-10T21:34:42.074079+0000","last_clean":"2023-06-10T21:34:42.074079+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:34:42.074079+0000","last_undegraded":"2023-06-10T21:34:42.074079+0000","last_fullsized":"2023-06-10T21:34:42.074079+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002189+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:34:42.002189+0000","last_peered":"2023-06-10T21:34:42.002189+0000","last_clean":"2023-06-10T21:34:42.002189+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:34:42.002189+0000","last_undegraded":"2023-06-10T21:34:42.002189+0000","last_fullsized":"2023-06-10T21:34:42.002189+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":25,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.195094+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:34:42.195094+0000","last_peered":"2023-06-10T21:34:42.195094+0000","last_clean":"2023-06-10T21:34:42.195094+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:34:42.195094+0000","last_undegraded":"2023-06-10T21:34:42.195094+0000","last_fullsized":"2023-06-10T21:34:42.195094+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":30,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002579+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:34:42.002579+0000","last_peered":"2023-06-10T21:34:42.002579+0000","last_clean":"2023-06-10T21:34:42.002579+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:34:42.002579+0000","last_undegraded":"2023-06-10T21:34:42.002579+0000","last_fullsized":"2023-06-10T21:34:42.002579+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":265,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.003920+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:34:42.003920+0000","last_peered":"2023-06-10T21:34:42.003920+0000","last_clean":"2023-06-10T21:34:42.003920+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:34:42.003920+0000","last_undegraded":"2023-06-10T21:34:42.003920+0000","last_fullsized":"2023-06-10T21:34:42.003920+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":173,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002372+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:34:42.002372+0000","last_peered":"2023-06-10T21:34:42.002372+0000","last_clean":"2023-06-10T21:34:42.002372+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:34:42.002372+0000","last_undegraded":"2023-06-10T21:34:42.002372+0000","last_fullsized":"2023-06-10T21:34:42.002372+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":34,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002586+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:34:42.002586+0000","last_peered":"2023-06-10T21:34:42.002586+0000","last_clean":"2023-06-10T21:34:42.002586+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:34:42.002586+0000","last_undegraded":"2023-06-10T21:34:42.002586+0000","last_fullsized":"2023-06-10T21:34:42.002586+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":302,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002674+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:34:42.002674+0000","last_peered":"2023-06-10T21:34:42.002674+0000","last_clean":"2023-06-10T21:34:42.002674+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:34:42.002674+0000","last_undegraded":"2023-06-10T21:34:42.002674+0000","last_fullsized":"2023-06-10T21:34:42.002674+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":237,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193280+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:34:42.193280+0000","last_peered":"2023-06-10T21:34:42.193280+0000","last_clean":"2023-06-10T21:34:42.193280+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:34:42.193280+0000","last_undegraded":"2023-06-10T21:34:42.193280+0000","last_fullsized":"2023-06-10T21:34:42.193280+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":159,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.007591+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:34:42.007591+0000","last_peered":"2023-06-10T21:34:42.007591+0000","last_clean":"2023-06-10T21:34:42.007591+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:34:42.007591+0000","last_undegraded":"2023-06-10T21:34:42.007591+0000","last_fullsized":"2023-06-10T21:34:42.007591+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":231,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.005303+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:34:42.005303+0000","last_peered":"2023-06-10T21:34:42.005303+0000","last_clean":"2023-06-10T21:34:42.005303+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:34:42.005303+0000","last_undegraded":"2023-06-10T21:34:42.005303+0000","last_fullsized":"2023-06-10T21:34:42.005303+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":279,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000557+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:34:42.000557+0000","last_peered":"2023-06-10T21:34:42.000557+0000","last_clean":"2023-06-10T21:34:42.000557+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:34:42.000557+0000","last_undegraded":"2023-06-10T21:34:42.000557+0000","last_fullsized":"2023-06-10T21:34:42.000557+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.007782+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:34:42.007782+0000","last_peered":"2023-06-10T21:34:42.007782+0000","last_clean":"2023-06-10T21:34:42.007782+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:34:42.007782+0000","last_undegraded":"2023-06-10T21:34:42.007782+0000","last_fullsized":"2023-06-10T21:34:42.007782+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":278,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.008027+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:34:42.008027+0000","last_peered":"2023-06-10T21:34:42.008027+0000","last_clean":"2023-06-10T21:34:42.008027+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:34:42.008027+0000","last_undegraded":"2023-06-10T21:34:42.008027+0000","last_fullsized":"2023-06-10T21:34:42.008027+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":255,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193475+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:34:42.193475+0000","last_peered":"2023-06-10T21:34:42.193475+0000","last_clean":"2023-06-10T21:34:42.193475+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:34:42.193475+0000","last_undegraded":"2023-06-10T21:34:42.193475+0000","last_fullsized":"2023-06-10T21:34:42.193475+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"303'9576","reported_seq":9620,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:08.076577+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:35:08.076577+0000","last_peered":"2023-06-10T21:35:08.076577+0000","last_clean":"2023-06-10T21:35:08.076577+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:35:08.076577+0000","last_undegraded":"2023-06-10T21:35:08.076577+0000","last_fullsized":"2023-06-10T21:35:08.076577+0000","mapping_epoch":269,"log_start":"282'3100","ondisk_log_start":"282'3100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63438848,"num_objects":968,"num_object_clones":0,"num_object_copies":2904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":28,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002818+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:34:42.002818+0000","last_peered":"2023-06-10T21:34:42.002818+0000","last_clean":"2023-06-10T21:34:42.002818+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:34:42.002818+0000","last_undegraded":"2023-06-10T21:34:42.002818+0000","last_fullsized":"2023-06-10T21:34:42.002818+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":24,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000270+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:34:42.000270+0000","last_peered":"2023-06-10T21:34:42.000270+0000","last_clean":"2023-06-10T21:34:42.000270+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:34:42.000270+0000","last_undegraded":"2023-06-10T21:34:42.000270+0000","last_fullsized":"2023-06-10T21:34:42.000270+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"303'9630","reported_seq":9674,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:08.142592+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:35:08.142592+0000","last_peered":"2023-06-10T21:35:08.142592+0000","last_clean":"2023-06-10T21:35:08.142592+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:35:08.142592+0000","last_undegraded":"2023-06-10T21:35:08.142592+0000","last_fullsized":"2023-06-10T21:35:08.142592+0000","mapping_epoch":269,"log_start":"282'3200","ondisk_log_start":"282'3200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63307776,"num_objects":966,"num_object_clones":0,"num_object_copies":2898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9630,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":28,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002270+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:34:42.002270+0000","last_peered":"2023-06-10T21:34:42.002270+0000","last_clean":"2023-06-10T21:34:42.002270+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:34:42.002270+0000","last_undegraded":"2023-06-10T21:34:42.002270+0000","last_fullsized":"2023-06-10T21:34:42.002270+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":176,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.195934+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:34:42.195934+0000","last_peered":"2023-06-10T21:34:42.195934+0000","last_clean":"2023-06-10T21:34:42.195934+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:34:42.195934+0000","last_undegraded":"2023-06-10T21:34:42.195934+0000","last_fullsized":"2023-06-10T21:34:42.195934+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"303'9906","reported_seq":9954,"reported_epoch":303,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:35:08.123650+0000","last_change":"2023-06-10T21:32:57.199647+0000","last_active":"2023-06-10T21:35:08.123650+0000","last_peered":"2023-06-10T21:35:08.123650+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:35:08.123650+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"283'3500","ondisk_log_start":"283'3500","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536000,"num_objects":1000,"num_object_clones":0,"num_object_copies":3000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9906,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":1000},{"shards":"0,3","objects":-4394}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":310,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.005188+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:34:42.005188+0000","last_peered":"2023-06-10T21:34:42.005188+0000","last_clean":"2023-06-10T21:34:42.005188+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:34:42.005188+0000","last_undegraded":"2023-06-10T21:34:42.005188+0000","last_fullsized":"2023-06-10T21:34:42.005188+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000688+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:34:42.000688+0000","last_peered":"2023-06-10T21:34:42.000688+0000","last_clean":"2023-06-10T21:34:42.000688+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:34:42.000688+0000","last_undegraded":"2023-06-10T21:34:42.000688+0000","last_fullsized":"2023-06-10T21:34:42.000688+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"303'9585","reported_seq":9627,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.610293+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:35:09.610293+0000","last_peered":"2023-06-10T21:35:09.610293+0000","last_clean":"2023-06-10T21:35:09.610293+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:35:09.610293+0000","last_undegraded":"2023-06-10T21:35:09.610293+0000","last_fullsized":"2023-06-10T21:35:09.610293+0000","mapping_epoch":269,"log_start":"281'2200","ondisk_log_start":"281'2200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7385,"log_dups_size":0,"ondisk_log_size":7385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58916864,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9585,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193396+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:34:42.193396+0000","last_peered":"2023-06-10T21:34:42.193396+0000","last_clean":"2023-06-10T21:34:42.193396+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:34:42.193396+0000","last_undegraded":"2023-06-10T21:34:42.193396+0000","last_fullsized":"2023-06-10T21:34:42.193396+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":126,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.008116+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:34:42.008116+0000","last_peered":"2023-06-10T21:34:42.008116+0000","last_clean":"2023-06-10T21:34:42.008116+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:34:42.008116+0000","last_undegraded":"2023-06-10T21:34:42.008116+0000","last_fullsized":"2023-06-10T21:34:42.008116+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"303'9509","reported_seq":19544,"reported_epoch":303,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:35:09.296372+0000","last_change":"2023-06-10T21:32:57.219186+0000","last_active":"2023-06-10T21:35:09.296372+0000","last_peered":"2023-06-10T21:35:09.296372+0000","last_clean":"2023-06-10T21:32:54.512955+0000","last_became_active":"2023-06-10T21:32:57.177768+0000","last_became_peered":"2023-06-10T21:32:57.177768+0000","last_unstale":"2023-06-10T21:35:09.296372+0000","last_undegraded":"2023-06-10T21:32:56.639930+0000","last_fullsized":"2023-06-10T21:32:56.518794+0000","mapping_epoch":289,"log_start":"276'600","ondisk_log_start":"276'600","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8909,"log_dups_size":0,"ondisk_log_size":8909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:37.584285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57606144,"num_objects":879,"num_object_clones":0,"num_object_copies":2637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":201,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9509,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4993,"num_bytes_recovered":306511872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[2,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2","objects":678}],"blocked_by":[],"up_primary":7,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":254,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193526+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:34:42.193526+0000","last_peered":"2023-06-10T21:34:42.193526+0000","last_clean":"2023-06-10T21:34:42.193526+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:34:42.193526+0000","last_undegraded":"2023-06-10T21:34:42.193526+0000","last_fullsized":"2023-06-10T21:34:42.193526+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193400+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:34:42.193400+0000","last_peered":"2023-06-10T21:34:42.193400+0000","last_clean":"2023-06-10T21:34:42.193400+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:34:42.193400+0000","last_undegraded":"2023-06-10T21:34:42.193400+0000","last_fullsized":"2023-06-10T21:34:42.193400+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"303'9775","reported_seq":9819,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.271211+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:35:09.271211+0000","last_peered":"2023-06-10T21:35:09.271211+0000","last_clean":"2023-06-10T21:35:09.271211+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:35:09.271211+0000","last_undegraded":"2023-06-10T21:35:09.271211+0000","last_fullsized":"2023-06-10T21:35:09.271211+0000","mapping_epoch":269,"log_start":"277'900","ondisk_log_start":"277'900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8875,"log_dups_size":0,"ondisk_log_size":8875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64159744,"num_objects":979,"num_object_clones":0,"num_object_copies":2937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9775,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.194293+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:34:42.194293+0000","last_peered":"2023-06-10T21:34:42.194293+0000","last_clean":"2023-06-10T21:34:42.194293+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:34:42.194293+0000","last_undegraded":"2023-06-10T21:34:42.194293+0000","last_fullsized":"2023-06-10T21:34:42.194293+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.966892+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:34:56.966892+0000","last_peered":"2023-06-10T21:34:56.966892+0000","last_clean":"2023-06-10T21:34:56.966892+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:34:56.966892+0000","last_undegraded":"2023-06-10T21:34:56.966892+0000","last_fullsized":"2023-06-10T21:34:56.966892+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"303'9550","reported_seq":9594,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.612081+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:35:06.612081+0000","last_peered":"2023-06-10T21:35:06.612081+0000","last_clean":"2023-06-10T21:35:06.612081+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:35:06.612081+0000","last_undegraded":"2023-06-10T21:35:06.612081+0000","last_fullsized":"2023-06-10T21:35:06.612081+0000","mapping_epoch":269,"log_start":"283'3700","ondisk_log_start":"283'3700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64749568,"num_objects":988,"num_object_clones":0,"num_object_copies":2964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9550,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":281,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:44.241488+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:34:44.241488+0000","last_peered":"2023-06-10T21:34:44.241488+0000","last_clean":"2023-06-10T21:34:44.241488+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:34:44.241488+0000","last_undegraded":"2023-06-10T21:34:44.241488+0000","last_fullsized":"2023-06-10T21:34:44.241488+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":77,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193814+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:34:42.193814+0000","last_peered":"2023-06-10T21:34:42.193814+0000","last_clean":"2023-06-10T21:34:42.193814+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:34:42.193814+0000","last_undegraded":"2023-06-10T21:34:42.193814+0000","last_fullsized":"2023-06-10T21:34:42.193814+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"303'9503","reported_seq":9547,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.602415+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:35:06.602415+0000","last_peered":"2023-06-10T21:35:06.602415+0000","last_clean":"2023-06-10T21:35:06.602415+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:35:06.602415+0000","last_undegraded":"2023-06-10T21:35:06.602415+0000","last_fullsized":"2023-06-10T21:35:06.602415+0000","mapping_epoch":269,"log_start":"283'3700","ondisk_log_start":"283'3700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091904,"num_objects":1039,"num_object_clones":0,"num_object_copies":3117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9503,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":257,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193980+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:34:42.193980+0000","last_peered":"2023-06-10T21:34:42.193980+0000","last_clean":"2023-06-10T21:34:42.193980+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:34:42.193980+0000","last_undegraded":"2023-06-10T21:34:42.193980+0000","last_fullsized":"2023-06-10T21:34:42.193980+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":213,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193692+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:34:42.193692+0000","last_peered":"2023-06-10T21:34:42.193692+0000","last_clean":"2023-06-10T21:34:42.193692+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:34:42.193692+0000","last_undegraded":"2023-06-10T21:34:42.193692+0000","last_fullsized":"2023-06-10T21:34:42.193692+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"303'9534","reported_seq":9578,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.291591+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:35:09.291591+0000","last_peered":"2023-06-10T21:35:09.291591+0000","last_clean":"2023-06-10T21:35:09.291591+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:35:09.291591+0000","last_undegraded":"2023-06-10T21:35:09.291591+0000","last_fullsized":"2023-06-10T21:35:09.291591+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8834,"log_dups_size":0,"ondisk_log_size":8834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58261528,"num_objects":890,"num_object_clones":0,"num_object_copies":2670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":890,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9534,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":127,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129644+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:34:48.129644+0000","last_peered":"2023-06-10T21:34:48.129644+0000","last_clean":"2023-06-10T21:34:48.129644+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:34:48.129644+0000","last_undegraded":"2023-06-10T21:34:48.129644+0000","last_fullsized":"2023-06-10T21:34:48.129644+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193897+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:34:42.193897+0000","last_peered":"2023-06-10T21:34:42.193897+0000","last_clean":"2023-06-10T21:34:42.193897+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:34:42.193897+0000","last_undegraded":"2023-06-10T21:34:42.193897+0000","last_fullsized":"2023-06-10T21:34:42.193897+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129557+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:34:48.129557+0000","last_peered":"2023-06-10T21:34:48.129557+0000","last_clean":"2023-06-10T21:34:48.129557+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:34:48.129557+0000","last_undegraded":"2023-06-10T21:34:48.129557+0000","last_fullsized":"2023-06-10T21:34:48.129557+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"303'9568","reported_seq":9611,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.589401+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:35:06.589401+0000","last_peered":"2023-06-10T21:35:06.589401+0000","last_clean":"2023-06-10T21:35:06.589401+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:35:06.589401+0000","last_undegraded":"2023-06-10T21:35:06.589401+0000","last_fullsized":"2023-06-10T21:35:06.589401+0000","mapping_epoch":269,"log_start":"283'3700","ondisk_log_start":"283'3700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5868,"log_dups_size":0,"ondisk_log_size":5868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64225280,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9568,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":173,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129572+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:34:48.129572+0000","last_peered":"2023-06-10T21:34:48.129572+0000","last_clean":"2023-06-10T21:34:48.129572+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:34:48.129572+0000","last_undegraded":"2023-06-10T21:34:48.129572+0000","last_fullsized":"2023-06-10T21:34:48.129572+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":247,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129605+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:34:48.129605+0000","last_peered":"2023-06-10T21:34:48.129605+0000","last_clean":"2023-06-10T21:34:48.129605+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:34:48.129605+0000","last_undegraded":"2023-06-10T21:34:48.129605+0000","last_fullsized":"2023-06-10T21:34:48.129605+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":136,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.003736+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:34:42.003736+0000","last_peered":"2023-06-10T21:34:42.003736+0000","last_clean":"2023-06-10T21:34:42.003736+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:34:42.003736+0000","last_undegraded":"2023-06-10T21:34:42.003736+0000","last_fullsized":"2023-06-10T21:34:42.003736+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"303'9480","reported_seq":9524,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.493140+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:35:06.493140+0000","last_peered":"2023-06-10T21:35:06.493140+0000","last_clean":"2023-06-10T21:35:06.493140+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:35:06.493140+0000","last_undegraded":"2023-06-10T21:35:06.493140+0000","last_fullsized":"2023-06-10T21:35:06.493140+0000","mapping_epoch":269,"log_start":"283'3700","ondisk_log_start":"283'3700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9480,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":147,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.003856+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:34:42.003856+0000","last_peered":"2023-06-10T21:34:42.003856+0000","last_clean":"2023-06-10T21:34:42.003856+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:34:42.003856+0000","last_undegraded":"2023-06-10T21:34:42.003856+0000","last_fullsized":"2023-06-10T21:34:42.003856+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.003926+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:34:42.003926+0000","last_peered":"2023-06-10T21:34:42.003926+0000","last_clean":"2023-06-10T21:34:42.003926+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:34:42.003926+0000","last_undegraded":"2023-06-10T21:34:42.003926+0000","last_fullsized":"2023-06-10T21:34:42.003926+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":196,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.979026+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:34:56.979026+0000","last_peered":"2023-06-10T21:34:56.979026+0000","last_clean":"2023-06-10T21:34:56.979026+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:34:56.979026+0000","last_undegraded":"2023-06-10T21:34:56.979026+0000","last_fullsized":"2023-06-10T21:34:56.979026+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"303'9346","reported_seq":9390,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.564096+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:35:06.564096+0000","last_peered":"2023-06-10T21:35:06.564096+0000","last_clean":"2023-06-10T21:35:06.564096+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:35:06.564096+0000","last_undegraded":"2023-06-10T21:35:06.564096+0000","last_fullsized":"2023-06-10T21:35:06.564096+0000","mapping_epoch":269,"log_start":"283'3500","ondisk_log_start":"283'3500","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66584576,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9346,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":327,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.007038+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:34:42.007038+0000","last_peered":"2023-06-10T21:34:42.007038+0000","last_clean":"2023-06-10T21:34:42.007038+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:34:42.007038+0000","last_undegraded":"2023-06-10T21:34:42.007038+0000","last_fullsized":"2023-06-10T21:34:42.007038+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":466,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000343+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:34:42.000343+0000","last_peered":"2023-06-10T21:34:42.000343+0000","last_clean":"2023-06-10T21:34:42.000343+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:34:42.000343+0000","last_undegraded":"2023-06-10T21:34:42.000343+0000","last_fullsized":"2023-06-10T21:34:42.000343+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":240,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.194395+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:34:42.194395+0000","last_peered":"2023-06-10T21:34:42.194395+0000","last_clean":"2023-06-10T21:34:42.194395+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:34:42.194395+0000","last_undegraded":"2023-06-10T21:34:42.194395+0000","last_fullsized":"2023-06-10T21:34:42.194395+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":117,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.005274+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:34:42.005274+0000","last_peered":"2023-06-10T21:34:42.005274+0000","last_clean":"2023-06-10T21:34:42.005274+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:34:42.005274+0000","last_undegraded":"2023-06-10T21:34:42.005274+0000","last_fullsized":"2023-06-10T21:34:42.005274+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"303'9581","reported_seq":9625,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:06.611989+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:35:06.611989+0000","last_peered":"2023-06-10T21:35:06.611989+0000","last_clean":"2023-06-10T21:35:06.611989+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:35:06.611989+0000","last_undegraded":"2023-06-10T21:35:06.611989+0000","last_fullsized":"2023-06-10T21:35:06.611989+0000","mapping_epoch":269,"log_start":"283'3800","ondisk_log_start":"283'3800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69402624,"num_objects":1059,"num_object_clones":0,"num_object_copies":3177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9581,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":322,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000413+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:34:42.000413+0000","last_peered":"2023-06-10T21:34:42.000413+0000","last_clean":"2023-06-10T21:34:42.000413+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:34:42.000413+0000","last_undegraded":"2023-06-10T21:34:42.000413+0000","last_fullsized":"2023-06-10T21:34:42.000413+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129541+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:34:48.129541+0000","last_peered":"2023-06-10T21:34:48.129541+0000","last_clean":"2023-06-10T21:34:48.129541+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:34:48.129541+0000","last_undegraded":"2023-06-10T21:34:48.129541+0000","last_fullsized":"2023-06-10T21:34:48.129541+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":127,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.980724+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:34:56.980724+0000","last_peered":"2023-06-10T21:34:56.980724+0000","last_clean":"2023-06-10T21:34:56.980724+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:34:56.980724+0000","last_undegraded":"2023-06-10T21:34:56.980724+0000","last_fullsized":"2023-06-10T21:34:56.980724+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"303'9607","reported_seq":9651,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.253688+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:35:09.253688+0000","last_peered":"2023-06-10T21:35:09.253688+0000","last_clean":"2023-06-10T21:35:09.253688+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:35:09.253688+0000","last_undegraded":"2023-06-10T21:35:09.253688+0000","last_fullsized":"2023-06-10T21:35:09.253688+0000","mapping_epoch":269,"log_start":"276'700","ondisk_log_start":"276'700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":8907,"log_dups_size":0,"ondisk_log_size":8907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60882944,"num_objects":929,"num_object_clones":0,"num_object_copies":2787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9607,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":307,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.005206+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:34:42.005206+0000","last_peered":"2023-06-10T21:34:42.005206+0000","last_clean":"2023-06-10T21:34:42.005206+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:34:42.005206+0000","last_undegraded":"2023-06-10T21:34:42.005206+0000","last_fullsized":"2023-06-10T21:34:42.005206+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.008003+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:34:42.008003+0000","last_peered":"2023-06-10T21:34:42.008003+0000","last_clean":"2023-06-10T21:34:42.008003+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:34:42.008003+0000","last_undegraded":"2023-06-10T21:34:42.008003+0000","last_fullsized":"2023-06-10T21:34:42.008003+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":123,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129620+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:34:48.129620+0000","last_peered":"2023-06-10T21:34:48.129620+0000","last_clean":"2023-06-10T21:34:48.129620+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:34:48.129620+0000","last_undegraded":"2023-06-10T21:34:48.129620+0000","last_fullsized":"2023-06-10T21:34:48.129620+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"303'9529","reported_seq":9571,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.667841+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:35:09.667841+0000","last_peered":"2023-06-10T21:35:09.667841+0000","last_clean":"2023-06-10T21:35:09.667841+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:35:09.667841+0000","last_undegraded":"2023-06-10T21:35:09.667841+0000","last_fullsized":"2023-06-10T21:35:09.667841+0000","mapping_epoch":269,"log_start":"281'2200","ondisk_log_start":"281'2200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56688640,"num_objects":865,"num_object_clones":0,"num_object_copies":2595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9529,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":280,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193579+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:34:42.193579+0000","last_peered":"2023-06-10T21:34:42.193579+0000","last_clean":"2023-06-10T21:34:42.193579+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:34:42.193579+0000","last_undegraded":"2023-06-10T21:34:42.193579+0000","last_fullsized":"2023-06-10T21:34:42.193579+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":77,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193673+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:34:42.193673+0000","last_peered":"2023-06-10T21:34:42.193673+0000","last_clean":"2023-06-10T21:34:42.193673+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:34:42.193673+0000","last_undegraded":"2023-06-10T21:34:42.193673+0000","last_fullsized":"2023-06-10T21:34:42.193673+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.618770+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:34:56.618770+0000","last_peered":"2023-06-10T21:34:56.618770+0000","last_clean":"2023-06-10T21:34:56.618770+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:34:56.618770+0000","last_undegraded":"2023-06-10T21:34:56.618770+0000","last_fullsized":"2023-06-10T21:34:56.618770+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"303'9494","reported_seq":9536,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:35:09.701989+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:35:09.701989+0000","last_peered":"2023-06-10T21:35:09.701989+0000","last_clean":"2023-06-10T21:35:09.701989+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:35:09.701989+0000","last_undegraded":"2023-06-10T21:35:09.701989+0000","last_fullsized":"2023-06-10T21:35:09.701989+0000","mapping_epoch":269,"log_start":"281'2100","ondisk_log_start":"281'2100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9494,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":24,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.002780+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:34:42.002780+0000","last_peered":"2023-06-10T21:34:42.002780+0000","last_clean":"2023-06-10T21:34:42.002780+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:34:42.002780+0000","last_undegraded":"2023-06-10T21:34:42.002780+0000","last_fullsized":"2023-06-10T21:34:42.002780+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":131,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.980881+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:34:56.980881+0000","last_peered":"2023-06-10T21:34:56.980881+0000","last_clean":"2023-06-10T21:34:56.980881+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:34:56.980881+0000","last_undegraded":"2023-06-10T21:34:56.980881+0000","last_fullsized":"2023-06-10T21:34:56.980881+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129519+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:34:48.129519+0000","last_peered":"2023-06-10T21:34:48.129519+0000","last_clean":"2023-06-10T21:34:48.129519+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:34:48.129519+0000","last_undegraded":"2023-06-10T21:34:48.129519+0000","last_fullsized":"2023-06-10T21:34:48.129519+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":126,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.008459+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:34:42.008459+0000","last_peered":"2023-06-10T21:34:42.008459+0000","last_clean":"2023-06-10T21:34:42.008459+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:34:42.008459+0000","last_undegraded":"2023-06-10T21:34:42.008459+0000","last_fullsized":"2023-06-10T21:34:42.008459+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":144,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.008087+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:34:42.008087+0000","last_peered":"2023-06-10T21:34:42.008087+0000","last_clean":"2023-06-10T21:34:42.008087+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:34:42.008087+0000","last_undegraded":"2023-06-10T21:34:42.008087+0000","last_fullsized":"2023-06-10T21:34:42.008087+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":28,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.000611+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:34:42.000611+0000","last_peered":"2023-06-10T21:34:42.000611+0000","last_clean":"2023-06-10T21:34:42.000611+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:34:42.000611+0000","last_undegraded":"2023-06-10T21:34:42.000611+0000","last_fullsized":"2023-06-10T21:34:42.000611+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193336+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:34:42.193336+0000","last_peered":"2023-06-10T21:34:42.193336+0000","last_clean":"2023-06-10T21:34:42.193336+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:34:42.193336+0000","last_undegraded":"2023-06-10T21:34:42.193336+0000","last_fullsized":"2023-06-10T21:34:42.193336+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":81,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.193582+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:34:42.193582+0000","last_peered":"2023-06-10T21:34:42.193582+0000","last_clean":"2023-06-10T21:34:42.193582+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:34:42.193582+0000","last_undegraded":"2023-06-10T21:34:42.193582+0000","last_fullsized":"2023-06-10T21:34:42.193582+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.194334+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:34:42.194334+0000","last_peered":"2023-06-10T21:34:42.194334+0000","last_clean":"2023-06-10T21:34:42.194334+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:34:42.194334+0000","last_undegraded":"2023-06-10T21:34:42.194334+0000","last_fullsized":"2023-06-10T21:34:42.194334+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":287,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:56.619001+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:34:56.619001+0000","last_peered":"2023-06-10T21:34:56.619001+0000","last_clean":"2023-06-10T21:34:56.619001+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:34:56.619001+0000","last_undegraded":"2023-06-10T21:34:56.619001+0000","last_fullsized":"2023-06-10T21:34:56.619001+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.197233+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:34:42.197233+0000","last_peered":"2023-06-10T21:34:42.197233+0000","last_clean":"2023-06-10T21:34:42.197233+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:34:42.197233+0000","last_undegraded":"2023-06-10T21:34:42.197233+0000","last_fullsized":"2023-06-10T21:34:42.197233+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":192,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:48.129750+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:34:48.129750+0000","last_peered":"2023-06-10T21:34:48.129750+0000","last_clean":"2023-06-10T21:34:48.129750+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:34:48.129750+0000","last_undegraded":"2023-06-10T21:34:48.129750+0000","last_fullsized":"2023-06-10T21:34:48.129750+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":36,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-06-10T21:34:42.003626+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:34:42.003626+0000","last_peered":"2023-06-10T21:34:42.003626+0000","last_clean":"2023-06-10T21:34:42.003626+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:34:42.003626+0000","last_undegraded":"2023-06-10T21:34:42.003626+0000","last_fullsized":"2023-06-10T21:34:42.003626+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1007157272,"num_objects":15369,"num_object_clones":0,"num_object_copies":46107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5654,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15369,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153173,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4993,"num_bytes_recovered":306511872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3664654336,"data_stored":3664642120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111873,"ondisk_log_size":111873,"up":48,"acting":46,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430657,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":883624,"kb_used_data":290400,"kb_used_omap":0,"kb_used_meta":593216,"kb_avail":92861528,"statfs":{"total":95995035648,"available":95090204672,"internally_reserved":0,"allocated":297369600,"data_stored":296804712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":607453184},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,3],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":4.2880000000000003,"15min":7.9009999999999998},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":26.878,"5min":184.83699999999999,"15min":184.83699999999999},"last":28.875},{"interface":"front","average":{"1min":3.3559999999999999,"5min":4.3019999999999996,"15min":8.1340000000000003},"min":{"1min":0.28599999999999998,"5min":0.23899999999999999,"15min":0.223},"max":{"1min":26.934999999999999,"5min":185.58000000000001,"15min":185.58000000000001},"last":28.943000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:34:13 2023","interfaces":[{"interface":"back","average":{"1min":4.8109999999999999,"5min":3.3130000000000002,"15min":3.2519999999999998},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.21299999999999999},"max":{"1min":59.268000000000001,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":4.4749999999999996,"5min":3.4769999999999999,"15min":3.754},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":59.256999999999998,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:34:26 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":6.4400000000000004,"15min":4.1849999999999996},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":89.055999999999997,"5min":172.33600000000001,"15min":172.33600000000001},"last":103.866},{"interface":"front","average":{"1min":7.0510000000000002,"5min":6.3639999999999999,"15min":4.8490000000000002},"min":{"1min":0.38300000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":88.787000000000006,"5min":172.185,"15min":212.13399999999999},"last":103.705}]},{"osd":3,"last update":"Sat Jun 10 21:34:07 2023","interfaces":[{"interface":"back","average":{"1min":8.5359999999999996,"5min":4.0730000000000004,"15min":3.024},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":70.841999999999999,"5min":99.266000000000005,"15min":99.266000000000005},"last":0.52200000000000002},{"interface":"front","average":{"1min":7.9180000000000001,"5min":3.714,"15min":2.46},"min":{"1min":0.38300000000000001,"5min":0.31,"15min":0.31},"max":{"1min":69.796999999999997,"5min":99.138000000000005,"15min":99.138000000000005},"last":0.55800000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.71499999999999997},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":0.93799999999999994,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.42099999999999999,"15min":1.0289999999999999},"min":{"1min":0.309,"5min":0.245,"15min":0.245},"max":{"1min":0.81000000000000005,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.51800000000000002,"15min":1.821},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":1.7869999999999999,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.438},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.44500000000000001,"15min":1.6160000000000001},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.69799999999999995,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.503}]},{"osd":6,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":1.077},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":2.516,"5min":3.2559999999999998,"15min":13.073},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48599999999999999,"15min":0.93700000000000006},"min":{"1min":0.34699999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73799999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.47499999999999998}]}]},{"osd":6,"up_from":0,"seq":180388626885,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566327,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1745552,"kb_used_data":1128776,"kb_used_omap":4,"kb_used_meta":616763,"kb_avail":91999600,"statfs":{"total":95995035648,"available":94207590400,"internally_reserved":0,"allocated":1155866624,"data_stored":1154913160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":631566222},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":5.4809999999999999,"15min":10.98},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.372,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.21099999999999999},{"interface":"front","average":{"1min":2.7629999999999999,"5min":5.3680000000000003,"15min":10.67},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":15.218,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":8.8239999999999998,"15min":4.9909999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":18.041,"5min":191.101,"15min":191.101},"last":134.79900000000001},{"interface":"front","average":{"1min":3,"5min":9.1199999999999992,"15min":4.7560000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.218},"max":{"1min":11.964,"5min":190.87899999999999,"15min":190.87899999999999},"last":1.677}]},{"osd":3,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.2290000000000001,"5min":4.3470000000000004,"15min":2.766},"min":{"1min":0.32300000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":42.694000000000003,"5min":80.584999999999994,"15min":80.584999999999994},"last":0.374},{"interface":"front","average":{"1min":5.3959999999999999,"5min":5.266,"15min":3.8759999999999999},"min":{"1min":0.40400000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":42.734999999999999,"5min":80.873999999999995,"15min":80.873999999999995},"last":0.84699999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.4430000000000001,"15min":1.4810000000000001},"min":{"1min":0.28699999999999998,"5min":0.28399999999999997,"15min":0.27700000000000002},"max":{"1min":5.032,"5min":16.059999999999999,"15min":16.059999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.024,"5min":1.504,"15min":1.3540000000000001},"min":{"1min":0.30099999999999999,"5min":0.28299999999999997,"15min":0.27400000000000002},"max":{"1min":4.8200000000000003,"5min":8.2739999999999991,"15min":14.460000000000001},"last":0.501}]},{"osd":5,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.0339999999999998,"15min":4.1349999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.7089999999999996,"5min":8.9009999999999998,"15min":383.83100000000002},"last":4.4290000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.46,"15min":3.7400000000000002},"min":{"1min":0.35299999999999998,"5min":0.34200000000000003,"15min":0.29699999999999999},"max":{"1min":5.0679999999999996,"5min":16.146999999999998,"15min":383.84800000000001},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":2.129,"15min":2.0310000000000001},"min":{"1min":0.34799999999999998,"5min":0.34000000000000002,"15min":0.33800000000000002},"max":{"1min":8.7550000000000008,"5min":8.8580000000000005,"15min":14.858000000000001},"last":4.3899999999999997},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.5129999999999999,"15min":1.643},"min":{"1min":0.40000000000000002,"5min":0.39100000000000001,"15min":0.32900000000000001},"max":{"1min":5.1360000000000001,"5min":11.476000000000001,"15min":13.48},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:34:14 2023","interfaces":[{"interface":"back","average":{"1min":4.3970000000000002,"5min":2.1709999999999998,"15min":2.5979999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32600000000000001},"max":{"1min":47.844000000000001,"5min":47.844000000000001,"15min":162.399},"last":0.48399999999999999},{"interface":"front","average":{"1min":4.4699999999999998,"5min":2.407,"15min":2.3759999999999999},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":47.753999999999998,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.46000000000000002}]}]},{"osd":0,"up_from":244,"seq":1047972020304,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":786524,"kb_used_data":520660,"kb_used_omap":10,"kb_used_meta":265845,"kb_avail":92958628,"statfs":{"total":95995035648,"available":95189635072,"internally_reserved":0,"allocated":533155840,"data_stored":532130618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":272225332},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":3.427,"5min":4.859,"15min":4.5590000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.202000000000002,"5min":139.68000000000001,"15min":139.68000000000001},"last":1.651},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.5590000000000002,"15min":5.5609999999999999},"min":{"1min":0.22800000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":19.751000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":5.415}]},{"osd":2,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":10.782999999999999,"5min":8.8140000000000001,"15min":6.9059999999999997},"min":{"1min":0.33800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":110.34399999999999,"5min":205.10900000000001,"15min":205.10900000000001},"last":1.379},{"interface":"front","average":{"1min":9.6579999999999995,"5min":8.8309999999999995,"15min":6.3470000000000004},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":103.56399999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":1.821}]},{"osd":3,"last update":"Sat Jun 10 21:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.871,"5min":4.5389999999999997,"15min":5.7039999999999997},"min":{"1min":0.23999999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":26.167999999999999,"5min":111.745,"15min":111.745},"last":0.251},{"interface":"front","average":{"1min":4.7510000000000003,"5min":5.5579999999999998,"15min":6.7080000000000002},"min":{"1min":0.30399999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":26.201000000000001,"5min":114.205,"15min":114.205},"last":4.3319999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":2.105,"15min":4.0990000000000002},"min":{"1min":0.32200000000000001,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":10.02,"5min":10.744999999999999,"15min":29.709},"last":5.3250000000000002},{"interface":"front","average":{"1min":3.7389999999999999,"5min":2.1589999999999998,"15min":3.569},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":18.856000000000002,"5min":18.856000000000002,"15min":18.856000000000002},"last":0.48199999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7709999999999999,"5min":2.1789999999999998,"15min":3.6499999999999999},"min":{"1min":0.38300000000000001,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":18.884,"5min":18.884,"15min":18.884},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.105,"15min":3.1110000000000002},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":8.0079999999999991,"5min":12.083,"15min":19.846},"last":1.76}]},{"osd":6,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.1960000000000002,"15min":4.2759999999999998},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.053000000000001,"5min":10.771000000000001,"15min":29.806000000000001},"last":5.391},{"interface":"front","average":{"1min":4.2889999999999997,"5min":2.3029999999999999,"15min":3.7160000000000002},"min":{"1min":0.44600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":18.812000000000001,"5min":18.812000000000001,"15min":18.812000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":2.3340000000000001,"15min":3.5030000000000001},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.9669999999999996,"5min":17.523,"15min":19.866},"last":1.73},{"interface":"front","average":{"1min":2.5659999999999998,"5min":2.3490000000000002,"15min":4.3259999999999996},"min":{"1min":0.376,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.083,"5min":17.792999999999999,"15min":29.77},"last":5.3620000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075716,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1089992,"kb_used_data":612224,"kb_used_omap":7,"kb_used_meta":477496,"kb_avail":92655160,"statfs":{"total":95995035648,"available":94878883840,"internally_reserved":0,"allocated":626917376,"data_stored":625971592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":488955985},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:41 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":4.6749999999999998,"15min":4.694},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":70.510999999999996,"5min":70.510999999999996,"15min":70.510999999999996},"last":5.3259999999999996},{"interface":"front","average":{"1min":7.1420000000000003,"5min":4.734,"15min":5.1719999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":67.828999999999994,"5min":67.828999999999994,"15min":67.828999999999994},"last":1.9399999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":5.5350000000000001,"15min":3.1160000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.19},"max":{"1min":12.734,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":5.4580000000000002,"5min":4.8010000000000002,"15min":2.742},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.214},"max":{"1min":16.503,"5min":147.81299999999999,"15min":147.81299999999999},"last":5.2549999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:07 2023","interfaces":[{"interface":"back","average":{"1min":13.648,"5min":6.2619999999999996,"15min":5.5810000000000004},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":97.296000000000006,"5min":138.03299999999999,"15min":138.03299999999999},"last":5.298},{"interface":"front","average":{"1min":12.912000000000001,"5min":6.1219999999999999,"15min":6.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":97.308999999999997,"5min":138.095,"15min":138.095},"last":1.597}]},{"osd":4,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.744,"15min":2.8359999999999999},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":4.5599999999999996,"5min":8.0969999999999995,"15min":224.00999999999999},"last":1.258},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.8620000000000001,"15min":3.2480000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":8.2739999999999991,"5min":29.486999999999998,"15min":223.28399999999999},"last":0.442}]},{"osd":5,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":1.819,"15min":1.5960000000000001},"min":{"1min":0.41199999999999998,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":10.554,"5min":10.554,"15min":14.180999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.548,"5min":1.7849999999999999,"15min":1.7529999999999999},"min":{"1min":0.38200000000000001,"5min":0.371,"15min":0.32900000000000001},"max":{"1min":4.5860000000000003,"5min":8.1989999999999998,"15min":16.125},"last":1.3460000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.8560000000000001,"15min":1.825},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":4.6130000000000004,"5min":8.1690000000000005,"15min":16.183},"last":1.296},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.2559999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.3870000000000005,"5min":29.538,"15min":29.538},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":5.5590000000000002,"5min":3.544,"15min":2.5529999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":41.164999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.36699999999999999},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4860000000000002,"15min":2.7999999999999998},"min":{"1min":0.43099999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":41.219999999999999,"5min":73.475999999999999,"15min":113.937},"last":1.3220000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824075,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1110740,"kb_used_data":872780,"kb_used_omap":7,"kb_used_meta":237944,"kb_avail":92634412,"statfs":{"total":95995035648,"available":94857637888,"internally_reserved":0,"allocated":893726720,"data_stored":892709297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":243654675},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,5,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.8460000000000001,"5min":4.2629999999999999,"15min":5.1200000000000001},"min":{"1min":0.22800000000000001,"5min":0.218,"15min":0.218},"max":{"1min":53.756999999999998,"5min":93.992999999999995,"15min":93.992999999999995},"last":1.488},{"interface":"front","average":{"1min":5.5069999999999997,"5min":3.9260000000000002,"15min":4.8639999999999999},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.216},"max":{"1min":48.247,"5min":95.623999999999995,"15min":95.623999999999995},"last":1.2629999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":5.3230000000000004,"5min":4.1130000000000004,"15min":3.7970000000000002},"min":{"1min":0.30399999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":20.574999999999999,"5min":20.574999999999999,"15min":20.574999999999999},"last":3.8300000000000001},{"interface":"front","average":{"1min":4.6429999999999998,"5min":3.2599999999999998,"15min":3.4100000000000001},"min":{"1min":0.314,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.545,"5min":17.384,"15min":17.384},"last":1.6000000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2240000000000002,"5min":4.6740000000000004,"15min":10.172000000000001},"min":{"1min":0.32300000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":13.529999999999999,"5min":87.555999999999997,"15min":87.555999999999997},"last":100.676},{"interface":"front","average":{"1min":5.1399999999999997,"5min":6.5449999999999999,"15min":11.743},"min":{"1min":0.35499999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.541,"5min":80.566999999999993,"15min":80.566999999999993},"last":103.032}]},{"osd":4,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":1.621,"15min":1.456},"min":{"1min":0.33800000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":13.417,"5min":13.417,"15min":13.417},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.133,"5min":2.835,"15min":2.7709999999999999},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.311},"max":{"1min":9.2300000000000004,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3029999999999999,"5min":1.383,"15min":1.389},"min":{"1min":0.42599999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.8810000000000002,"5min":6.891,"15min":6.891},"last":1.569},{"interface":"front","average":{"1min":2.931,"5min":1.762,"15min":1.54},"min":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.44600000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":2.8580000000000001,"15min":2.7949999999999999},"min":{"1min":0.505,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":9.2579999999999991,"5min":19.805,"15min":19.805},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.4370000000000001,"15min":1.4390000000000001},"min":{"1min":0.45600000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.9039999999999999,"5min":6.9809999999999999,"15min":6.9809999999999999},"last":1.6279999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:34:11 2023","interfaces":[{"interface":"back","average":{"1min":6.5149999999999997,"5min":3.54,"15min":3.032},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":79.984999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":5.7229999999999999,"5min":2.1070000000000002,"15min":1.645},"min":{"1min":0.48199999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":80.272999999999996,"5min":80.272999999999996,"15min":80.272999999999996},"last":1.538}]}]},{"osd":4,"up_from":30,"seq":128849019341,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":845092,"kb_used_data":239772,"kb_used_omap":0,"kb_used_meta":605312,"kb_avail":92900060,"statfs":{"total":95995035648,"available":95129661440,"internally_reserved":0,"allocated":245526528,"data_stored":244964208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":619839488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2809999999999997,"5min":3.9420000000000002,"15min":8.4320000000000004},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":60.923999999999999,"5min":234.203,"15min":234.203},"last":0.39900000000000002},{"interface":"front","average":{"1min":5.1479999999999997,"5min":4.9189999999999996,"15min":8.266},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":64.399000000000001,"5min":234.19,"15min":234.19},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.782,"15min":2.9159999999999999},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.7170000000000001,"5min":13.461,"15min":182.035},"last":0.32300000000000001},{"interface":"front","average":{"1min":3.5049999999999999,"5min":2.1360000000000001,"15min":2.8359999999999999},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.151,"5min":10.255000000000001,"15min":181.73099999999999},"last":3.6560000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":22.809999999999999,"5min":6.6989999999999998,"15min":3.294},"min":{"1min":0.495,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":255.386,"5min":255.386,"15min":255.386},"last":1.585},{"interface":"front","average":{"1min":24.347999999999999,"5min":6.8520000000000003,"15min":3.2949999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":255.29599999999999,"5min":255.29599999999999,"15min":255.29599999999999},"last":6.5060000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":17.739000000000001,"5min":7.5629999999999997,"15min":4.1150000000000002},"min":{"1min":0.57699999999999996,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":102.94,"5min":234.91900000000001,"15min":234.91900000000001},"last":1.0820000000000001},{"interface":"front","average":{"1min":12.029999999999999,"5min":8.1590000000000007,"15min":4.0819999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":90.489999999999995,"5min":235.05099999999999,"15min":235.05099999999999},"last":1.2569999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":1.0009999999999999},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.58699999999999997,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.46600000000000003,"15min":1.016},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.67900000000000005,"5min":5.25,"15min":11.904999999999999},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.43099999999999999,"15min":1.0249999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.53900000000000003,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.432,"5min":0.46600000000000003,"15min":1.0900000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.59499999999999997,"5min":1.71,"15min":13.845000000000001},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.77500000000000002,"15min":1.2789999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.38},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.79600000000000004,"15min":1.2210000000000001},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.57199999999999995,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.41699999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823112,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":512950272,"data_stored":512950272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1145638912,"data_stored":1145634840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":602673152,"data_stored":602669080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":877461504,"data_stored":877461504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":235868160,"data_stored":235864088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":290062336,"data_stored":290062336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:35:13.697 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:13.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:13.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:13.844 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:13.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:13.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:13.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:14.025 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:14.073 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:14.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:14.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:14.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:14.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: osdmap e306: 8 total, 8 up, 6 in 2023-06-10T21:35:14.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:35:14.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:14.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:14 smithi152 ceph-mon[156819]: pgmap v1122: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 879 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 4678/42255 objects degraded (11.071%); 797/42255 objects misplaced (1.886%); 2.9 MiB/s, 99 objects/s recovering 2023-06-10T21:35:14.237 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:14.370 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:14.387 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:13 smithi064 ceph-mon[180297]: osdmap e306: 8 total, 8 up, 6 in 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:35:14.388 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:14.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[180297]: pgmap v1122: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 879 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 4678/42255 objects degraded (11.071%); 797/42255 objects misplaced (1.886%); 2.9 MiB/s, 99 objects/s recovering 2023-06-10T21:35:14.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:14.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: osdmap e306: 8 total, 8 up, 6 in 2023-06-10T21:35:14.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:14.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:35:14.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:35:14.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:14.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:14 smithi064 ceph-mon[186706]: pgmap v1122: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 87 active+clean; 879 MiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 473 op/s; 4678/42255 objects degraded (11.071%); 797/42255 objects misplaced (1.886%); 2.9 MiB/s, 99 objects/s recovering 2023-06-10T21:35:14.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:14.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:14.757 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:14.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:14.822 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:14.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:14.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:15.116 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:15.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:15.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:15.254 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:15.302 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:15.307 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:15 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.307 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:15 smithi152 ceph-mon[156819]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:15.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:15.403 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:15 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:15 smithi064 ceph-mon[180297]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:15 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.404 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:15 smithi064 ceph-mon[186706]: failed to encode map e306 with expected crc 2023-06-10T21:35:15.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:15.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:15.536 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:35:15.546 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:35:15.546 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:35:15.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:35:15.546 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:35:15.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111472 10.b 10:d6587fef:::benchmark_data_smithi064_312713_object20716:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312613+0000", 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 454.22082483100002, 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022103983, 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:35:15.547 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111472 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312613+0000", 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312613+0000", 2023-06-10T21:35:15.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312611+0000", 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312618+0000", 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2099999999999998e-06 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312630+0000", 2023-06-10T21:35:15.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2046e-05 2023-06-10T21:35:15.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312639+0000", 2023-06-10T21:35:15.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5850000000000006e-06 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312713+0000", 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3405000000000006e-05 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312720+0000", 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3590000000000002e-06 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.317970+0000", 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052496670000000004 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.317976+0000", 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7089999999999996e-06 2023-06-10T21:35:15.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.333213+0000", 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0152370049999999 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.334717+0000", 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001503463 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111475 10.2 10:4c97a12a:::benchmark_data_smithi064_312713_object20719:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.312656+0000", 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 454.22078133299999, 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.009721249, 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:35:15.558 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111475 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312656+0000", 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312656+0000", 2023-06-10T21:35:15.560 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312656+0000", 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.561 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312660+0000", 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5710000000000002e-06 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312667+0000", 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0539999999999997e-06 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.312673+0000", 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.46e-06 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.322789+0000", 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010116190000000001 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:35:15.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.322800+0000", 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0474e-05 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.320009+0000", 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99720966700000002 2023-06-10T21:35:15.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.322378+0000", 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002368308 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.566 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111489 10.2 10:46ad09a6:::benchmark_data_smithi064_312713_object20733:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.337340+0000", 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 454.19609736000001, 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99754554399999995, 2023-06-10T21:35:15.567 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111489 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.568 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337340+0000", 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337340+0000", 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337339+0000", 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:35:15.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337348+0000", 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1579999999999994e-06 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337378+0000", 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0694999999999997e-05 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337389+0000", 2023-06-10T21:35:15.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0271999999999999e-05 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337449+0000", 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.037e-05 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.337460+0000", 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0978000000000001e-05 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.322619+0000", 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98515897100000005 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.323368+0000", 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00074912199999999996 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.323505+0000", 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013725599999999999 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:35:15.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.329931+0000", 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064261379999999996 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.334775+0000", 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048430610000000001 2023-06-10T21:35:15.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.334818+0000", 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3373e-05 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.334886+0000", 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8044000000000001e-05 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.579 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111493 10.b 10:d8ded582:::benchmark_data_smithi064_312713_object20737:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:27:41.356453+0000", 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 454.17698500900002, 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99114899000000001, 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:35:15.580 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111493 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356453+0000", 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356453+0000", 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356451+0000", 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356459+0000", 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3669999999999997e-06 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356476+0000", 2023-06-10T21:35:15.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7056000000000001e-05 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356486+0000", 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9599999999999995e-06 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356553+0000", 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6509999999999998e-05 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:41.356560+0000", 2023-06-10T21:35:15.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.605e-06 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.335129+0000", 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97856839600000001 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.335864+0000", 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073489300000000003 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:35:15.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.335990+0000", 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012601800000000001 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.338530+0000", 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025401920000000001 2023-06-10T21:35:15.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.347492+0000", 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089618089999999994 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.347528+0000", 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5966000000000001e-05 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:27:42.347602+0000", 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4021000000000002e-05 2023-06-10T21:35:15.591 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24678 11.6 e279/269 11:677a8f62:::benchmark_data_smithi064_385725_object24677:head v 279'1564, mlcod=279'1562)", 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:35:15.592 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.418993293, 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.284365914, 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114466+0000", 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2447e-05 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114476+0000", 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9420000000000007e-06 2023-06-10T21:35:15.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114483+0000", 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7329999999999997e-06 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115009+0000", 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052553099999999998 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115028+0000", 2023-06-10T21:35:15.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8950999999999999e-05 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116760+0000", 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001732063 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398810+0000", 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28205071100000001 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24683 11.e e279/269 11:7b1e2b59:::benchmark_data_smithi064_385725_object24682:head v 279'1506, mlcod=279'1502)", 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.418251064, 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28366207199999999, 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.601 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115186+0000", 2023-06-10T21:35:15.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115216+0000", 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9923000000000001e-05 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115227+0000", 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0413e-05 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115233+0000", 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4470000000000003e-06 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115458+0000", 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022539899999999999 2023-06-10T21:35:15.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115476+0000", 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7776999999999999e-05 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398819+0000", 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28334313500000002 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398849+0000", 2023-06-10T21:35:15.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9493999999999999e-05 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24682 11.a e279/269 11:59387607:::benchmark_data_smithi064_385725_object24681:head v 279'1501, mlcod=279'1498)", 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:35:15.609 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.417096531, 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28636024700000001, 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116403+0000", 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2799999999999995e-05 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116415+0000", 2023-06-10T21:35:15.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375e-05 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116422+0000", 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6580000000000002e-06 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398803+0000", 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28238095299999999 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398836+0000", 2023-06-10T21:35:15.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2336999999999999e-05 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402680+0000", 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038444540000000002 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402701+0000", 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1370999999999999e-05 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24681 11.5 e279/272 11:adf48b79:::benchmark_data_smithi064_385725_object24680:head v 279'1595, mlcod=279'1591)", 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.417001473, 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28619006400000002, 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.618 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:35:15.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116461+0000", 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4776000000000001e-05 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116469+0000", 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0469999999999994e-06 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116474+0000", 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7200000000000003e-06 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398798+0000", 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28232310799999999 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398829+0000", 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1575000000000002e-05 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402581+0000", 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037519649999999999 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402626+0000", 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5272999999999998e-05 2023-06-10T21:35:15.625 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24684 11.3 e279/272 11:cb1db951:::benchmark_data_smithi064_385725_object24683:head v 279'1548, mlcod=279'1546)", 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:35:15.626 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.41628903200001, 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.285234922, 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117148+0000", 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117171+0000", 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2493999999999998e-05 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398742+0000", 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28157169199999998 2023-06-10T21:35:15.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398762+0000", 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9312000000000001e-05 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398806+0000", 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4169999999999999e-05 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398833+0000", 2023-06-10T21:35:15.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7229999999999998e-05 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402321+0000", 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034878449999999998 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402384+0000", 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2801999999999999e-05 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24687 11.b e279/269 11:d00b5318:::benchmark_data_smithi064_385725_object24686:head v 279'1542, mlcod=279'1540)", 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 218.41510681099999, 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28407989, 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.635 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118330+0000", 2023-06-10T21:35:15.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118361+0000", 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0694000000000001e-05 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398783+0000", 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.280422161 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398792+0000", 2023-06-10T21:35:15.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9940000000000006e-06 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398895+0000", 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103235 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398921+0000", 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5743000000000001e-05 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402386+0000", 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034653610000000001 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402411+0000", 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4371000000000001e-05 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.642 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:25920 11.5 e280/272 11:a9b9458c:::benchmark_data_smithi064_385725_object25919:head v 280'1670, mlcod=280'1667)", 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 217.14700863799999, 2023-06-10T21:35:15.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22712411299999999, 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:35:15.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386450+0000", 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1576000000000001e-05 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386457+0000", 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0119999999999999e-06 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386463+0000", 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6799999999999998e-06 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.387488+0000", 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001024552 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.387505+0000", 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7699000000000001e-05 2023-06-10T21:35:15.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613522+0000", 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22601626399999999 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613553+0000", 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1594999999999998e-05 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.651 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:25926 11.4 e280/269 11:2939867b:::benchmark_data_smithi064_385725_object25925:head v 280'1586, mlcod=280'1583)", 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 217.146703371, 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.226442118, 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386819+0000", 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5186999999999996e-05 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386828+0000", 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2189999999999993e-06 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386835+0000", 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3579999999999999e-06 2023-06-10T21:35:15.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.388201+0000", 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013661770000000001 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.388214+0000", 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3159999999999999e-05 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613149+0000", 2023-06-10T21:35:15.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22493552 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613177+0000", 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7114000000000002e-05 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.659 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:143495 11.3 e302/289 11:cf547ac5:::benchmark_data_smithi064_385725_object59222:head v 302'8855, mlcod=284'4090)", 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:48.384609+0000", 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 27.148828767000001, 2023-06-10T21:35:15.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.094486019000000004, 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384609+0000", 2023-06-10T21:35:15.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384609+0000", 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384609+0000", 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384612+0000", 2023-06-10T21:35:15.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3519999999999998e-06 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384616+0000", 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9700000000000001e-06 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.384619+0000", 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0340000000000001e-06 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.385684+0000", 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010645839999999999 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.385711+0000", 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7056999999999999e-05 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.390351+0000", 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046405539999999999 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:48.479095+0000", 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.088743670999999996 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.668 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152666 11.7 e303/269 11:e1db0db2:::benchmark_data_smithi064_385725_object68393:head v 303'9544, mlcod=303'9542)", 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.160910+0000", 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3725278939999992, 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.097743254000000002, 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160910+0000", 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160910+0000", 2023-06-10T21:35:15.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160910+0000", 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160914+0000", 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3409999999999996e-06 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160920+0000", 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4509999999999997e-06 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160925+0000", 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7480000000000001e-06 2023-06-10T21:35:15.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.162123+0000", 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001197493 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.162142+0000", 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9635000000000001e-05 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.258607+0000", 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096465151999999998 2023-06-10T21:35:15.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.258653+0000", 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5753000000000001e-05 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.676 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152667 11.7 e303/269 11:e272983f:::benchmark_data_smithi064_385725_object68394:head v 303'9545, mlcod=303'9542)", 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.160937+0000", 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3725012440000004, 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.103905608, 2023-06-10T21:35:15.677 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160937+0000", 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160937+0000", 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160936+0000", 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160943+0000", 2023-06-10T21:35:15.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1019999999999999e-06 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160949+0000", 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7470000000000002e-06 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.160954+0000", 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3959999999999999e-06 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.169232+0000", 2023-06-10T21:35:15.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0082786990000000005 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.169250+0000", 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8189000000000002e-05 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.264806+0000", 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.095555444000000003 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.264842+0000", 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6242000000000003e-05 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.685 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152676 11.f e303/269 11:f04f4ecc:::benchmark_data_smithi064_385725_object68403:head v 303'9454, mlcod=303'9452)", 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.161004+0000", 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3724339709999995, 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10103590699999999, 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161004+0000", 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161004+0000", 2023-06-10T21:35:15.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161003+0000", 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161008+0000", 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5900000000000001e-06 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161015+0000", 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.391e-06 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.161020+0000", 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7299999999999996e-06 2023-06-10T21:35:15.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.165520+0000", 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044994299999999996 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.165537+0000", 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7252e-05 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.262010+0000", 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096473242000000001 2023-06-10T21:35:15.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.262040+0000", 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9595e-05 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.693 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152675 11.7 e303/269 11:ef471c86:::benchmark_data_smithi064_385725_object68402:head v 303'9546, mlcod=303'9542)", 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.167826+0000", 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3656117800000001, 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.097042792000000003, 2023-06-10T21:35:15.694 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167826+0000", 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167826+0000", 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167825+0000", 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167832+0000", 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1289999999999998e-06 2023-06-10T21:35:15.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167842+0000", 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9059999999999997e-06 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.167849+0000", 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6100000000000002e-06 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.258689+0000", 2023-06-10T21:35:15.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.090840038999999997 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.258711+0000", 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1933000000000002e-05 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.264847+0000", 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061358860000000001 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.264869+0000", 2023-06-10T21:35:15.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2004000000000001e-05 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152673 11.0 e303/269 11:05ca9e1f:::benchmark_data_smithi064_385725_object68400:head v 303'9525, mlcod=303'9522)", 2023-06-10T21:35:15.702 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.168385+0000", 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.365052575, 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.092661785999999996, 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168385+0000", 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168385+0000", 2023-06-10T21:35:15.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168385+0000", 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168391+0000", 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6529999999999997e-06 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168402+0000", 2023-06-10T21:35:15.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1181e-05 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168410+0000", 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1189999999999999e-06 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168589+0000", 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017989699999999999 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.168608+0000", 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8536e-05 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.261001+0000", 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.092393170999999996 2023-06-10T21:35:15.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.261047+0000", 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5858e-05 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152680 11.0 e303/269 11:0f9b3706:::benchmark_data_smithi064_385725_object68407:head v 303'9526, mlcod=303'9522)", 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.172319+0000", 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3611184789999999, 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.091450883999999996, 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.711 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172319+0000", 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172319+0000", 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172318+0000", 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172326+0000", 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6799999999999993e-06 2023-06-10T21:35:15.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172337+0000", 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0823e-05 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172345+0000", 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6459999999999993e-06 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172476+0000", 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013113900000000001 2023-06-10T21:35:15.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172495+0000", 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8941e-05 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.263736+0000", 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.091241289000000003 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.263770+0000", 2023-06-10T21:35:15.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4261999999999998e-05 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:152679 11.7 e303/269 11:e6e16abe:::benchmark_data_smithi064_385725_object68406:head v 303'9547, mlcod=303'9542)", 2023-06-10T21:35:15.719 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:07.172403+0000", 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 8.3610347550000004, 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.092618160000000005, 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:35:15.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172403+0000", 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172403+0000", 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:35:15.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172403+0000", 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172406+0000", 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0900000000000001e-06 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172411+0000", 2023-06-10T21:35:15.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2809999999999999e-06 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.172415+0000", 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9389999999999999e-06 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.262031+0000", 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.089615574000000003 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.262051+0000", 2023-06-10T21:35:15.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0424999999999998e-05 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.264990+0000", 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029390050000000002 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:35:15.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:07.265021+0000", 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1053999999999998e-05 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:35:15.728 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:35:15.728 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:35:15.886 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:15.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:15.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:16.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:16.076 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:16 smithi064 ceph-mon[180297]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 823 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 4477/39552 objects degraded (11.319%); 797/39552 objects misplaced (2.015%); 1.2 MiB/s, 53 objects/s recovering 2023-06-10T21:35:16.077 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4678/42255 objects degraded (11.071%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:16.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:16 smithi064 ceph-mon[186706]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 823 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 4477/39552 objects degraded (11.319%); 797/39552 objects misplaced (2.015%); 1.2 MiB/s, 53 objects/s recovering 2023-06-10T21:35:16.077 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4678/42255 objects degraded (11.071%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:16.087 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:16 smithi152 ceph-mon[156819]: pgmap v1123: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 823 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 367 op/s; 4477/39552 objects degraded (11.319%); 797/39552 objects misplaced (2.015%); 1.2 MiB/s, 53 objects/s recovering 2023-06-10T21:35:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4678/42255 objects degraded (11.071%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:35:16.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:35:17.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:17.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:17.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:18.082 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:17 smithi064 ceph-mon[180297]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 772 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 438 op/s; 4477/37086 objects degraded (12.072%); 797/37086 objects misplaced (2.149%); 1.1 MiB/s, 46 objects/s recovering 2023-06-10T21:35:18.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:17 smithi064 ceph-mon[186706]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 772 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 438 op/s; 4477/37086 objects degraded (12.072%); 797/37086 objects misplaced (2.149%); 1.1 MiB/s, 46 objects/s recovering 2023-06-10T21:35:18.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:18.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:17 smithi152 ceph-mon[156819]: pgmap v1124: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 772 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 438 op/s; 4477/37086 objects degraded (12.072%); 797/37086 objects misplaced (2.149%); 1.1 MiB/s, 46 objects/s recovering 2023-06-10T21:35:18.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:35:18.508 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:18.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:18.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:18.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:18.710 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:18.802 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:18.803 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:19.010 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:19.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:19.130 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:19.224 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:19.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:19.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:19.500 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:19.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:19.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:19.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:19.635 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:19.681 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:19.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:19.717 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:35:19.763 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:35:19.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:19.809 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:35:19.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:19.862 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:19.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:19.882 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:35:19.925 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:35:19.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:19.983 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:35:20.022 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:35:20.069 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:35:20.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:20.115 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.117 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:35:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/395090856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:20.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:20 smithi064 ceph-mon[180297]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 667 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 4477/32070 objects degraded (13.960%); 604/32070 objects misplaced (1.883%); 2.4 MiB/s, 64 objects/s recovering 2023-06-10T21:35:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/395090856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:20.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:20 smithi064 ceph-mon[186706]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 667 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 4477/32070 objects degraded (13.960%); 604/32070 objects misplaced (1.883%); 2.4 MiB/s, 64 objects/s recovering 2023-06-10T21:35:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/395090856' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:20 smithi152 ceph-mon[156819]: pgmap v1125: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 667 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 4477/32070 objects degraded (13.960%); 604/32070 objects misplaced (1.883%); 2.4 MiB/s, 64 objects/s recovering 2023-06-10T21:35:21.055 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4477/32070 objects degraded (13.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:21.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4477/32070 objects degraded (13.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:21.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4477/32070 objects degraded (13.960%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:21.960 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:21.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:22.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:22.104 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:22.150 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:22.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:22.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:22.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:22.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:22 smithi064 ceph-mon[180297]: pgmap v1126: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 605 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 4477/29091 objects degraded (15.390%); 604/29091 objects misplaced (2.076%); 2.0 MiB/s, 51 objects/s recovering 2023-06-10T21:35:22.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:22 smithi064 ceph-mon[186706]: pgmap v1126: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 605 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 4477/29091 objects degraded (15.390%); 604/29091 objects misplaced (2.076%); 2.0 MiB/s, 51 objects/s recovering 2023-06-10T21:35:22.349 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:22.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:22 smithi152 ceph-mon[156819]: pgmap v1126: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 605 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 481 op/s; 4477/29091 objects degraded (15.390%); 604/29091 objects misplaced (2.076%); 2.0 MiB/s, 51 objects/s recovering 2023-06-10T21:35:22.484 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:22.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:22.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:22.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:22.674 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:22.994 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:22] "GET /metrics HTTP/1.1" 200 224065 "" "Prometheus/2.33.4" 2023-06-10T21:35:23.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:23.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:23.857 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:23.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:23.950 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:23.999 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:24.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:24.012 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:23 smithi064 ceph-mon[180297]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 574 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4477/27594 objects degraded (16.225%); 416/27594 objects misplaced (1.508%); 2.9 MiB/s, 64 objects/s recovering 2023-06-10T21:35:24.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:24.218 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:24.259 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:23 smithi064 ceph-mon[186706]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 574 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4477/27594 objects degraded (16.225%); 416/27594 objects misplaced (1.508%); 2.9 MiB/s, 64 objects/s recovering 2023-06-10T21:35:24.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:23 smithi152 ceph-mon[156819]: pgmap v1127: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 574 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 4477/27594 objects degraded (16.225%); 416/27594 objects misplaced (1.508%); 2.9 MiB/s, 64 objects/s recovering 2023-06-10T21:35:24.695 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:24.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:24.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:24.909 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:25.217 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:35:25.389 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:35:25.410 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:35:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:35:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:35:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:35:26.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:35:26.497 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:26.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: osdmap e307: 8 total, 8 up, 6 in 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 504 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 4477/24222 objects degraded (18.483%); 416/24222 objects misplaced (1.717%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:26.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4477/27594 objects degraded (16.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:26.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:26.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:26.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:35:26.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: osdmap e307: 8 total, 8 up, 6 in 2023-06-10T21:35:26.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:26.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 504 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 4477/24222 objects degraded (18.483%); 416/24222 objects misplaced (1.717%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:26.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4477/27594 objects degraded (16.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:26.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:26.631 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1743103867' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: osdmap e307: 8 total, 8 up, 6 in 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: failed to encode map e307 with expected crc 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: from='client.36381 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:26.632 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: pgmap v1129: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 504 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 510 op/s; 4477/24222 objects degraded (18.483%); 416/24222 objects misplaced (1.717%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:26.633 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4477/27594 objects degraded (16.225%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:26.633 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:26.683 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:26.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:26.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:26.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:26.876 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:28.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:28.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:28.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:28.173 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:28.221 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:28.229 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:27 smithi152 ceph-mon[156819]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 439 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 4477/21126 objects degraded (21.192%); 416/21126 objects misplaced (1.969%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:28.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:28.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:28.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:28.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:27 smithi064 ceph-mon[180297]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 439 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 4477/21126 objects degraded (21.192%); 416/21126 objects misplaced (1.969%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:27 smithi064 ceph-mon[186706]: pgmap v1130: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 439 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 4477/21126 objects degraded (21.192%); 416/21126 objects misplaced (1.969%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:28.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:29.493 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:29.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:29.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:29.609 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1128,"stamp":"2023-06-10T21:35:25.178430+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":528095142,"num_objects":8074,"num_object_clones":0,"num_object_copies":24222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4477,"num_objects_misplaced":416,"num_objects_unfound":0,"num_objects_dirty":8074,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":160753,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5736,"num_bytes_recovered":341122770,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111317,"ondisk_log_size":111317,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5085892,"kb_used_data":2237972,"kb_used_omap":30,"kb_used_meta":2847329,"kb_avail":557385020,"statfs":{"total":575970213888,"available":570762260480,"internally_reserved":0,"allocated":2291683328,"data_stored":2286578879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2915664934},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-393936896,"num_objects":-6011,"num_object_clones":0,"num_object_copies":-18033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-201,"num_objects_misplaced":-381,"num_objects_unfound":0,"num_objects_dirty":-6011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":582,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.285586"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":299,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.061466+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:35:12.061466+0000","last_peered":"2023-06-10T21:35:12.061466+0000","last_clean":"2023-06-10T21:35:12.061466+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:35:12.061466+0000","last_undegraded":"2023-06-10T21:35:12.061466+0000","last_fullsized":"2023-06-10T21:35:12.061466+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":240,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086902+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:35:12.086902+0000","last_peered":"2023-06-10T21:35:12.086902+0000","last_clean":"2023-06-10T21:35:12.086902+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:35:12.086902+0000","last_undegraded":"2023-06-10T21:35:12.086902+0000","last_fullsized":"2023-06-10T21:35:12.086902+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":215,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086828+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:35:12.086828+0000","last_peered":"2023-06-10T21:35:12.086828+0000","last_clean":"2023-06-10T21:35:12.086828+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:35:12.086828+0000","last_undegraded":"2023-06-10T21:35:12.086828+0000","last_fullsized":"2023-06-10T21:35:12.086828+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":163,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076838+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:35:12.076838+0000","last_peered":"2023-06-10T21:35:12.076838+0000","last_clean":"2023-06-10T21:35:12.076838+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:35:12.076838+0000","last_undegraded":"2023-06-10T21:35:12.076838+0000","last_fullsized":"2023-06-10T21:35:12.076838+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":280,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086607+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:35:12.086607+0000","last_peered":"2023-06-10T21:35:12.086607+0000","last_clean":"2023-06-10T21:35:12.086607+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:35:12.086607+0000","last_undegraded":"2023-06-10T21:35:12.086607+0000","last_fullsized":"2023-06-10T21:35:12.086607+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":273,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073477+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:35:12.073477+0000","last_peered":"2023-06-10T21:35:12.073477+0000","last_clean":"2023-06-10T21:35:12.073477+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:35:12.073477+0000","last_undegraded":"2023-06-10T21:35:12.073477+0000","last_fullsized":"2023-06-10T21:35:12.073477+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":162,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073576+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:35:12.073576+0000","last_peered":"2023-06-10T21:35:12.073576+0000","last_clean":"2023-06-10T21:35:12.073576+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:35:12.073576+0000","last_undegraded":"2023-06-10T21:35:12.073576+0000","last_fullsized":"2023-06-10T21:35:12.073576+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":254,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086118+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:35:12.086118+0000","last_peered":"2023-06-10T21:35:12.086118+0000","last_clean":"2023-06-10T21:35:12.086118+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:35:12.086118+0000","last_undegraded":"2023-06-10T21:35:12.086118+0000","last_fullsized":"2023-06-10T21:35:12.086118+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":27,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067996+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:35:12.067996+0000","last_peered":"2023-06-10T21:35:12.067996+0000","last_clean":"2023-06-10T21:35:12.067996+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:35:12.067996+0000","last_undegraded":"2023-06-10T21:35:12.067996+0000","last_fullsized":"2023-06-10T21:35:12.067996+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:13.000373+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:35:13.000373+0000","last_peered":"2023-06-10T21:35:13.000373+0000","last_clean":"2023-06-10T21:35:13.000373+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:35:13.000373+0000","last_undegraded":"2023-06-10T21:35:13.000373+0000","last_fullsized":"2023-06-10T21:35:13.000373+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":33,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067892+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:35:12.067892+0000","last_peered":"2023-06-10T21:35:12.067892+0000","last_clean":"2023-06-10T21:35:12.067892+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:35:12.067892+0000","last_undegraded":"2023-06-10T21:35:12.067892+0000","last_fullsized":"2023-06-10T21:35:12.067892+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":268,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.061381+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:35:12.061381+0000","last_peered":"2023-06-10T21:35:12.061381+0000","last_clean":"2023-06-10T21:35:12.061381+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:35:12.061381+0000","last_undegraded":"2023-06-10T21:35:12.061381+0000","last_fullsized":"2023-06-10T21:35:12.061381+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":176,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068212+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:35:12.068212+0000","last_peered":"2023-06-10T21:35:12.068212+0000","last_clean":"2023-06-10T21:35:12.068212+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:35:12.068212+0000","last_undegraded":"2023-06-10T21:35:12.068212+0000","last_fullsized":"2023-06-10T21:35:12.068212+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":37,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068390+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:35:12.068390+0000","last_peered":"2023-06-10T21:35:12.068390+0000","last_clean":"2023-06-10T21:35:12.068390+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:35:12.068390+0000","last_undegraded":"2023-06-10T21:35:12.068390+0000","last_fullsized":"2023-06-10T21:35:12.068390+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":305,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068432+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:35:12.068432+0000","last_peered":"2023-06-10T21:35:12.068432+0000","last_clean":"2023-06-10T21:35:12.068432+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:35:12.068432+0000","last_undegraded":"2023-06-10T21:35:12.068432+0000","last_fullsized":"2023-06-10T21:35:12.068432+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086256+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:35:12.086256+0000","last_peered":"2023-06-10T21:35:12.086256+0000","last_clean":"2023-06-10T21:35:12.086256+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:35:12.086256+0000","last_undegraded":"2023-06-10T21:35:12.086256+0000","last_fullsized":"2023-06-10T21:35:12.086256+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076846+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:35:12.076846+0000","last_peered":"2023-06-10T21:35:12.076846+0000","last_clean":"2023-06-10T21:35:12.076846+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:35:12.076846+0000","last_undegraded":"2023-06-10T21:35:12.076846+0000","last_fullsized":"2023-06-10T21:35:12.076846+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":234,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068129+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:35:12.068129+0000","last_peered":"2023-06-10T21:35:12.068129+0000","last_clean":"2023-06-10T21:35:12.068129+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:35:12.068129+0000","last_undegraded":"2023-06-10T21:35:12.068129+0000","last_fullsized":"2023-06-10T21:35:12.068129+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":282,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067772+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:35:12.067772+0000","last_peered":"2023-06-10T21:35:12.067772+0000","last_clean":"2023-06-10T21:35:12.067772+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:35:12.067772+0000","last_undegraded":"2023-06-10T21:35:12.067772+0000","last_fullsized":"2023-06-10T21:35:12.067772+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":162,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076668+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:35:12.076668+0000","last_peered":"2023-06-10T21:35:12.076668+0000","last_clean":"2023-06-10T21:35:12.076668+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:35:12.076668+0000","last_undegraded":"2023-06-10T21:35:12.076668+0000","last_fullsized":"2023-06-10T21:35:12.076668+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":281,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.074173+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:35:12.074173+0000","last_peered":"2023-06-10T21:35:12.074173+0000","last_clean":"2023-06-10T21:35:12.074173+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:35:12.074173+0000","last_undegraded":"2023-06-10T21:35:12.074173+0000","last_fullsized":"2023-06-10T21:35:12.074173+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":258,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086775+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:35:12.086775+0000","last_peered":"2023-06-10T21:35:12.086775+0000","last_clean":"2023-06-10T21:35:12.086775+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:35:12.086775+0000","last_undegraded":"2023-06-10T21:35:12.086775+0000","last_fullsized":"2023-06-10T21:35:12.086775+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"306'10047","reported_seq":10094,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:23.137643+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:35:23.137643+0000","last_peered":"2023-06-10T21:35:23.137643+0000","last_clean":"2023-06-10T21:35:23.137643+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:35:23.137643+0000","last_undegraded":"2023-06-10T21:35:23.137643+0000","last_fullsized":"2023-06-10T21:35:23.137643+0000","mapping_epoch":269,"log_start":"283'3600","ondisk_log_start":"283'3600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10047,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":31,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068103+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:35:12.068103+0000","last_peered":"2023-06-10T21:35:12.068103+0000","last_clean":"2023-06-10T21:35:12.068103+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:35:12.068103+0000","last_undegraded":"2023-06-10T21:35:12.068103+0000","last_fullsized":"2023-06-10T21:35:12.068103+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067611+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:35:12.067611+0000","last_peered":"2023-06-10T21:35:12.067611+0000","last_clean":"2023-06-10T21:35:12.067611+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:35:12.067611+0000","last_undegraded":"2023-06-10T21:35:12.067611+0000","last_fullsized":"2023-06-10T21:35:12.067611+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"306'10087","reported_seq":10133,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:23.144586+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:35:23.144586+0000","last_peered":"2023-06-10T21:35:23.144586+0000","last_clean":"2023-06-10T21:35:23.144586+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:35:23.144586+0000","last_undegraded":"2023-06-10T21:35:23.144586+0000","last_fullsized":"2023-06-10T21:35:23.144586+0000","mapping_epoch":269,"log_start":"283'3700","ondisk_log_start":"283'3700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6387,"log_dups_size":0,"ondisk_log_size":6387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33357824,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10087,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":31,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068046+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:35:12.068046+0000","last_peered":"2023-06-10T21:35:12.068046+0000","last_clean":"2023-06-10T21:35:12.068046+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:35:12.068046+0000","last_undegraded":"2023-06-10T21:35:12.068046+0000","last_fullsized":"2023-06-10T21:35:12.068046+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086901+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:35:12.086901+0000","last_peered":"2023-06-10T21:35:12.086901+0000","last_clean":"2023-06-10T21:35:12.086901+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:35:12.086901+0000","last_undegraded":"2023-06-10T21:35:12.086901+0000","last_fullsized":"2023-06-10T21:35:12.086901+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"306'10388","reported_seq":11442,"reported_epoch":306,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:35:23.154437+0000","last_change":"2023-06-10T21:35:11.043628+0000","last_active":"2023-06-10T21:35:23.154437+0000","last_peered":"2023-06-10T21:35:23.154437+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:35:23.154437+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"283'4000","ondisk_log_start":"283'4000","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6388,"log_dups_size":0,"ondisk_log_size":6388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33947648,"num_objects":518,"num_object_clones":0,"num_object_copies":1554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4477,"num_objects_misplaced":416,"num_objects_unfound":0,"num_objects_dirty":518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10388,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":501,"num_bytes_recovered":32833536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":416},{"shards":"0,3","objects":-4375}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":313,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068011+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:35:12.068011+0000","last_peered":"2023-06-10T21:35:12.068011+0000","last_clean":"2023-06-10T21:35:12.068011+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:35:12.068011+0000","last_undegraded":"2023-06-10T21:35:12.068011+0000","last_fullsized":"2023-06-10T21:35:12.068011+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":27,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067782+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:35:12.067782+0000","last_peered":"2023-06-10T21:35:12.067782+0000","last_clean":"2023-06-10T21:35:12.067782+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:35:12.067782+0000","last_undegraded":"2023-06-10T21:35:12.067782+0000","last_fullsized":"2023-06-10T21:35:12.067782+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"306'10043","reported_seq":10088,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.623104+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:35:24.623104+0000","last_peered":"2023-06-10T21:35:24.623104+0000","last_clean":"2023-06-10T21:35:24.623104+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:35:24.623104+0000","last_undegraded":"2023-06-10T21:35:24.623104+0000","last_fullsized":"2023-06-10T21:35:24.623104+0000","mapping_epoch":269,"log_start":"281'2700","ondisk_log_start":"281'2700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28901376,"num_objects":441,"num_object_clones":0,"num_object_copies":1323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":187,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086424+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:35:12.086424+0000","last_peered":"2023-06-10T21:35:12.086424+0000","last_clean":"2023-06-10T21:35:12.086424+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:35:12.086424+0000","last_undegraded":"2023-06-10T21:35:12.086424+0000","last_fullsized":"2023-06-10T21:35:12.086424+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":130,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:13.008118+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:35:13.008118+0000","last_peered":"2023-06-10T21:35:13.008118+0000","last_clean":"2023-06-10T21:35:13.008118+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:35:13.008118+0000","last_undegraded":"2023-06-10T21:35:13.008118+0000","last_fullsized":"2023-06-10T21:35:13.008118+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"306'9943","reported_seq":20395,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.655502+0000","last_change":"2023-06-10T21:35:13.080168+0000","last_active":"2023-06-10T21:35:24.655502+0000","last_peered":"2023-06-10T21:35:24.655502+0000","last_clean":"2023-06-10T21:35:24.655502+0000","last_became_active":"2023-06-10T21:35:13.008964+0000","last_became_peered":"2023-06-10T21:35:13.008964+0000","last_unstale":"2023-06-10T21:35:24.655502+0000","last_undegraded":"2023-06-10T21:35:24.655502+0000","last_fullsized":"2023-06-10T21:35:24.655502+0000","mapping_epoch":305,"log_start":"281'2570","ondisk_log_start":"281'2570","created":269,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9943,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5194,"num_bytes_recovered":306511872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":257,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086807+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:35:12.086807+0000","last_peered":"2023-06-10T21:35:12.086807+0000","last_clean":"2023-06-10T21:35:12.086807+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:35:12.086807+0000","last_undegraded":"2023-06-10T21:35:12.086807+0000","last_fullsized":"2023-06-10T21:35:12.086807+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086507+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:35:12.086507+0000","last_peered":"2023-06-10T21:35:12.086507+0000","last_clean":"2023-06-10T21:35:12.086507+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:35:12.086507+0000","last_undegraded":"2023-06-10T21:35:12.086507+0000","last_fullsized":"2023-06-10T21:35:12.086507+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"306'10256","reported_seq":10304,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.302603+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:35:24.302603+0000","last_peered":"2023-06-10T21:35:24.302603+0000","last_clean":"2023-06-10T21:35:24.302603+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:35:24.302603+0000","last_undegraded":"2023-06-10T21:35:24.302603+0000","last_fullsized":"2023-06-10T21:35:24.302603+0000","mapping_epoch":269,"log_start":"277'1100","ondisk_log_start":"277'1100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9156,"log_dups_size":0,"ondisk_log_size":9156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10256,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":80,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.087052+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:35:12.087052+0000","last_peered":"2023-06-10T21:35:12.087052+0000","last_clean":"2023-06-10T21:35:12.087052+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:35:12.087052+0000","last_undegraded":"2023-06-10T21:35:12.087052+0000","last_fullsized":"2023-06-10T21:35:12.087052+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":298,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059252+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:35:12.059252+0000","last_peered":"2023-06-10T21:35:12.059252+0000","last_clean":"2023-06-10T21:35:12.059252+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:35:12.059252+0000","last_undegraded":"2023-06-10T21:35:12.059252+0000","last_fullsized":"2023-06-10T21:35:12.059252+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"306'9996","reported_seq":10043,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.541484+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:35:21.541484+0000","last_peered":"2023-06-10T21:35:21.541484+0000","last_clean":"2023-06-10T21:35:21.541484+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:35:21.541484+0000","last_undegraded":"2023-06-10T21:35:21.541484+0000","last_fullsized":"2023-06-10T21:35:21.541484+0000","mapping_epoch":269,"log_start":"284'4200","ondisk_log_start":"284'4200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5796,"log_dups_size":0,"ondisk_log_size":5796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35520512,"num_objects":542,"num_object_clones":0,"num_object_copies":1626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9996,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":284,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059315+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:35:12.059315+0000","last_peered":"2023-06-10T21:35:12.059315+0000","last_clean":"2023-06-10T21:35:12.059315+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:35:12.059315+0000","last_undegraded":"2023-06-10T21:35:12.059315+0000","last_fullsized":"2023-06-10T21:35:12.059315+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086828+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:35:12.086828+0000","last_peered":"2023-06-10T21:35:12.086828+0000","last_clean":"2023-06-10T21:35:12.086828+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:35:12.086828+0000","last_undegraded":"2023-06-10T21:35:12.086828+0000","last_fullsized":"2023-06-10T21:35:12.086828+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"306'9964","reported_seq":10012,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.416763+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:35:21.416763+0000","last_peered":"2023-06-10T21:35:21.416763+0000","last_clean":"2023-06-10T21:35:21.416763+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:35:21.416763+0000","last_undegraded":"2023-06-10T21:35:21.416763+0000","last_fullsized":"2023-06-10T21:35:21.416763+0000","mapping_epoch":269,"log_start":"284'4100","ondisk_log_start":"284'4100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9964,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":260,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086458+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:35:12.086458+0000","last_peered":"2023-06-10T21:35:12.086458+0000","last_clean":"2023-06-10T21:35:12.086458+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:35:12.086458+0000","last_undegraded":"2023-06-10T21:35:12.086458+0000","last_fullsized":"2023-06-10T21:35:12.086458+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":216,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.087046+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:35:12.087046+0000","last_peered":"2023-06-10T21:35:12.087046+0000","last_clean":"2023-06-10T21:35:12.087046+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:35:12.087046+0000","last_undegraded":"2023-06-10T21:35:12.087046+0000","last_fullsized":"2023-06-10T21:35:12.087046+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"306'9989","reported_seq":10037,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.284146+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:35:24.284146+0000","last_peered":"2023-06-10T21:35:24.284146+0000","last_clean":"2023-06-10T21:35:24.284146+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:35:24.284146+0000","last_undegraded":"2023-06-10T21:35:24.284146+0000","last_fullsized":"2023-06-10T21:35:24.284146+0000","mapping_epoch":269,"log_start":"277'800","ondisk_log_start":"277'800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9189,"log_dups_size":0,"ondisk_log_size":9189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442648,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9989,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.080351+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:35:12.080351+0000","last_peered":"2023-06-10T21:35:12.080351+0000","last_clean":"2023-06-10T21:35:12.080351+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:35:12.080351+0000","last_undegraded":"2023-06-10T21:35:12.080351+0000","last_fullsized":"2023-06-10T21:35:12.080351+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":310,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086989+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:35:12.086989+0000","last_peered":"2023-06-10T21:35:12.086989+0000","last_clean":"2023-06-10T21:35:12.086989+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:35:12.086989+0000","last_undegraded":"2023-06-10T21:35:12.086989+0000","last_fullsized":"2023-06-10T21:35:12.086989+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076696+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:35:12.076696+0000","last_peered":"2023-06-10T21:35:12.076696+0000","last_clean":"2023-06-10T21:35:12.076696+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:35:12.076696+0000","last_undegraded":"2023-06-10T21:35:12.076696+0000","last_fullsized":"2023-06-10T21:35:12.076696+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"306'10005","reported_seq":10052,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.564522+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:35:21.564522+0000","last_peered":"2023-06-10T21:35:21.564522+0000","last_clean":"2023-06-10T21:35:21.564522+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:35:21.564522+0000","last_undegraded":"2023-06-10T21:35:21.564522+0000","last_fullsized":"2023-06-10T21:35:21.564522+0000","mapping_epoch":269,"log_start":"284'4200","ondisk_log_start":"284'4200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5805,"log_dups_size":0,"ondisk_log_size":5805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35586048,"num_objects":543,"num_object_clones":0,"num_object_copies":1629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10005,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":176,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076769+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:35:12.076769+0000","last_peered":"2023-06-10T21:35:12.076769+0000","last_clean":"2023-06-10T21:35:12.076769+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:35:12.076769+0000","last_undegraded":"2023-06-10T21:35:12.076769+0000","last_fullsized":"2023-06-10T21:35:12.076769+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":250,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073616+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:35:12.073616+0000","last_peered":"2023-06-10T21:35:12.073616+0000","last_clean":"2023-06-10T21:35:12.073616+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:35:12.073616+0000","last_undegraded":"2023-06-10T21:35:12.073616+0000","last_fullsized":"2023-06-10T21:35:12.073616+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":139,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059381+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:35:12.059381+0000","last_peered":"2023-06-10T21:35:12.059381+0000","last_clean":"2023-06-10T21:35:12.059381+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:35:12.059381+0000","last_undegraded":"2023-06-10T21:35:12.059381+0000","last_fullsized":"2023-06-10T21:35:12.059381+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"306'9954","reported_seq":10001,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.552472+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:35:21.552472+0000","last_peered":"2023-06-10T21:35:21.552472+0000","last_clean":"2023-06-10T21:35:21.552472+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:35:21.552472+0000","last_undegraded":"2023-06-10T21:35:21.552472+0000","last_fullsized":"2023-06-10T21:35:21.552472+0000","mapping_epoch":269,"log_start":"284'4100","ondisk_log_start":"284'4100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37748736,"num_objects":576,"num_object_clones":0,"num_object_copies":1728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9954,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":150,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059406+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:35:12.059406+0000","last_peered":"2023-06-10T21:35:12.059406+0000","last_clean":"2023-06-10T21:35:12.059406+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:35:12.059406+0000","last_undegraded":"2023-06-10T21:35:12.059406+0000","last_fullsized":"2023-06-10T21:35:12.059406+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.060020+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:35:12.060020+0000","last_peered":"2023-06-10T21:35:12.060020+0000","last_clean":"2023-06-10T21:35:12.060020+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:35:12.060020+0000","last_undegraded":"2023-06-10T21:35:12.060020+0000","last_fullsized":"2023-06-10T21:35:12.060020+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":199,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073654+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:35:12.073654+0000","last_peered":"2023-06-10T21:35:12.073654+0000","last_clean":"2023-06-10T21:35:12.073654+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:35:12.073654+0000","last_undegraded":"2023-06-10T21:35:12.073654+0000","last_fullsized":"2023-06-10T21:35:12.073654+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"306'9809","reported_seq":9856,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.577794+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:35:21.577794+0000","last_peered":"2023-06-10T21:35:21.577794+0000","last_clean":"2023-06-10T21:35:21.577794+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:35:21.577794+0000","last_undegraded":"2023-06-10T21:35:21.577794+0000","last_fullsized":"2023-06-10T21:35:21.577794+0000","mapping_epoch":269,"log_start":"284'4000","ondisk_log_start":"284'4000","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9809,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":330,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.061266+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:35:12.061266+0000","last_peered":"2023-06-10T21:35:12.061266+0000","last_clean":"2023-06-10T21:35:12.061266+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:35:12.061266+0000","last_undegraded":"2023-06-10T21:35:12.061266+0000","last_fullsized":"2023-06-10T21:35:12.061266+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":469,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067606+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:35:12.067606+0000","last_peered":"2023-06-10T21:35:12.067606+0000","last_clean":"2023-06-10T21:35:12.067606+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:35:12.067606+0000","last_undegraded":"2023-06-10T21:35:12.067606+0000","last_fullsized":"2023-06-10T21:35:12.067606+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":244,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.087224+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:35:12.087224+0000","last_peered":"2023-06-10T21:35:12.087224+0000","last_clean":"2023-06-10T21:35:12.087224+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:35:12.087224+0000","last_undegraded":"2023-06-10T21:35:12.087224+0000","last_fullsized":"2023-06-10T21:35:12.087224+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068089+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:35:12.068089+0000","last_peered":"2023-06-10T21:35:12.068089+0000","last_clean":"2023-06-10T21:35:12.068089+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:35:12.068089+0000","last_undegraded":"2023-06-10T21:35:12.068089+0000","last_fullsized":"2023-06-10T21:35:12.068089+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"306'10040","reported_seq":10087,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:21.564647+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:35:21.564647+0000","last_peered":"2023-06-10T21:35:21.564647+0000","last_clean":"2023-06-10T21:35:21.564647+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:35:21.564647+0000","last_undegraded":"2023-06-10T21:35:21.564647+0000","last_fullsized":"2023-06-10T21:35:21.564647+0000","mapping_epoch":269,"log_start":"284'4200","ondisk_log_start":"284'4200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10040,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":325,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067700+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:35:12.067700+0000","last_peered":"2023-06-10T21:35:12.067700+0000","last_clean":"2023-06-10T21:35:12.067700+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:35:12.067700+0000","last_undegraded":"2023-06-10T21:35:12.067700+0000","last_fullsized":"2023-06-10T21:35:12.067700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":145,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.074703+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:35:12.074703+0000","last_peered":"2023-06-10T21:35:12.074703+0000","last_clean":"2023-06-10T21:35:12.074703+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:35:12.074703+0000","last_undegraded":"2023-06-10T21:35:12.074703+0000","last_fullsized":"2023-06-10T21:35:12.074703+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":130,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076594+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:35:12.076594+0000","last_peered":"2023-06-10T21:35:12.076594+0000","last_clean":"2023-06-10T21:35:12.076594+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:35:12.076594+0000","last_undegraded":"2023-06-10T21:35:12.076594+0000","last_fullsized":"2023-06-10T21:35:12.076594+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"306'10056","reported_seq":10103,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.233037+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:35:24.233037+0000","last_peered":"2023-06-10T21:35:24.233037+0000","last_clean":"2023-06-10T21:35:24.233037+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:35:24.233037+0000","last_undegraded":"2023-06-10T21:35:24.233037+0000","last_fullsized":"2023-06-10T21:35:24.233037+0000","mapping_epoch":269,"log_start":"277'900","ondisk_log_start":"277'900","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9156,"log_dups_size":0,"ondisk_log_size":9156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31457280,"num_objects":480,"num_object_clones":0,"num_object_copies":1440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10056,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":310,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068034+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:35:12.068034+0000","last_peered":"2023-06-10T21:35:12.068034+0000","last_clean":"2023-06-10T21:35:12.068034+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:35:12.068034+0000","last_undegraded":"2023-06-10T21:35:12.068034+0000","last_fullsized":"2023-06-10T21:35:12.068034+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059460+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:35:12.059460+0000","last_peered":"2023-06-10T21:35:12.059460+0000","last_clean":"2023-06-10T21:35:12.059460+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:35:12.059460+0000","last_undegraded":"2023-06-10T21:35:12.059460+0000","last_fullsized":"2023-06-10T21:35:12.059460+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":126,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073764+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:35:12.073764+0000","last_peered":"2023-06-10T21:35:12.073764+0000","last_clean":"2023-06-10T21:35:12.073764+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:35:12.073764+0000","last_undegraded":"2023-06-10T21:35:12.073764+0000","last_fullsized":"2023-06-10T21:35:12.073764+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"306'9990","reported_seq":10035,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.577242+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:35:24.577242+0000","last_peered":"2023-06-10T21:35:24.577242+0000","last_clean":"2023-06-10T21:35:24.577242+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:35:24.577242+0000","last_undegraded":"2023-06-10T21:35:24.577242+0000","last_fullsized":"2023-06-10T21:35:24.577242+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26476544,"num_objects":404,"num_object_clones":0,"num_object_copies":1212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9990,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":283,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.087149+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:35:12.087149+0000","last_peered":"2023-06-10T21:35:12.087149+0000","last_clean":"2023-06-10T21:35:12.087149+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:35:12.087149+0000","last_undegraded":"2023-06-10T21:35:12.087149+0000","last_fullsized":"2023-06-10T21:35:12.087149+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":80,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086661+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:35:12.086661+0000","last_peered":"2023-06-10T21:35:12.086661+0000","last_clean":"2023-06-10T21:35:12.086661+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:35:12.086661+0000","last_undegraded":"2023-06-10T21:35:12.086661+0000","last_fullsized":"2023-06-10T21:35:12.086661+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":265,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086268+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:35:12.086268+0000","last_peered":"2023-06-10T21:35:12.086268+0000","last_clean":"2023-06-10T21:35:12.086268+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:35:12.086268+0000","last_undegraded":"2023-06-10T21:35:12.086268+0000","last_fullsized":"2023-06-10T21:35:12.086268+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"306'9925","reported_seq":9970,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-06-10T21:35:24.649829+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:35:24.649829+0000","last_peered":"2023-06-10T21:35:24.649829+0000","last_clean":"2023-06-10T21:35:24.649829+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:35:24.649829+0000","last_undegraded":"2023-06-10T21:35:24.649829+0000","last_fullsized":"2023-06-10T21:35:24.649829+0000","mapping_epoch":269,"log_start":"281'2600","ondisk_log_start":"281'2600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28246016,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9925,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.068019+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:35:12.068019+0000","last_peered":"2023-06-10T21:35:12.068019+0000","last_clean":"2023-06-10T21:35:12.068019+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:35:12.068019+0000","last_undegraded":"2023-06-10T21:35:12.068019+0000","last_fullsized":"2023-06-10T21:35:12.068019+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":134,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.076755+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:35:12.076755+0000","last_peered":"2023-06-10T21:35:12.076755+0000","last_clean":"2023-06-10T21:35:12.076755+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:35:12.076755+0000","last_undegraded":"2023-06-10T21:35:12.076755+0000","last_fullsized":"2023-06-10T21:35:12.076755+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.074498+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:35:12.074498+0000","last_peered":"2023-06-10T21:35:12.074498+0000","last_clean":"2023-06-10T21:35:12.074498+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:35:12.074498+0000","last_undegraded":"2023-06-10T21:35:12.074498+0000","last_fullsized":"2023-06-10T21:35:12.074498+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":129,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073862+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:35:12.073862+0000","last_peered":"2023-06-10T21:35:12.073862+0000","last_clean":"2023-06-10T21:35:12.073862+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:35:12.073862+0000","last_undegraded":"2023-06-10T21:35:12.073862+0000","last_fullsized":"2023-06-10T21:35:12.073862+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059604+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:35:12.059604+0000","last_peered":"2023-06-10T21:35:12.059604+0000","last_clean":"2023-06-10T21:35:12.059604+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:35:12.059604+0000","last_undegraded":"2023-06-10T21:35:12.059604+0000","last_fullsized":"2023-06-10T21:35:12.059604+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":31,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.067880+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:35:12.067880+0000","last_peered":"2023-06-10T21:35:12.067880+0000","last_clean":"2023-06-10T21:35:12.067880+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:35:12.067880+0000","last_undegraded":"2023-06-10T21:35:12.067880+0000","last_fullsized":"2023-06-10T21:35:12.067880+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":188,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086707+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:35:12.086707+0000","last_peered":"2023-06-10T21:35:12.086707+0000","last_clean":"2023-06-10T21:35:12.086707+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:35:12.086707+0000","last_undegraded":"2023-06-10T21:35:12.086707+0000","last_fullsized":"2023-06-10T21:35:12.086707+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086144+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:35:12.086144+0000","last_peered":"2023-06-10T21:35:12.086144+0000","last_clean":"2023-06-10T21:35:12.086144+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:35:12.086144+0000","last_undegraded":"2023-06-10T21:35:12.086144+0000","last_fullsized":"2023-06-10T21:35:12.086144+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":244,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.087134+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:35:12.087134+0000","last_peered":"2023-06-10T21:35:12.087134+0000","last_clean":"2023-06-10T21:35:12.087134+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:35:12.087134+0000","last_undegraded":"2023-06-10T21:35:12.087134+0000","last_fullsized":"2023-06-10T21:35:12.087134+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":290,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086500+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:35:12.086500+0000","last_peered":"2023-06-10T21:35:12.086500+0000","last_clean":"2023-06-10T21:35:12.086500+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:35:12.086500+0000","last_undegraded":"2023-06-10T21:35:12.086500+0000","last_fullsized":"2023-06-10T21:35:12.086500+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":293,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.086458+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:35:12.086458+0000","last_peered":"2023-06-10T21:35:12.086458+0000","last_clean":"2023-06-10T21:35:12.086458+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:35:12.086458+0000","last_undegraded":"2023-06-10T21:35:12.086458+0000","last_fullsized":"2023-06-10T21:35:12.086458+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":195,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.073748+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:35:12.073748+0000","last_peered":"2023-06-10T21:35:12.073748+0000","last_clean":"2023-06-10T21:35:12.073748+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:35:12.073748+0000","last_undegraded":"2023-06-10T21:35:12.073748+0000","last_fullsized":"2023-06-10T21:35:12.073748+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":39,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-06-10T21:35:12.059798+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:35:12.059798+0000","last_peered":"2023-06-10T21:35:12.059798+0000","last_clean":"2023-06-10T21:35:12.059798+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:35:12.059798+0000","last_undegraded":"2023-06-10T21:35:12.059798+0000","last_fullsized":"2023-06-10T21:35:12.059798+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":527499288,"num_objects":8050,"num_object_clones":0,"num_object_copies":24150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4477,"num_objects_misplaced":416,"num_objects_unfound":0,"num_objects_dirty":8050,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":160492,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5695,"num_bytes_recovered":339345408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2242129920,"data_stored":2242117704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111122,"ondisk_log_size":111122,"up":48,"acting":47,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430660,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":746432,"kb_used_data":153720,"kb_used_omap":0,"kb_used_meta":592704,"kb_avail":92998720,"statfs":{"total":95995035648,"available":95230689280,"internally_reserved":0,"allocated":157409280,"data_stored":156843452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606928896},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7930000000000001,"5min":4.2880000000000003,"15min":7.9009999999999998},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":26.878,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.3559999999999999,"5min":4.3019999999999996,"15min":8.1340000000000003},"min":{"1min":0.28599999999999998,"5min":0.23899999999999999,"15min":0.223},"max":{"1min":26.934999999999999,"5min":185.58000000000001,"15min":185.58000000000001},"last":11.587999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":3.2210000000000001,"15min":3.2930000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":4.4669999999999996,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.0649999999999999,"5min":3.6539999999999999,"15min":3.8570000000000002},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":8.1020000000000003,"5min":87.438000000000002,"15min":212.20599999999999},"last":3.5299999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:34:26 2023","interfaces":[{"interface":"back","average":{"1min":6.6399999999999997,"5min":6.4400000000000004,"15min":4.1849999999999996},"min":{"1min":0.32300000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":89.055999999999997,"5min":172.33600000000001,"15min":172.33600000000001},"last":4.125},{"interface":"front","average":{"1min":7.0510000000000002,"5min":6.3639999999999999,"15min":4.8490000000000002},"min":{"1min":0.38300000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":88.787000000000006,"5min":172.185,"15min":212.13399999999999},"last":4.2469999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2229999999999999,"5min":4.218,"15min":3.0720000000000001},"min":{"1min":0.375,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":23.611999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":1.8959999999999999},{"interface":"front","average":{"1min":2.6800000000000002,"5min":3.883,"15min":2.516},"min":{"1min":0.36399999999999999,"5min":0.31,"15min":0.31},"max":{"1min":23.591999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":3.556}]},{"osd":4,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.71499999999999997},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":0.93799999999999994,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.315},{"interface":"front","average":{"1min":0.441,"5min":0.42099999999999999,"15min":1.0289999999999999},"min":{"1min":0.309,"5min":0.245,"15min":0.245},"max":{"1min":0.81000000000000005,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.29199999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.51800000000000002,"15min":1.821},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":1.7869999999999999,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.44500000000000001,"15min":1.6160000000000001},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.69799999999999995,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.378}]},{"osd":6,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":1.077},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":2.516,"5min":3.2559999999999998,"15min":13.073},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48599999999999999,"15min":0.93700000000000006},"min":{"1min":0.34699999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73799999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.35199999999999998}]}]},{"osd":6,"up_from":0,"seq":180388626888,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566330,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1432236,"kb_used_data":774948,"kb_used_omap":4,"kb_used_meta":656891,"kb_avail":92312916,"statfs":{"total":95995035648,"available":94528425984,"internally_reserved":0,"allocated":793546752,"data_stored":792588641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":672657294},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":5.4809999999999999,"15min":10.98},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":13.372,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.218},{"interface":"front","average":{"1min":2.7629999999999999,"5min":5.3680000000000003,"15min":10.67},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.221},"max":{"1min":15.218,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.7679999999999998,"5min":8.8239999999999998,"15min":4.9909999999999997},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":18.041,"5min":191.101,"15min":191.101},"last":12.956},{"interface":"front","average":{"1min":3,"5min":9.1199999999999992,"15min":4.7560000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.218},"max":{"1min":11.964,"5min":190.87899999999999,"15min":190.87899999999999},"last":13.074999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":3.9079999999999999,"5min":4.7329999999999997,"15min":2.895},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.23799999999999999},"max":{"1min":15.130000000000001,"5min":80.584999999999994,"15min":80.584999999999994},"last":22.317},{"interface":"front","average":{"1min":3.593,"5min":5.3479999999999999,"15min":3.9039999999999999},"min":{"1min":0.35199999999999998,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":10.961,"5min":80.873999999999995,"15min":80.873999999999995},"last":23.111999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.355,"15min":1.548},"min":{"1min":0.35799999999999998,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":9.3659999999999997,"5min":10.036,"15min":16.059999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.589,"15min":1.452},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":8.8629999999999995,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.438}]},{"osd":5,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":2.0550000000000002,"15min":4.2169999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.035,"5min":10.035,"15min":383.83100000000002},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.3680000000000001,"15min":3.8050000000000002},"min":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":9.3160000000000007,"5min":10.116,"15min":383.84800000000001},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.069,"15min":2.1139999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":14.858000000000001},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.4990000000000001,"15min":1.7210000000000001},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":9.3849999999999998,"5min":10.066000000000001,"15min":13.48},"last":0.499}]},{"osd":7,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.0579999999999998,"15min":2.661},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":9.343,"5min":47.844000000000001,"15min":162.399},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.4369999999999998,"15min":2.476},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":8.8360000000000003,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.82899999999999996}]}]},{"osd":0,"up_from":244,"seq":1047972020307,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528380,"kb_used_data":266484,"kb_used_omap":10,"kb_used_meta":261749,"kb_avail":93216772,"statfs":{"total":95995035648,"available":95453974528,"internally_reserved":0,"allocated":272879616,"data_stored":271846069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":268031028},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":3.427,"5min":4.859,"15min":4.5590000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":22.202000000000002,"5min":139.68000000000001,"15min":139.68000000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":3.9529999999999998,"5min":4.5590000000000002,"15min":5.5609999999999999},"min":{"1min":0.22800000000000001,"5min":0.219,"15min":0.21199999999999999},"max":{"1min":19.751000000000001,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":10.782999999999999,"5min":8.8140000000000001,"15min":6.9059999999999997},"min":{"1min":0.33800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":110.34399999999999,"5min":205.10900000000001,"15min":205.10900000000001},"last":0.31},{"interface":"front","average":{"1min":9.6579999999999995,"5min":8.8309999999999995,"15min":6.3470000000000004},"min":{"1min":0.33700000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":103.56399999999999,"5min":208.34299999999999,"15min":208.34299999999999},"last":0.248}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":4.1340000000000003,"15min":5.569},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.934999999999999,"5min":111.745,"15min":111.745},"last":0.35599999999999998},{"interface":"front","average":{"1min":4.29,"5min":4.9589999999999996,"15min":6.5090000000000003},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":12.438000000000001,"5min":114.205,"15min":114.205},"last":1.177}]},{"osd":4,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":2.105,"15min":4.0990000000000002},"min":{"1min":0.32200000000000001,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":10.02,"5min":10.744999999999999,"15min":29.709},"last":0.59099999999999997},{"interface":"front","average":{"1min":3.7389999999999999,"5min":2.1589999999999998,"15min":3.569},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":18.856000000000002,"5min":18.856000000000002,"15min":18.856000000000002},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7709999999999999,"5min":2.1789999999999998,"15min":3.6499999999999999},"min":{"1min":0.38300000000000001,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":18.884,"5min":18.884,"15min":18.884},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.0990000000000002,"5min":2.105,"15min":3.1110000000000002},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":8.0079999999999991,"5min":12.083,"15min":19.846},"last":0.92200000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.1960000000000002,"15min":4.2759999999999998},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":10.053000000000001,"5min":10.771000000000001,"15min":29.806000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":4.2889999999999997,"5min":2.3029999999999999,"15min":3.7160000000000002},"min":{"1min":0.44600000000000001,"5min":0.373,"15min":0.373},"max":{"1min":18.812000000000001,"5min":18.812000000000001,"15min":18.812000000000001},"last":0.71199999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:34:49 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":2.3340000000000001,"15min":3.5030000000000001},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.9669999999999996,"5min":17.523,"15min":19.866},"last":0.86699999999999999},{"interface":"front","average":{"1min":2.5659999999999998,"5min":2.3490000000000002,"15min":4.3259999999999996},"min":{"1min":0.376,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.083,"5min":17.792999999999999,"15min":29.77},"last":0.65800000000000003}]}]},{"osd":2,"up_from":120,"seq":515396075719,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796712,"kb_used_data":301152,"kb_used_omap":7,"kb_used_meta":495544,"kb_avail":92948440,"statfs":{"total":95995035648,"available":95179202560,"internally_reserved":0,"allocated":308379648,"data_stored":307425392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":507437137},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:41 2023","interfaces":[{"interface":"back","average":{"1min":8.048,"5min":4.6749999999999998,"15min":4.694},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":70.510999999999996,"5min":70.510999999999996,"15min":70.510999999999996},"last":4.1509999999999998},{"interface":"front","average":{"1min":7.1420000000000003,"5min":4.734,"15min":5.1719999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":67.828999999999994,"5min":67.828999999999994,"15min":67.828999999999994},"last":3.9380000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":5.5350000000000001,"15min":3.1160000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.19},"max":{"1min":12.734,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":5.4580000000000002,"5min":4.8010000000000002,"15min":2.742},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.214},"max":{"1min":16.503,"5min":147.81299999999999,"15min":147.81299999999999},"last":4.1740000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:35:07 2023","interfaces":[{"interface":"back","average":{"1min":13.648,"5min":6.2619999999999996,"15min":5.5810000000000004},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":97.296000000000006,"5min":138.03299999999999,"15min":138.03299999999999},"last":5.0549999999999997},{"interface":"front","average":{"1min":12.912000000000001,"5min":6.1219999999999999,"15min":6.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":97.308999999999997,"5min":138.095,"15min":138.095},"last":4.9740000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":1.744,"15min":2.8359999999999999},"min":{"1min":0.34599999999999997,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":4.5599999999999996,"5min":8.0969999999999995,"15min":224.00999999999999},"last":3.883},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.8620000000000001,"15min":3.2480000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":8.2739999999999991,"5min":29.486999999999998,"15min":223.28399999999999},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":1.819,"15min":1.5960000000000001},"min":{"1min":0.41199999999999998,"5min":0.34399999999999997,"15min":0.29699999999999999},"max":{"1min":10.554,"5min":10.554,"15min":14.180999999999999},"last":4.0919999999999996},{"interface":"front","average":{"1min":1.548,"5min":1.7849999999999999,"15min":1.7529999999999999},"min":{"1min":0.38200000000000001,"5min":0.371,"15min":0.32900000000000001},"max":{"1min":4.5860000000000003,"5min":8.1989999999999998,"15min":16.125},"last":3.8050000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.8560000000000001,"15min":1.825},"min":{"1min":0.45700000000000002,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":4.6130000000000004,"5min":8.1690000000000005,"15min":16.183},"last":3.8450000000000002},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.8650000000000002,"15min":2.2559999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.3870000000000005,"5min":29.538,"15min":29.538},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:34:34 2023","interfaces":[{"interface":"back","average":{"1min":5.5590000000000002,"5min":3.544,"15min":2.5529999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":41.164999999999999,"5min":73.445999999999998,"15min":113.913},"last":4.1260000000000003},{"interface":"front","average":{"1min":4.0819999999999999,"5min":3.4860000000000002,"15min":2.7999999999999998},"min":{"1min":0.43099999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":41.219999999999999,"5min":73.475999999999999,"15min":113.937},"last":3.9089999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824078,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":863928,"kb_used_data":617776,"kb_used_omap":7,"kb_used_meta":246136,"kb_avail":92881224,"statfs":{"total":95995035648,"available":95110373376,"internally_reserved":0,"allocated":632602624,"data_stored":631573177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":252043283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":8.3510000000000009,"5min":4.8239999999999998,"15min":5.3070000000000004},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":105.09,"5min":105.09,"15min":105.09},"last":0.20300000000000001},{"interface":"front","average":{"1min":8.1940000000000008,"5min":4.4980000000000002,"15min":5.0540000000000003},"min":{"1min":0.51200000000000001,"5min":0.246,"15min":0.216},"max":{"1min":99.852999999999994,"5min":99.852999999999994,"15min":99.852999999999994},"last":1.2170000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0459999999999994,"5min":5.194,"15min":4.157},"min":{"1min":0.25800000000000001,"5min":0.248,"15min":0.23100000000000001},"max":{"1min":94.766000000000005,"5min":94.766000000000005,"15min":94.766000000000005},"last":1.5840000000000001},{"interface":"front","average":{"1min":10.343,"5min":4.6310000000000002,"15min":3.867},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":94.691000000000003,"5min":94.691000000000003,"15min":94.691000000000003},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.5120000000000005,"5min":3.992,"15min":9.9450000000000003},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":100.676,"5min":100.676,"15min":100.676},"last":1.1919999999999999},{"interface":"front","average":{"1min":10.388,"5min":5.7539999999999996,"15min":11.48},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":103.032,"5min":103.032,"15min":103.032},"last":1.8160000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.575,"15min":1.4410000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":5.0170000000000003,"5min":13.417,"15min":13.417},"last":1.089},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.8260000000000001,"15min":2.7679999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.616,"5min":19.728000000000002,"15min":19.728000000000002},"last":1.637}]},{"osd":5,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5569999999999999,"15min":1.4470000000000001},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.71,"15min":1.5229999999999999},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0629999999999997,"5min":13.470000000000001,"15min":13.470000000000001},"last":1.147}]},{"osd":6,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":2.847,"15min":2.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.537000000000001,"5min":19.805,"15min":19.805},"last":1.669},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6140000000000001,"15min":1.498},"min":{"1min":0.433,"5min":0.433,"15min":0.39100000000000001},"max":{"1min":9.7289999999999992,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.524,"15min":3.0259999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.648999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":1.6950000000000001},{"interface":"front","average":{"1min":2.3050000000000002,"5min":2.2850000000000001,"15min":1.704},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.36699999999999999},"max":{"1min":9.8239999999999998,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.44}]}]},{"osd":4,"up_from":30,"seq":128849019344,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":718204,"kb_used_data":123892,"kb_used_omap":0,"kb_used_meta":594304,"kb_avail":93026948,"statfs":{"total":95995035648,"available":95259594752,"internally_reserved":0,"allocated":126865408,"data_stored":126302148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608567296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:34:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2809999999999997,"5min":3.9420000000000002,"15min":8.4320000000000004},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":60.923999999999999,"5min":234.203,"15min":234.203},"last":37.503},{"interface":"front","average":{"1min":5.1479999999999997,"5min":4.9189999999999996,"15min":8.266},"min":{"1min":0.28100000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":64.399000000000001,"5min":234.19,"15min":234.19},"last":37.399999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:34:25 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.782,"15min":2.9159999999999999},"min":{"1min":0.27000000000000002,"5min":0.26100000000000001,"15min":0.25900000000000001},"max":{"1min":3.7170000000000001,"5min":13.461,"15min":182.035},"last":0.24299999999999999},{"interface":"front","average":{"1min":3.5049999999999999,"5min":2.1360000000000001,"15min":2.8359999999999999},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":10.151,"5min":10.255000000000001,"15min":181.73099999999999},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":22.809999999999999,"5min":6.6989999999999998,"15min":3.294},"min":{"1min":0.495,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":255.386,"5min":255.386,"15min":255.386},"last":0.25600000000000001},{"interface":"front","average":{"1min":24.347999999999999,"5min":6.8520000000000003,"15min":3.2949999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":255.29599999999999,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":17.739000000000001,"5min":7.5629999999999997,"15min":4.1150000000000002},"min":{"1min":0.57699999999999996,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":102.94,"5min":234.91900000000001,"15min":234.91900000000001},"last":36.366999999999997},{"interface":"front","average":{"1min":12.029999999999999,"5min":8.1590000000000007,"15min":4.0819999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":90.489999999999995,"5min":235.05099999999999,"15min":235.05099999999999},"last":36.564}]},{"osd":5,"last update":"Sat Jun 10 21:34:33 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":1.0009999999999999},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.58699999999999997,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.46600000000000003,"15min":1.016},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.67900000000000005,"5min":5.25,"15min":11.904999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.43099999999999999,"15min":1.0249999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.53900000000000003,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.46600000000000003,"15min":1.0900000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.59499999999999997,"5min":1.71,"15min":13.845000000000001},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.77500000000000002,"15min":1.2789999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.38},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.79600000000000004,"15min":1.2210000000000001},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.57199999999999995,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.42299999999999999}]}]},{"osd":5,"up_from":0,"seq":154618823116,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":268500992,"data_stored":268500992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":785780736,"data_stored":785776664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":302780416,"data_stored":302776344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":617349120,"data_stored":617349120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":117575680,"data_stored":117571608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":150142976,"data_stored":150142976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:35:29.616 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:35:29.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:29.646 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:29.694 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:29.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:29.799 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:29.826 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:29.869 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:29.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:29.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:29.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:30.056 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:30.075 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:30.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:30 smithi064 ceph-mon[180297]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 4477/16302 objects degraded (27.463%); 213/16302 objects misplaced (1.307%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:30.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:30 smithi064 ceph-mon[186706]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 4477/16302 objects degraded (27.463%); 213/16302 objects misplaced (1.307%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:30.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:30.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:30.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:30.340 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:30 smithi152 ceph-mon[156819]: pgmap v1131: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 525 op/s; 4477/16302 objects degraded (27.463%); 213/16302 objects misplaced (1.307%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:30.343 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:30.391 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:30.397 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:35:30.397 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:35:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:30.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:30.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:30.525 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:30.569 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:30.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:30.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:30.703 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:30.749 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:30.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:30.861 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:30.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:31.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:35:31.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4477/16302 objects degraded (27.463%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:31.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4477/16302 objects degraded (27.463%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:31.316 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:31.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:31.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:31.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4477/16302 objects degraded (27.463%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:31.449 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:31.500 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:31.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:31.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:31.748 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:32.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:32 smithi064 ceph-mon[180297]: pgmap v1132: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 278 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 4477/13395 objects degraded (33.423%); 213/13395 objects misplaced (1.590%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:32.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:32 smithi064 ceph-mon[186706]: pgmap v1132: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 278 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 4477/13395 objects degraded (33.423%); 213/13395 objects misplaced (1.590%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:32.331 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:32.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:32.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:32.441 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:32.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:32 smithi152 ceph-mon[156819]: pgmap v1132: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 278 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 4477/13395 objects degraded (33.423%); 213/13395 objects misplaced (1.590%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:35:32.468 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:32.515 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:32.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:32.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:32.623 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:32.649 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:32.692 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:32.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:32.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:32.827 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:32.868 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:32.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:32.977 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:33.003 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:33.050 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:33.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:32] "GET /metrics HTTP/1.1" 200 224065 "" "Prometheus/2.33.4" 2023-06-10T21:35:33.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:33.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:33.405 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:33.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:35:33.449 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:33.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:33.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:33.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:33.558 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:33.579 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:33.604 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:33.607 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:33.652 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:33.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:33.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:33.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:33.872 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:34.045 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:34.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:34.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:34.237 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:34.303 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:34 smithi064 ceph-mon[180297]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 246 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 4477/11838 objects degraded (37.819%); 7/11838 objects misplaced (0.059%); 2.6 MiB/s, 40 objects/s recovering 2023-06-10T21:35:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:34 smithi064 ceph-mon[186706]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 246 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 4477/11838 objects degraded (37.819%); 7/11838 objects misplaced (0.059%); 2.6 MiB/s, 40 objects/s recovering 2023-06-10T21:35:34.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:34.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:34 smithi152 ceph-mon[156819]: pgmap v1133: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 246 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 4477/11838 objects degraded (37.819%); 7/11838 objects misplaced (0.059%); 2.6 MiB/s, 40 objects/s recovering 2023-06-10T21:35:34.748 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:35:34.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:34.786 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:34.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:35:34.833 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:34.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:34.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:34.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:35:34.928 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:35:34.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:34.949 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:35:34.993 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:35:35.030 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:35:35.066 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:35.077 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:35:35.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.107 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.108 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.108 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.108 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:35:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1201414935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1201414935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1201414935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:36 smithi064 ceph-mon[180297]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 184 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 4477/8880 objects degraded (50.417%); 7/8880 objects misplaced (0.079%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4477/11838 objects degraded (37.819%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:36 smithi064 ceph-mon[186706]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 184 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 4477/8880 objects degraded (50.417%); 7/8880 objects misplaced (0.079%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4477/11838 objects degraded (37.819%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:36 smithi152 ceph-mon[156819]: pgmap v1134: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 184 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 4477/8880 objects degraded (50.417%); 7/8880 objects misplaced (0.079%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:35:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4477/11838 objects degraded (37.819%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:36.768 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:36.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:36.877 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:36.952 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:37.706 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:37.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:37.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:37.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:37.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:38.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:38.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:38.171 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:37 smithi064 ceph-mon[180297]: pgmap v1135: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 4477/6063 objects degraded (73.841%); 7/6063 objects misplaced (0.115%); 2.1 MiB/s, 34 objects/s recovering 2023-06-10T21:35:38.172 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:37 smithi064 ceph-mon[186706]: pgmap v1135: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 4477/6063 objects degraded (73.841%); 7/6063 objects misplaced (0.115%); 2.1 MiB/s, 34 objects/s recovering 2023-06-10T21:35:38.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:38.203 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:38.248 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:38.255 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:37 smithi152 ceph-mon[156819]: pgmap v1135: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 125 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 4477/6063 objects degraded (73.841%); 7/6063 objects misplaced (0.115%); 2.1 MiB/s, 34 objects/s recovering 2023-06-10T21:35:38.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:38.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:38.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:38.453 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:38.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:38.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:38.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:38.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:38.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:38.791 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 84270 objects 2023-06-10T21:35:38.791 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :165.594 2023-06-10T21:35:38.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:38.966 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:35:38.966 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4477/6063 objects degraded (73.841%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1612,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":307,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":2021,"data_bytes":131405734,"bytes_used":4013821952,"bytes_avail":571956391936,"bytes_total":575970213888,"degraded_objects":4477,"degraded_total":6063,"degraded_ratio":0.73841332673593929,"misplaced_objects":7,"misplaced_total":6063,"misplaced_ratio":0.0011545439551377205,"recovering_objects_per_sec":34,"recovering_bytes_per_sec":2232038,"recovering_keys_per_sec":0,"num_objects_recovered":409,"num_bytes_recovered":26804224,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":504},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":25,"modified":"2023-06-10T21:34:59.873283+0000","services":{}},"progress_events":{"08ad03ac-ce84-41c1-8854-b18550dc6087":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 1s)","progress":0.98876404762268066,"add_to_ceph_s":true},"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:35:39.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2923861216' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:35:39.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2923861216' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:35:39.326 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-06-10T21:35:39.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-06-10T21:35:39.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2923861216' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:35:39.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:35:39.982 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:39.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:39.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:40.084 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:39 smithi064 ceph-mon[186706]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 32 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 4275/1578 objects degraded (270.913%); 2.2 MiB/s, 56 objects/s recovering 2023-06-10T21:35:40.084 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:39 smithi064 ceph-mon[180297]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 32 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 4275/1578 objects degraded (270.913%); 2.2 MiB/s, 56 objects/s recovering 2023-06-10T21:35:40.086 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:40.112 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:40.153 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:40.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:40.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:40.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:40.285 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:39 smithi152 ceph-mon[156819]: pgmap v1136: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 32 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 542 op/s; 4275/1578 objects degraded (270.913%); 2.2 MiB/s, 56 objects/s recovering 2023-06-10T21:35:40.287 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:40.331 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:40.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:40.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:40.467 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:41.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:40 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4275/1578 objects degraded (270.913%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:41.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4275/1578 objects degraded (270.913%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:41.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4275/1578 objects degraded (270.913%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:41.635 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:35:41.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:35:41.682 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:41.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:41.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:41.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:41.812 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:41.854 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:41.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:41.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:41.994 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:42.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:42 smithi064 ceph-mon[186706]: pgmap v1137: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 4275/222 objects degraded (1925.676%); 1.1 MiB/s, 39 objects/s recovering 2023-06-10T21:35:42.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:42 smithi064 ceph-mon[180297]: pgmap v1137: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 4275/222 objects degraded (1925.676%); 1.1 MiB/s, 39 objects/s recovering 2023-06-10T21:35:42.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:42 smithi152 ceph-mon[156819]: pgmap v1137: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 3.7 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 446 op/s; 4275/222 objects degraded (1925.676%); 1.1 MiB/s, 39 objects/s recovering 2023-06-10T21:35:42.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:42.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:42.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:42.445 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:42.488 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:42.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:42.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:42.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:42.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:42.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:42.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:42.779 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:42.805 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:42.847 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:42.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:42.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:42.954 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:42.980 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:43.021 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:35:43.023 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:43.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:43.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:42] "GET /metrics HTTP/1.1" 200 224063 "" "Prometheus/2.33.4" 2023-06-10T21:35:43.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2622001826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:43 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2622001826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:43 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.162 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:43.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2622001826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:43 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:35:43.210 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:43.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:43.314 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:43.341 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:43.382 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:43.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:43.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:43.521 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:44.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:35:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[180297]: failed to encode map e308 with expected crc 2023-06-10T21:35:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:44.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[180297]: osdmap e308: 8 total, 8 up, 6 in 2023-06-10T21:35:44.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[180297]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:44.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:35:44.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[186706]: failed to encode map e308 with expected crc 2023-06-10T21:35:44.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:44.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[186706]: osdmap e308: 8 total, 8 up, 6 in 2023-06-10T21:35:44.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:44 smithi064 ceph-mon[186706]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:44.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:44 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:35:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:44 smithi152 ceph-mon[156819]: failed to encode map e308 with expected crc 2023-06-10T21:35:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:35:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:44 smithi152 ceph-mon[156819]: osdmap e308: 8 total, 8 up, 6 in 2023-06-10T21:35:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:44 smithi152 ceph-mon[156819]: pgmap v1139: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:45.319 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:45.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:45 smithi064 ceph-mon[186706]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:45 smithi064 ceph-mon[186706]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:45 smithi064 ceph-mon[180297]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:45 smithi064 ceph-mon[180297]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:45.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:45.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:45 smithi152 ceph-mon[156819]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:45 smithi152 ceph-mon[156819]: failed to encode map e308 with expected crc 2023-06-10T21:35:45.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:45.458 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:45.496 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:45.538 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:45.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:45.644 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:45.676 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:45.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:45.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:45.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:45.831 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:45.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:45.902 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:45.963 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:45.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:45.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:46.067 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:46.093 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:46.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[180297]: failed to encode map e308 with expected crc 2023-06-10T21:35:46.113 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[180297]: pgmap v1140: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 293 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:46.114 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 3641/72 objects degraded (5056.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:46.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[186706]: failed to encode map e308 with expected crc 2023-06-10T21:35:46.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[186706]: pgmap v1140: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 293 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:46.114 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 3641/72 objects degraded (5056.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:46.135 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:46.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:46.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:46.268 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:46.287 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:35:46.288 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:35:46.288 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:35:46.289 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:35:46.289 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:35:46.308 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:46.311 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:46 smithi152 ceph-mon[156819]: failed to encode map e308 with expected crc 2023-06-10T21:35:46.311 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:46 smithi152 ceph-mon[156819]: pgmap v1140: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 293 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:46.311 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 3641/72 objects degraded (5056.944%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:46.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:46.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:46.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:35:46.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:46.481 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:35:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:46.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:35:46.587 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:46.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:46.739 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:46.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:46.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:46.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:46.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:47.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:35:48.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:47 smithi064 ceph-mon[186706]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:48.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:47 smithi064 ceph-mon[180297]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:47 smithi152 ceph-mon[156819]: pgmap v1141: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 3641/72 objects degraded (5056.944%); 45 KiB/s, 90 objects/s recovering 2023-06-10T21:35:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:49.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:49.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:35:49.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:35:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:50 smithi152 ceph-mon[156819]: pgmap v1142: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 50 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:50.583 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:50 smithi064 ceph-mon[180297]: pgmap v1142: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 50 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:50.584 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:50 smithi064 ceph-mon[186706]: pgmap v1142: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 50 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:51.396 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:51.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 2832/72 objects degraded (3933.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:51.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 2832/72 objects degraded (3933.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 2832/72 objects degraded (3933.333%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:51.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:51.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:35:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:52 smithi152 ceph-mon[156819]: pgmap v1143: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:52 smithi064 ceph-mon[180297]: pgmap v1143: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:52.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:52 smithi064 ceph-mon[186706]: pgmap v1143: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 2832/72 objects degraded (3933.333%); 0 B/s, 144 objects/s recovering 2023-06-10T21:35:52.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:52.968 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:35:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:35:52] "GET /metrics HTTP/1.1" 200 224034 "" "Prometheus/2.33.4" 2023-06-10T21:35:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:52.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:53.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:53.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:53.159 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:53.531 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:53.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:53.638 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:53.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:53.706 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:53.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:53.811 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:53.838 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:53.882 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:53.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:53.936 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:35:53.994 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:54.019 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:54.063 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:54.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:54.078 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:53 smithi064 ceph-mon[186706]: pgmap v1144: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 1774/72 objects degraded (2463.889%); 0 B/s, 229 objects/s recovering 2023-06-10T21:35:54.078 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:53 smithi064 ceph-mon[180297]: pgmap v1144: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 1774/72 objects degraded (2463.889%); 0 B/s, 229 objects/s recovering 2023-06-10T21:35:54.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:54.205 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:54.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:54.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:54.435 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:35:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:53 smithi152 ceph-mon[156819]: pgmap v1144: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 1774/72 objects degraded (2463.889%); 0 B/s, 229 objects/s recovering 2023-06-10T21:35:54.472 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:35:54.511 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:35:54.514 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:35:54.548 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:35:54.558 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:54.584 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:35:54.612 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:35:54.634 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:35:54.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:35:54.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.654 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.655 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.656 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:35:54.696 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:35:54.697 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:35:55.009 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4175162609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:55.010 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[186706]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:55.010 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:35:55.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4175162609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:55.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[180297]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:55.010 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:35:55.014 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_6' removed 2023-06-10T21:35:55.332 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:35:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:55.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:35:55.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4175162609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:35:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:55 smithi152 ceph-mon[156819]: from='client.56266 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:35:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:35:55.468 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:56.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:35:56.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[186706]: osdmap e309: 8 total, 8 up, 6 in 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[186706]: failed to encode map e309 with expected crc 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[186706]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[180297]: osdmap e309: 8 total, 8 up, 6 in 2023-06-10T21:35:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[180297]: failed to encode map e309 with expected crc 2023-06-10T21:35:56.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[180297]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:56.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/655315972' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:35:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:56 smithi152 ceph-mon[156819]: osdmap e309: 8 total, 8 up, 6 in 2023-06-10T21:35:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:56 smithi152 ceph-mon[156819]: failed to encode map e309 with expected crc 2023-06-10T21:35:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:56 smithi152 ceph-mon[156819]: pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:35:56.670 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:56.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:56.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:56.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:56.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:56.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:35:56.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:56.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:35:56.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:35:56.993 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:57.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:57 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized) 2023-06-10T21:35:57.396 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:57 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:35:57.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:57 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized) 2023-06-10T21:35:57.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:57 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:35:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:57 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1774/72 objects degraded (2463.889%), 1 pg degraded, 1 pg undersized) 2023-06-10T21:35:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:57 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:35:57.595 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:35:57.596 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:35:57.596 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-06-10T21:35:57.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_7 16 2023-06-10T21:35:58.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[186706]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[186706]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[186706]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[186706]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[180297]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[180297]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[180297]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:58 smithi064 ceph-mon[180297]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:58.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:58 smithi152 ceph-mon[156819]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:58 smithi152 ceph-mon[156819]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:58 smithi152 ceph-mon[156819]: failed to encode map e309 with expected crc 2023-06-10T21:35:58.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:58 smithi152 ceph-mon[156819]: pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:35:58.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:35:58.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:58.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:35:58.550 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:35:58.582 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:58.630 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:35:58.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:58.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:35:58.736 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:35:58.766 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:58.810 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:35:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:58.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:35:58.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:35:58.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:59.062 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:35:59.224 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:35:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:35:59.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:35:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:35:59.321 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1144,"stamp":"2023-06-10T21:35:53.915351+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":168803,"num_write_kb":5395727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8914,"num_bytes_recovered":368385746,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111367,"ondisk_log_size":111367,"up":267,"acting":266,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3474868,"kb_used_data":621124,"kb_used_omap":30,"kb_used_meta":2853665,"kb_avail":558996044,"statfs":{"total":575970213888,"available":572411949056,"internally_reserved":0,"allocated":636030976,"data_stored":630908918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2922152998},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-3276800,"num_objects":-50,"num_object_clones":0,"num_object_copies":-150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":50,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2501,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.902450"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":301,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.193333+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:35:25.193333+0000","last_peered":"2023-06-10T21:35:25.193333+0000","last_clean":"2023-06-10T21:35:25.193333+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:35:25.193333+0000","last_undegraded":"2023-06-10T21:35:25.193333+0000","last_fullsized":"2023-06-10T21:35:25.193333+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":242,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.229289+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:35:25.229289+0000","last_peered":"2023-06-10T21:35:25.229289+0000","last_clean":"2023-06-10T21:35:25.229289+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:35:25.229289+0000","last_undegraded":"2023-06-10T21:35:25.229289+0000","last_fullsized":"2023-06-10T21:35:25.229289+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":217,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226892+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:35:25.226892+0000","last_peered":"2023-06-10T21:35:25.226892+0000","last_clean":"2023-06-10T21:35:25.226892+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:35:25.226892+0000","last_undegraded":"2023-06-10T21:35:25.226892+0000","last_fullsized":"2023-06-10T21:35:25.226892+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":165,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.205395+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:35:25.205395+0000","last_peered":"2023-06-10T21:35:25.205395+0000","last_clean":"2023-06-10T21:35:25.205395+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:35:25.205395+0000","last_undegraded":"2023-06-10T21:35:25.205395+0000","last_fullsized":"2023-06-10T21:35:25.205395+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":282,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.227220+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:35:25.227220+0000","last_peered":"2023-06-10T21:35:25.227220+0000","last_clean":"2023-06-10T21:35:25.227220+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:35:25.227220+0000","last_undegraded":"2023-06-10T21:35:25.227220+0000","last_fullsized":"2023-06-10T21:35:25.227220+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":275,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203839+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:35:25.203839+0000","last_peered":"2023-06-10T21:35:25.203839+0000","last_clean":"2023-06-10T21:35:25.203839+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:35:25.203839+0000","last_undegraded":"2023-06-10T21:35:25.203839+0000","last_fullsized":"2023-06-10T21:35:25.203839+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-06-10T21:35:44.076116+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:35:44.076116+0000","last_peered":"2023-06-10T21:35:44.076116+0000","last_clean":"2023-06-10T21:35:44.076116+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:35:44.076116+0000","last_undegraded":"2023-06-10T21:35:44.076116+0000","last_fullsized":"2023-06-10T21:35:44.076116+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":256,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226767+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:35:25.226767+0000","last_peered":"2023-06-10T21:35:25.226767+0000","last_clean":"2023-06-10T21:35:25.226767+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:35:25.226767+0000","last_undegraded":"2023-06-10T21:35:25.226767+0000","last_fullsized":"2023-06-10T21:35:25.226767+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":29,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210881+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:35:25.210881+0000","last_peered":"2023-06-10T21:35:25.210881+0000","last_clean":"2023-06-10T21:35:25.210881+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:35:25.210881+0000","last_undegraded":"2023-06-10T21:35:25.210881+0000","last_fullsized":"2023-06-10T21:35:25.210881+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":31,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226378+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:35:25.226378+0000","last_peered":"2023-06-10T21:35:25.226378+0000","last_clean":"2023-06-10T21:35:25.226378+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:35:25.226378+0000","last_undegraded":"2023-06-10T21:35:25.226378+0000","last_fullsized":"2023-06-10T21:35:25.226378+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":35,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210748+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:35:25.210748+0000","last_peered":"2023-06-10T21:35:25.210748+0000","last_clean":"2023-06-10T21:35:25.210748+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:35:25.210748+0000","last_undegraded":"2023-06-10T21:35:25.210748+0000","last_fullsized":"2023-06-10T21:35:25.210748+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":271,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:39.349335+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:35:39.349335+0000","last_peered":"2023-06-10T21:35:39.349335+0000","last_clean":"2023-06-10T21:35:39.349335+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:35:39.349335+0000","last_undegraded":"2023-06-10T21:35:39.349335+0000","last_fullsized":"2023-06-10T21:35:39.349335+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":178,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210829+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:35:25.210829+0000","last_peered":"2023-06-10T21:35:25.210829+0000","last_clean":"2023-06-10T21:35:25.210829+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:35:25.210829+0000","last_undegraded":"2023-06-10T21:35:25.210829+0000","last_fullsized":"2023-06-10T21:35:25.210829+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":39,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210664+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:35:25.210664+0000","last_peered":"2023-06-10T21:35:25.210664+0000","last_clean":"2023-06-10T21:35:25.210664+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:35:25.210664+0000","last_undegraded":"2023-06-10T21:35:25.210664+0000","last_fullsized":"2023-06-10T21:35:25.210664+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":307,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210696+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:35:25.210696+0000","last_peered":"2023-06-10T21:35:25.210696+0000","last_clean":"2023-06-10T21:35:25.210696+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:35:25.210696+0000","last_undegraded":"2023-06-10T21:35:25.210696+0000","last_fullsized":"2023-06-10T21:35:25.210696+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":242,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.225959+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:35:25.225959+0000","last_peered":"2023-06-10T21:35:25.225959+0000","last_clean":"2023-06-10T21:35:25.225959+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:35:25.225959+0000","last_undegraded":"2023-06-10T21:35:25.225959+0000","last_fullsized":"2023-06-10T21:35:25.225959+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":165,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:33.215085+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:35:33.215085+0000","last_peered":"2023-06-10T21:35:33.215085+0000","last_clean":"2023-06-10T21:35:33.215085+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:35:33.215085+0000","last_undegraded":"2023-06-10T21:35:33.215085+0000","last_fullsized":"2023-06-10T21:35:33.215085+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":236,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210932+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:35:25.210932+0000","last_peered":"2023-06-10T21:35:25.210932+0000","last_clean":"2023-06-10T21:35:25.210932+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:35:25.210932+0000","last_undegraded":"2023-06-10T21:35:25.210932+0000","last_fullsized":"2023-06-10T21:35:25.210932+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":284,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210493+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:35:25.210493+0000","last_peered":"2023-06-10T21:35:25.210493+0000","last_clean":"2023-06-10T21:35:25.210493+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:35:25.210493+0000","last_undegraded":"2023-06-10T21:35:25.210493+0000","last_fullsized":"2023-06-10T21:35:25.210493+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-06-10T21:35:44.075324+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:35:44.075324+0000","last_peered":"2023-06-10T21:35:44.075324+0000","last_clean":"2023-06-10T21:35:44.075324+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:35:44.075324+0000","last_undegraded":"2023-06-10T21:35:44.075324+0000","last_fullsized":"2023-06-10T21:35:44.075324+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":284,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:33.215022+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:35:33.215022+0000","last_peered":"2023-06-10T21:35:33.215022+0000","last_clean":"2023-06-10T21:35:33.215022+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:35:33.215022+0000","last_undegraded":"2023-06-10T21:35:33.215022+0000","last_fullsized":"2023-06-10T21:35:33.215022+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":260,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226288+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:35:25.226288+0000","last_peered":"2023-06-10T21:35:25.226288+0000","last_clean":"2023-06-10T21:35:25.226288+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:35:25.226288+0000","last_undegraded":"2023-06-10T21:35:25.226288+0000","last_fullsized":"2023-06-10T21:35:25.226288+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"307'10544","reported_seq":10594,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.740101+0000","last_change":"2023-06-10T21:31:13.308660+0000","last_active":"2023-06-10T21:35:38.740101+0000","last_peered":"2023-06-10T21:35:38.740101+0000","last_clean":"2023-06-10T21:35:38.740101+0000","last_became_active":"2023-06-10T21:31:13.308350+0000","last_became_peered":"2023-06-10T21:31:13.308350+0000","last_unstale":"2023-06-10T21:35:38.740101+0000","last_undegraded":"2023-06-10T21:35:38.740101+0000","last_fullsized":"2023-06-10T21:35:38.740101+0000","mapping_epoch":269,"log_start":"284'4100","ondisk_log_start":"284'4100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:26:44.841607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10544,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":33,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.211086+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:35:25.211086+0000","last_peered":"2023-06-10T21:35:25.211086+0000","last_clean":"2023-06-10T21:35:25.211086+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:35:25.211086+0000","last_undegraded":"2023-06-10T21:35:25.211086+0000","last_fullsized":"2023-06-10T21:35:25.211086+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":29,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210329+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:35:25.210329+0000","last_peered":"2023-06-10T21:35:25.210329+0000","last_clean":"2023-06-10T21:35:25.210329+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:35:25.210329+0000","last_undegraded":"2023-06-10T21:35:25.210329+0000","last_fullsized":"2023-06-10T21:35:25.210329+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.6","version":"307'10596","reported_seq":10646,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.771974+0000","last_change":"2023-06-10T21:31:13.312339+0000","last_active":"2023-06-10T21:35:38.771974+0000","last_peered":"2023-06-10T21:35:38.771974+0000","last_clean":"2023-06-10T21:35:38.771974+0000","last_became_active":"2023-06-10T21:31:13.312053+0000","last_became_peered":"2023-06-10T21:31:13.312053+0000","last_unstale":"2023-06-10T21:35:38.771974+0000","last_undegraded":"2023-06-10T21:35:38.771974+0000","last_fullsized":"2023-06-10T21:35:38.771974+0000","mapping_epoch":269,"log_start":"284'4200","ondisk_log_start":"284'4200","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:43:51.180550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10596,"num_write_kb":339072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":33,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210929+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:35:25.210929+0000","last_peered":"2023-06-10T21:35:25.210929+0000","last_clean":"2023-06-10T21:35:25.210929+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:35:25.210929+0000","last_undegraded":"2023-06-10T21:35:25.210929+0000","last_fullsized":"2023-06-10T21:35:25.210929+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:26.701982+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:35:26.701982+0000","last_peered":"2023-06-10T21:35:26.701982+0000","last_clean":"2023-06-10T21:35:26.701982+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:35:26.701982+0000","last_undegraded":"2023-06-10T21:35:26.701982+0000","last_fullsized":"2023-06-10T21:35:26.701982+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"307'10906","reported_seq":18318,"reported_epoch":308,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:35:53.227796+0000","last_change":"2023-06-10T21:35:11.043628+0000","last_active":"2023-06-10T21:35:53.227796+0000","last_peered":"2023-06-10T21:35:53.227796+0000","last_clean":"2023-06-10T21:32:54.476638+0000","last_became_active":"2023-06-10T21:32:57.185008+0000","last_became_peered":"2023-06-10T21:32:57.185008+0000","last_unstale":"2023-06-10T21:35:53.227796+0000","last_undegraded":"2023-06-10T21:32:56.563037+0000","last_fullsized":"2023-06-10T21:32:56.523761+0000","mapping_epoch":289,"log_start":"284'4500","ondisk_log_start":"284'4500","created":269,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:37:52.547470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10906,"num_write_kb":348992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3679,"num_bytes_recovered":60096512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[0,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,3","objects":-1774}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":315,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210796+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:35:25.210796+0000","last_peered":"2023-06-10T21:35:25.210796+0000","last_clean":"2023-06-10T21:35:25.210796+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:35:25.210796+0000","last_undegraded":"2023-06-10T21:35:25.210796+0000","last_fullsized":"2023-06-10T21:35:25.210796+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210505+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:35:25.210505+0000","last_peered":"2023-06-10T21:35:25.210505+0000","last_clean":"2023-06-10T21:35:25.210505+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:35:25.210505+0000","last_undegraded":"2023-06-10T21:35:25.210505+0000","last_fullsized":"2023-06-10T21:35:25.210505+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"307'10484","reported_seq":10531,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.777916+0000","last_change":"2023-06-10T21:31:13.312368+0000","last_active":"2023-06-10T21:35:38.777916+0000","last_peered":"2023-06-10T21:35:38.777916+0000","last_clean":"2023-06-10T21:35:38.777916+0000","last_became_active":"2023-06-10T21:31:13.312063+0000","last_became_peered":"2023-06-10T21:31:13.312063+0000","last_unstale":"2023-06-10T21:35:38.777916+0000","last_undegraded":"2023-06-10T21:35:38.777916+0000","last_fullsized":"2023-06-10T21:35:38.777916+0000","mapping_epoch":269,"log_start":"282'3100","ondisk_log_start":"282'3100","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10484,"num_write_kb":335488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":190,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:41.722951+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:35:41.722951+0000","last_peered":"2023-06-10T21:35:41.722951+0000","last_clean":"2023-06-10T21:35:41.722951+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:35:41.722951+0000","last_undegraded":"2023-06-10T21:35:41.722951+0000","last_fullsized":"2023-06-10T21:35:41.722951+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":132,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203388+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:35:25.203388+0000","last_peered":"2023-06-10T21:35:25.203388+0000","last_clean":"2023-06-10T21:35:25.203388+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:35:25.203388+0000","last_undegraded":"2023-06-10T21:35:25.203388+0000","last_fullsized":"2023-06-10T21:35:25.203388+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"307'10388","reported_seq":20842,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.787146+0000","last_change":"2023-06-10T21:35:13.080168+0000","last_active":"2023-06-10T21:35:38.787146+0000","last_peered":"2023-06-10T21:35:38.787146+0000","last_clean":"2023-06-10T21:35:38.787146+0000","last_became_active":"2023-06-10T21:35:13.008964+0000","last_became_peered":"2023-06-10T21:35:13.008964+0000","last_unstale":"2023-06-10T21:35:38.787146+0000","last_undegraded":"2023-06-10T21:35:38.787146+0000","last_fullsized":"2023-06-10T21:35:38.787146+0000","mapping_epoch":305,"log_start":"282'3070","ondisk_log_start":"282'3070","created":269,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7318,"log_dups_size":0,"ondisk_log_size":7318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10388,"num_write_kb":332416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5194,"num_bytes_recovered":306511872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":260,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:41.722934+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:35:41.722934+0000","last_peered":"2023-06-10T21:35:41.722934+0000","last_clean":"2023-06-10T21:35:41.722934+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:35:41.722934+0000","last_undegraded":"2023-06-10T21:35:41.722934+0000","last_fullsized":"2023-06-10T21:35:41.722934+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":180,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:41.722960+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:35:41.722960+0000","last_peered":"2023-06-10T21:35:41.722960+0000","last_clean":"2023-06-10T21:35:41.722960+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:35:41.722960+0000","last_undegraded":"2023-06-10T21:35:41.722960+0000","last_fullsized":"2023-06-10T21:35:41.722960+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"307'10754","reported_seq":10804,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.772223+0000","last_change":"2023-06-10T21:31:13.164165+0000","last_active":"2023-06-10T21:35:38.772223+0000","last_peered":"2023-06-10T21:35:38.772223+0000","last_clean":"2023-06-10T21:35:38.772223+0000","last_became_active":"2023-06-10T21:31:13.163887+0000","last_became_peered":"2023-06-10T21:31:13.163887+0000","last_unstale":"2023-06-10T21:35:38.772223+0000","last_undegraded":"2023-06-10T21:35:38.772223+0000","last_fullsized":"2023-06-10T21:35:38.772223+0000","mapping_epoch":269,"log_start":"280'1600","ondisk_log_start":"280'1600","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9154,"log_dups_size":0,"ondisk_log_size":9154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:19.721588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":344128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:26.701996+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:35:26.701996+0000","last_peered":"2023-06-10T21:35:26.701996+0000","last_clean":"2023-06-10T21:35:26.701996+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:35:26.701996+0000","last_undegraded":"2023-06-10T21:35:26.701996+0000","last_fullsized":"2023-06-10T21:35:26.701996+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":300,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.192943+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:35:25.192943+0000","last_peered":"2023-06-10T21:35:25.192943+0000","last_clean":"2023-06-10T21:35:25.192943+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:35:25.192943+0000","last_undegraded":"2023-06-10T21:35:25.192943+0000","last_fullsized":"2023-06-10T21:35:25.192943+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"307'10538","reported_seq":10588,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.768844+0000","last_change":"2023-06-10T21:31:13.163927+0000","last_active":"2023-06-10T21:35:38.768844+0000","last_peered":"2023-06-10T21:35:38.768844+0000","last_clean":"2023-06-10T21:35:38.768844+0000","last_became_active":"2023-06-10T21:31:13.163548+0000","last_became_peered":"2023-06-10T21:31:13.163548+0000","last_unstale":"2023-06-10T21:35:38.768844+0000","last_undegraded":"2023-06-10T21:35:38.768844+0000","last_fullsized":"2023-06-10T21:35:38.768844+0000","mapping_epoch":269,"log_start":"285'4700","ondisk_log_start":"285'4700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:31.162000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10538,"num_write_kb":337216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":286,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.196933+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:35:25.196933+0000","last_peered":"2023-06-10T21:35:25.196933+0000","last_clean":"2023-06-10T21:35:25.196933+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:35:25.196933+0000","last_undegraded":"2023-06-10T21:35:25.196933+0000","last_fullsized":"2023-06-10T21:35:25.196933+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:41.722994+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:35:41.722994+0000","last_peered":"2023-06-10T21:35:41.722994+0000","last_clean":"2023-06-10T21:35:41.722994+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:35:41.722994+0000","last_undegraded":"2023-06-10T21:35:41.722994+0000","last_fullsized":"2023-06-10T21:35:41.722994+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"307'10542","reported_seq":10592,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.788961+0000","last_change":"2023-06-10T21:31:13.306572+0000","last_active":"2023-06-10T21:35:38.788961+0000","last_peered":"2023-06-10T21:35:38.788961+0000","last_clean":"2023-06-10T21:35:38.788961+0000","last_became_active":"2023-06-10T21:31:13.305905+0000","last_became_peered":"2023-06-10T21:31:13.305905+0000","last_unstale":"2023-06-10T21:35:38.788961+0000","last_undegraded":"2023-06-10T21:35:38.788961+0000","last_fullsized":"2023-06-10T21:35:38.788961+0000","mapping_epoch":269,"log_start":"285'4700","ondisk_log_start":"285'4700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:26:09.817167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10542,"num_write_kb":337344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":263,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:26.701908+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:35:26.701908+0000","last_peered":"2023-06-10T21:35:26.701908+0000","last_clean":"2023-06-10T21:35:26.701908+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:35:26.701908+0000","last_undegraded":"2023-06-10T21:35:26.701908+0000","last_fullsized":"2023-06-10T21:35:26.701908+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":219,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:41.723014+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:35:41.723014+0000","last_peered":"2023-06-10T21:35:41.723014+0000","last_clean":"2023-06-10T21:35:41.723014+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:35:41.723014+0000","last_undegraded":"2023-06-10T21:35:41.723014+0000","last_fullsized":"2023-06-10T21:35:41.723014+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"307'10424","reported_seq":10475,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-06-10T21:35:44.166477+0000","last_change":"2023-06-10T21:31:13.311963+0000","last_active":"2023-06-10T21:35:44.166477+0000","last_peered":"2023-06-10T21:35:44.166477+0000","last_clean":"2023-06-10T21:35:44.166477+0000","last_became_active":"2023-06-10T21:31:13.311659+0000","last_became_peered":"2023-06-10T21:31:13.311659+0000","last_unstale":"2023-06-10T21:35:44.166477+0000","last_undegraded":"2023-06-10T21:35:44.166477+0000","last_fullsized":"2023-06-10T21:35:44.166477+0000","mapping_epoch":269,"log_start":"278'1300","ondisk_log_start":"278'1300","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:25:35.237294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10424,"num_write_kb":333505,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":132,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.205529+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:35:25.205529+0000","last_peered":"2023-06-10T21:35:25.205529+0000","last_clean":"2023-06-10T21:35:25.205529+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:35:25.205529+0000","last_undegraded":"2023-06-10T21:35:25.205529+0000","last_fullsized":"2023-06-10T21:35:25.205529+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":312,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.229368+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:35:25.229368+0000","last_peered":"2023-06-10T21:35:25.229368+0000","last_clean":"2023-06-10T21:35:25.229368+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:35:25.229368+0000","last_undegraded":"2023-06-10T21:35:25.229368+0000","last_fullsized":"2023-06-10T21:35:25.229368+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.204569+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:35:25.204569+0000","last_peered":"2023-06-10T21:35:25.204569+0000","last_clean":"2023-06-10T21:35:25.204569+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:35:25.204569+0000","last_undegraded":"2023-06-10T21:35:25.204569+0000","last_fullsized":"2023-06-10T21:35:25.204569+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.b","version":"307'10548","reported_seq":10598,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-06-10T21:35:43.082636+0000","last_change":"2023-06-10T21:31:13.306250+0000","last_active":"2023-06-10T21:35:43.082636+0000","last_peered":"2023-06-10T21:35:43.082636+0000","last_clean":"2023-06-10T21:35:43.082636+0000","last_became_active":"2023-06-10T21:31:13.305959+0000","last_became_peered":"2023-06-10T21:31:13.305959+0000","last_unstale":"2023-06-10T21:35:43.082636+0000","last_undegraded":"2023-06-10T21:35:43.082636+0000","last_fullsized":"2023-06-10T21:35:43.082636+0000","mapping_epoch":269,"log_start":"285'4700","ondisk_log_start":"285'4700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:47.604554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10548,"num_write_kb":337536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":178,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.204648+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:35:25.204648+0000","last_peered":"2023-06-10T21:35:25.204648+0000","last_clean":"2023-06-10T21:35:25.204648+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:35:25.204648+0000","last_undegraded":"2023-06-10T21:35:25.204648+0000","last_fullsized":"2023-06-10T21:35:25.204648+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":252,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203523+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:35:25.203523+0000","last_peered":"2023-06-10T21:35:25.203523+0000","last_clean":"2023-06-10T21:35:25.203523+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:35:25.203523+0000","last_undegraded":"2023-06-10T21:35:25.203523+0000","last_fullsized":"2023-06-10T21:35:25.203523+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.193069+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:35:25.193069+0000","last_peered":"2023-06-10T21:35:25.193069+0000","last_clean":"2023-06-10T21:35:25.193069+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:35:25.193069+0000","last_undegraded":"2023-06-10T21:35:25.193069+0000","last_fullsized":"2023-06-10T21:35:25.193069+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"307'10530","reported_seq":10580,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.787758+0000","last_change":"2023-06-10T21:31:13.162508+0000","last_active":"2023-06-10T21:35:38.787758+0000","last_peered":"2023-06-10T21:35:38.787758+0000","last_clean":"2023-06-10T21:35:38.787758+0000","last_became_active":"2023-06-10T21:31:13.162199+0000","last_became_peered":"2023-06-10T21:31:13.162199+0000","last_unstale":"2023-06-10T21:35:38.787758+0000","last_undegraded":"2023-06-10T21:35:38.787758+0000","last_fullsized":"2023-06-10T21:35:38.787758+0000","mapping_epoch":269,"log_start":"285'4700","ondisk_log_start":"285'4700","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5830,"log_dups_size":0,"ondisk_log_size":5830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:51:26.342926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10530,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":152,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.197005+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:35:25.197005+0000","last_peered":"2023-06-10T21:35:25.197005+0000","last_clean":"2023-06-10T21:35:25.197005+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:35:25.197005+0000","last_undegraded":"2023-06-10T21:35:25.197005+0000","last_fullsized":"2023-06-10T21:35:25.197005+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:39.349295+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:35:39.349295+0000","last_peered":"2023-06-10T21:35:39.349295+0000","last_clean":"2023-06-10T21:35:39.349295+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:35:39.349295+0000","last_undegraded":"2023-06-10T21:35:39.349295+0000","last_fullsized":"2023-06-10T21:35:39.349295+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":201,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203993+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:35:25.203993+0000","last_peered":"2023-06-10T21:35:25.203993+0000","last_clean":"2023-06-10T21:35:25.203993+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:35:25.203993+0000","last_undegraded":"2023-06-10T21:35:25.203993+0000","last_fullsized":"2023-06-10T21:35:25.203993+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.a","version":"307'10362","reported_seq":10412,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.788078+0000","last_change":"2023-06-10T21:31:13.312315+0000","last_active":"2023-06-10T21:35:38.788078+0000","last_peered":"2023-06-10T21:35:38.788078+0000","last_clean":"2023-06-10T21:35:38.788078+0000","last_became_active":"2023-06-10T21:31:13.312036+0000","last_became_peered":"2023-06-10T21:31:13.312036+0000","last_unstale":"2023-06-10T21:35:38.788078+0000","last_undegraded":"2023-06-10T21:35:38.788078+0000","last_fullsized":"2023-06-10T21:35:38.788078+0000","mapping_epoch":269,"log_start":"285'4500","ondisk_log_start":"285'4500","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:13.019512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10362,"num_write_kb":331584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":332,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.194757+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:35:25.194757+0000","last_peered":"2023-06-10T21:35:25.194757+0000","last_clean":"2023-06-10T21:35:25.194757+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:35:25.194757+0000","last_undegraded":"2023-06-10T21:35:25.194757+0000","last_fullsized":"2023-06-10T21:35:25.194757+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":471,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210396+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:35:25.210396+0000","last_peered":"2023-06-10T21:35:25.210396+0000","last_clean":"2023-06-10T21:35:25.210396+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:35:25.210396+0000","last_undegraded":"2023-06-10T21:35:25.210396+0000","last_fullsized":"2023-06-10T21:35:25.210396+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":246,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.228441+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:35:25.228441+0000","last_peered":"2023-06-10T21:35:25.228441+0000","last_clean":"2023-06-10T21:35:25.228441+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:35:25.228441+0000","last_undegraded":"2023-06-10T21:35:25.228441+0000","last_fullsized":"2023-06-10T21:35:25.228441+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210876+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:35:25.210876+0000","last_peered":"2023-06-10T21:35:25.210876+0000","last_clean":"2023-06-10T21:35:25.210876+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:35:25.210876+0000","last_undegraded":"2023-06-10T21:35:25.210876+0000","last_fullsized":"2023-06-10T21:35:25.210876+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"307'10640","reported_seq":10690,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.775867+0000","last_change":"2023-06-10T21:31:13.307434+0000","last_active":"2023-06-10T21:35:38.775867+0000","last_peered":"2023-06-10T21:35:38.775867+0000","last_clean":"2023-06-10T21:35:38.775867+0000","last_became_active":"2023-06-10T21:31:13.307189+0000","last_became_peered":"2023-06-10T21:31:13.307189+0000","last_unstale":"2023-06-10T21:35:38.775867+0000","last_undegraded":"2023-06-10T21:35:38.775867+0000","last_fullsized":"2023-06-10T21:35:38.775867+0000","mapping_epoch":269,"log_start":"285'4800","ondisk_log_start":"285'4800","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:22:14.269491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10640,"num_write_kb":340480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":327,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210494+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:35:25.210494+0000","last_peered":"2023-06-10T21:35:25.210494+0000","last_clean":"2023-06-10T21:35:25.210494+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:35:25.210494+0000","last_undegraded":"2023-06-10T21:35:25.210494+0000","last_fullsized":"2023-06-10T21:35:25.210494+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203750+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:35:25.203750+0000","last_peered":"2023-06-10T21:35:25.203750+0000","last_clean":"2023-06-10T21:35:25.203750+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:35:25.203750+0000","last_undegraded":"2023-06-10T21:35:25.203750+0000","last_fullsized":"2023-06-10T21:35:25.203750+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":132,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.205173+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:35:25.205173+0000","last_peered":"2023-06-10T21:35:25.205173+0000","last_clean":"2023-06-10T21:35:25.205173+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:35:25.205173+0000","last_undegraded":"2023-06-10T21:35:25.205173+0000","last_fullsized":"2023-06-10T21:35:25.205173+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"307'10536","reported_seq":10586,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-06-10T21:35:44.166392+0000","last_change":"2023-06-10T21:31:13.311445+0000","last_active":"2023-06-10T21:35:44.166392+0000","last_peered":"2023-06-10T21:35:44.166392+0000","last_clean":"2023-06-10T21:35:44.166392+0000","last_became_active":"2023-06-10T21:31:13.311141+0000","last_became_peered":"2023-06-10T21:31:13.311141+0000","last_unstale":"2023-06-10T21:35:44.166392+0000","last_undegraded":"2023-06-10T21:35:44.166392+0000","last_fullsized":"2023-06-10T21:35:44.166392+0000","mapping_epoch":269,"log_start":"279'1400","ondisk_log_start":"279'1400","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":9136,"log_dups_size":0,"ondisk_log_size":9136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:09:56.293653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10536,"num_write_kb":337152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":312,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210853+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:35:25.210853+0000","last_peered":"2023-06-10T21:35:25.210853+0000","last_clean":"2023-06-10T21:35:25.210853+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:35:25.210853+0000","last_undegraded":"2023-06-10T21:35:25.210853+0000","last_fullsized":"2023-06-10T21:35:25.210853+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.193348+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:35:25.193348+0000","last_peered":"2023-06-10T21:35:25.193348+0000","last_clean":"2023-06-10T21:35:25.193348+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:35:25.193348+0000","last_undegraded":"2023-06-10T21:35:25.193348+0000","last_fullsized":"2023-06-10T21:35:25.193348+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":128,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203659+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:35:25.203659+0000","last_peered":"2023-06-10T21:35:25.203659+0000","last_clean":"2023-06-10T21:35:25.203659+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:35:25.203659+0000","last_undegraded":"2023-06-10T21:35:25.203659+0000","last_fullsized":"2023-06-10T21:35:25.203659+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"307'10394","reported_seq":10441,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.742955+0000","last_change":"2023-06-10T21:31:13.308614+0000","last_active":"2023-06-10T21:35:38.742955+0000","last_peered":"2023-06-10T21:35:38.742955+0000","last_clean":"2023-06-10T21:35:38.742955+0000","last_became_active":"2023-06-10T21:31:13.308346+0000","last_became_peered":"2023-06-10T21:31:13.308346+0000","last_unstale":"2023-06-10T21:35:38.742955+0000","last_undegraded":"2023-06-10T21:35:38.742955+0000","last_fullsized":"2023-06-10T21:35:38.742955+0000","mapping_epoch":269,"log_start":"282'3000","ondisk_log_start":"282'3000","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10394,"num_write_kb":332608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":286,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:26.702145+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:35:26.702145+0000","last_peered":"2023-06-10T21:35:26.702145+0000","last_clean":"2023-06-10T21:35:26.702145+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:35:26.702145+0000","last_undegraded":"2023-06-10T21:35:26.702145+0000","last_fullsized":"2023-06-10T21:35:26.702145+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":83,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:26.702114+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:35:26.702114+0000","last_peered":"2023-06-10T21:35:26.702114+0000","last_clean":"2023-06-10T21:35:26.702114+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:35:26.702114+0000","last_undegraded":"2023-06-10T21:35:26.702114+0000","last_fullsized":"2023-06-10T21:35:26.702114+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":267,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226943+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:35:25.226943+0000","last_peered":"2023-06-10T21:35:25.226943+0000","last_clean":"2023-06-10T21:35:25.226943+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:35:25.226943+0000","last_undegraded":"2023-06-10T21:35:25.226943+0000","last_fullsized":"2023-06-10T21:35:25.226943+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"307'10356","reported_seq":10403,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:38.787879+0000","last_change":"2023-06-10T21:31:13.312395+0000","last_active":"2023-06-10T21:35:38.787879+0000","last_peered":"2023-06-10T21:35:38.787879+0000","last_clean":"2023-06-10T21:35:38.787879+0000","last_became_active":"2023-06-10T21:31:13.312107+0000","last_became_peered":"2023-06-10T21:31:13.312107+0000","last_unstale":"2023-06-10T21:35:38.787879+0000","last_undegraded":"2023-06-10T21:35:38.787879+0000","last_fullsized":"2023-06-10T21:35:38.787879+0000","mapping_epoch":269,"log_start":"282'3000","ondisk_log_start":"282'3000","created":269,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:31:12.139414+0000","last_clean_scrub_stamp":"2023-06-10T21:31:12.139414+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":29,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.211048+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:35:25.211048+0000","last_peered":"2023-06-10T21:35:25.211048+0000","last_clean":"2023-06-10T21:35:25.211048+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:35:25.211048+0000","last_undegraded":"2023-06-10T21:35:25.211048+0000","last_fullsized":"2023-06-10T21:35:25.211048+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":136,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.205317+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:35:25.205317+0000","last_peered":"2023-06-10T21:35:25.205317+0000","last_clean":"2023-06-10T21:35:25.205317+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:35:25.205317+0000","last_undegraded":"2023-06-10T21:35:25.205317+0000","last_fullsized":"2023-06-10T21:35:25.205317+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":264,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203508+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:35:25.203508+0000","last_peered":"2023-06-10T21:35:25.203508+0000","last_clean":"2023-06-10T21:35:25.203508+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:35:25.203508+0000","last_undegraded":"2023-06-10T21:35:25.203508+0000","last_fullsized":"2023-06-10T21:35:25.203508+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":131,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.203744+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:35:25.203744+0000","last_peered":"2023-06-10T21:35:25.203744+0000","last_clean":"2023-06-10T21:35:25.203744+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:35:25.203744+0000","last_undegraded":"2023-06-10T21:35:25.203744+0000","last_fullsized":"2023-06-10T21:35:25.203744+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:34.348851+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:35:34.348851+0000","last_peered":"2023-06-10T21:35:34.348851+0000","last_clean":"2023-06-10T21:35:34.348851+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:35:34.348851+0000","last_undegraded":"2023-06-10T21:35:34.348851+0000","last_fullsized":"2023-06-10T21:35:34.348851+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":33,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.210642+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:35:25.210642+0000","last_peered":"2023-06-10T21:35:25.210642+0000","last_clean":"2023-06-10T21:35:25.210642+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:35:25.210642+0000","last_undegraded":"2023-06-10T21:35:25.210642+0000","last_fullsized":"2023-06-10T21:35:25.210642+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.227443+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:35:25.227443+0000","last_peered":"2023-06-10T21:35:25.227443+0000","last_clean":"2023-06-10T21:35:25.227443+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:35:25.227443+0000","last_undegraded":"2023-06-10T21:35:25.227443+0000","last_fullsized":"2023-06-10T21:35:25.227443+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":86,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.229091+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:35:25.229091+0000","last_peered":"2023-06-10T21:35:25.229091+0000","last_clean":"2023-06-10T21:35:25.229091+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:35:25.229091+0000","last_undegraded":"2023-06-10T21:35:25.229091+0000","last_fullsized":"2023-06-10T21:35:25.229091+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":246,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.228364+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:35:25.228364+0000","last_peered":"2023-06-10T21:35:25.228364+0000","last_clean":"2023-06-10T21:35:25.228364+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:35:25.228364+0000","last_undegraded":"2023-06-10T21:35:25.228364+0000","last_fullsized":"2023-06-10T21:35:25.228364+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":292,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.227737+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:35:25.227737+0000","last_peered":"2023-06-10T21:35:25.227737+0000","last_clean":"2023-06-10T21:35:25.227737+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:35:25.227737+0000","last_undegraded":"2023-06-10T21:35:25.227737+0000","last_fullsized":"2023-06-10T21:35:25.227737+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":295,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.226641+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:35:25.226641+0000","last_peered":"2023-06-10T21:35:25.226641+0000","last_clean":"2023-06-10T21:35:25.226641+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:35:25.226641+0000","last_undegraded":"2023-06-10T21:35:25.226641+0000","last_fullsized":"2023-06-10T21:35:25.226641+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":197,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.204077+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:35:25.204077+0000","last_peered":"2023-06-10T21:35:25.204077+0000","last_clean":"2023-06-10T21:35:25.204077+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:35:25.204077+0000","last_undegraded":"2023-06-10T21:35:25.204077+0000","last_fullsized":"2023-06-10T21:35:25.204077+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":41,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-06-10T21:35:25.192766+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:35:25.192766+0000","last_peered":"2023-06-10T21:35:25.192766+0000","last_clean":"2023-06-10T21:35:25.192766+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:35:25.192766+0000","last_undegraded":"2023-06-10T21:35:25.192766+0000","last_fullsized":"2023-06-10T21:35:25.192766+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":168542,"num_write_kb":5393281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8873,"num_bytes_recovered":366608384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":613810176,"data_stored":613810176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111172,"ondisk_log_size":111172,"up":48,"acting":47,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430665,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":592784,"kb_used_data":3272,"kb_used_omap":0,"kb_used_meta":589504,"kb_avail":93152368,"statfs":{"total":95995035648,"available":95388024832,"internally_reserved":0,"allocated":3350528,"data_stored":2784054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603652096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":4.5199999999999996,"15min":7.4340000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":28.875,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":3.734,"5min":4.7830000000000004,"15min":7.7130000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":28.943000000000001,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.436}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":3.2210000000000001,"15min":3.2930000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":4.4669999999999996,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.0649999999999999,"5min":3.6539999999999999,"15min":3.8570000000000002},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":8.1020000000000003,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":6.5759999999999996,"5min":7.1050000000000004,"15min":4.5890000000000004},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":103.866,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":7.5999999999999996,"5min":7.4850000000000003,"15min":5.3200000000000003},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":103.705,"5min":172.185,"15min":212.13399999999999},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Jun 10 21:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2229999999999999,"5min":4.218,"15min":3.0720000000000001},"min":{"1min":0.375,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":23.611999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.6800000000000002,"5min":3.883,"15min":2.516},"min":{"1min":0.36399999999999999,"5min":0.31,"15min":0.31},"max":{"1min":23.591999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":0.59399999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.71499999999999997},"min":{"1min":0.28199999999999997,"5min":0.249,"15min":0.249},"max":{"1min":0.93799999999999994,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.441,"5min":0.42099999999999999,"15min":1.0289999999999999},"min":{"1min":0.309,"5min":0.245,"15min":0.245},"max":{"1min":0.81000000000000005,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.51800000000000002,"15min":1.821},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":1.7869999999999999,"5min":3.6280000000000001,"15min":211.99799999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.44500000000000001,"15min":1.6160000000000001},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.29299999999999998},"max":{"1min":0.69799999999999995,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.63}]},{"osd":6,"last update":"Sat Jun 10 21:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":1.077},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":2.516,"5min":3.2559999999999998,"15min":13.073},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.48599999999999999,"15min":0.93700000000000006},"min":{"1min":0.34699999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73799999999999999,"5min":1.897,"15min":10.874000000000001},"last":0.71899999999999997}]}]},{"osd":6,"up_from":0,"seq":180388626893,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566336,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1009920,"kb_used_data":348920,"kb_used_omap":4,"kb_used_meta":660987,"kb_avail":92735232,"statfs":{"total":95995035648,"available":94960877568,"internally_reserved":0,"allocated":357294080,"data_stored":356331981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":676851598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:48 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":4.9550000000000001,"15min":10.315},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":28.507999999999999,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.221},{"interface":"front","average":{"1min":3.714,"5min":4.5700000000000003,"15min":10.029},"min":{"1min":0.27300000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":28.045999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.37}]},{"osd":2,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":11.297000000000001,"5min":9.1150000000000002,"15min":5.7110000000000003},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":134.79900000000001,"5min":191.101,"15min":191.101},"last":0.27500000000000002},{"interface":"front","average":{"1min":4.8479999999999999,"5min":7.702,"15min":5.0469999999999997},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.218},"max":{"1min":15.923,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.314}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":3.9079999999999999,"5min":4.7329999999999997,"15min":2.895},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.23799999999999999},"max":{"1min":15.130000000000001,"5min":80.584999999999994,"15min":80.584999999999994},"last":0.45400000000000001},{"interface":"front","average":{"1min":3.593,"5min":5.3479999999999999,"15min":3.9039999999999999},"min":{"1min":0.35199999999999998,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":10.961,"5min":80.873999999999995,"15min":80.873999999999995},"last":0.29999999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.355,"15min":1.548},"min":{"1min":0.35799999999999998,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":9.3659999999999997,"5min":10.036,"15min":16.059999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.589,"15min":1.452},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":8.8629999999999995,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.499}]},{"osd":5,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":2.0550000000000002,"15min":4.2169999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.035,"5min":10.035,"15min":383.83100000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.3680000000000001,"15min":3.8050000000000002},"min":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":9.3160000000000007,"5min":10.116,"15min":383.84800000000001},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.069,"15min":2.1139999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":14.858000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.4990000000000001,"15min":1.7210000000000001},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":9.3849999999999998,"5min":10.066000000000001,"15min":13.48},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.0579999999999998,"15min":2.661},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":9.343,"5min":47.844000000000001,"15min":162.399},"last":0.435},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.4369999999999998,"15min":2.476},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":8.8360000000000003,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.44500000000000001}]}]},{"osd":0,"up_from":244,"seq":1047972020313,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265552,"kb_used_data":3784,"kb_used_omap":10,"kb_used_meta":261749,"kb_avail":93479600,"statfs":{"total":95995035648,"available":95723110400,"internally_reserved":0,"allocated":3874816,"data_stored":2836916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":268031028},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":4.0869999999999997,"15min":4.391},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":4.6500000000000004,"5min":139.68000000000001,"15min":139.68000000000001},"last":0.379},{"interface":"front","average":{"1min":4.5839999999999996,"5min":4.5140000000000002,"15min":5.4619999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":15.403,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2569999999999997,"5min":7.1600000000000001,"15min":6.7930000000000001},"min":{"1min":0.31,"5min":0.216,"15min":0.216},"max":{"1min":24.166,"5min":132.245,"15min":205.10900000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":4.0499999999999998,"5min":6.7140000000000004,"15min":6.2770000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":13.06,"5min":132.26599999999999,"15min":208.34299999999999},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":4.1340000000000003,"15min":5.569},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.934999999999999,"5min":111.745,"15min":111.745},"last":0.41199999999999998},{"interface":"front","average":{"1min":4.29,"5min":4.9589999999999996,"15min":6.5090000000000003},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":12.438000000000001,"5min":114.205,"15min":114.205},"last":0.32100000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.9710000000000001,"15min":3.9460000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":8.9550000000000001,"5min":10.744999999999999,"15min":29.709},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.4049999999999998,"5min":2.2440000000000002,"15min":3.5110000000000001},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":24.222999999999999,"5min":24.222999999999999,"15min":24.222999999999999},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":2.2639999999999998,"15min":3.5859999999999999},"min":{"1min":0.438,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":24.288,"5min":24.288,"15min":24.288},"last":0.495},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.8169999999999999,"15min":3.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":6.9870000000000001,"5min":8.0079999999999991,"15min":19.846},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.056,"15min":4.1100000000000003},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":9,"5min":10.771000000000001,"15min":29.806000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":3.4430000000000001,"5min":2.3839999999999999,"15min":3.6509999999999998},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":24.260999999999999,"5min":24.260999999999999,"15min":24.260999999999999},"last":0.46600000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.0449999999999999,"15min":3.3839999999999999},"min":{"1min":0.48199999999999998,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.0149999999999997,"5min":17.523,"15min":19.866},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.843,"5min":2.2120000000000002,"15min":4.1609999999999996},"min":{"1min":0.44800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0259999999999998,"5min":17.792999999999999,"15min":29.77},"last":0.58299999999999996}]}]},{"osd":2,"up_from":120,"seq":515396075724,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494588,"kb_used_data":3124,"kb_used_omap":7,"kb_used_meta":491448,"kb_avail":93250564,"statfs":{"total":95995035648,"available":95488577536,"internally_reserved":0,"allocated":3198976,"data_stored":2240732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":503242833},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7000000000000002,"5min":4.6079999999999997,"15min":4.694},"min":{"1min":0.217,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.9,"5min":70.510999999999996,"15min":70.510999999999996},"last":0.28000000000000003},{"interface":"front","average":{"1min":5.0209999999999999,"5min":4.4329999999999998,"15min":5.1479999999999997},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":11.733000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":5.0979999999999999,"15min":3.2799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.19},"max":{"1min":7.7910000000000004,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.26700000000000002},{"interface":"front","average":{"1min":4.3780000000000001,"5min":4.968,"15min":3.0059999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.214},"max":{"1min":9.4619999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:07 2023","interfaces":[{"interface":"back","average":{"1min":13.648,"5min":6.2619999999999996,"15min":5.5810000000000004},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":97.296000000000006,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":12.912000000000001,"5min":6.1219999999999999,"15min":6.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":97.308999999999997,"5min":138.095,"15min":138.095},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6879999999999999,"15min":2.976},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":224.00999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.831,"5min":2.3029999999999999,"15min":3.3260000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":7.7030000000000003,"5min":29.486999999999998,"15min":223.28399999999999},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.681,"15min":1.671},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":5.4470000000000001,"5min":10.554,"15min":14.180999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.7879999999999998,"5min":1.73,"15min":1.8939999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":7.3200000000000003,"5min":7.3200000000000003,"15min":16.125},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8140000000000001,"5min":1.802,"15min":1.966},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":7.4180000000000001,"5min":7.4180000000000001,"15min":16.183},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.1339999999999999,"5min":2.4239999999999999,"15min":2.3500000000000001},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.7510000000000003,"5min":29.538,"15min":29.538},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":3.4630000000000001,"15min":2.6429999999999998},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.8399999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.4340000000000002,"15min":2.9390000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.3620000000000001,"5min":73.475999999999999,"15min":113.937},"last":0.48399999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824084,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":505488,"kb_used_data":259336,"kb_used_omap":7,"kb_used_meta":246136,"kb_avail":93239664,"statfs":{"total":95995035648,"available":95477415936,"internally_reserved":0,"allocated":265560064,"data_stored":264522557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":252043283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":8.3510000000000009,"5min":4.8239999999999998,"15min":5.3070000000000004},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":105.09,"5min":105.09,"15min":105.09},"last":0.24199999999999999},{"interface":"front","average":{"1min":8.1940000000000008,"5min":4.4980000000000002,"15min":5.0540000000000003},"min":{"1min":0.51200000000000001,"5min":0.246,"15min":0.216},"max":{"1min":99.852999999999994,"5min":99.852999999999994,"15min":99.852999999999994},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0459999999999994,"5min":5.194,"15min":4.157},"min":{"1min":0.25800000000000001,"5min":0.248,"15min":0.23100000000000001},"max":{"1min":94.766000000000005,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.40699999999999997},{"interface":"front","average":{"1min":10.343,"5min":4.6310000000000002,"15min":3.867},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":94.691000000000003,"5min":94.691000000000003,"15min":94.691000000000003},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.5120000000000005,"5min":3.992,"15min":9.9450000000000003},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":100.676,"5min":100.676,"15min":100.676},"last":0.26100000000000001},{"interface":"front","average":{"1min":10.388,"5min":5.7539999999999996,"15min":11.48},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":103.032,"5min":103.032,"15min":103.032},"last":0.307}]},{"osd":4,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.575,"15min":1.4410000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":5.0170000000000003,"5min":13.417,"15min":13.417},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.8260000000000001,"15min":2.7679999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.616,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5569999999999999,"15min":1.4470000000000001},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.71,"15min":1.5229999999999999},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0629999999999997,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":2.847,"15min":2.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.537000000000001,"5min":19.805,"15min":19.805},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6140000000000001,"15min":1.498},"min":{"1min":0.433,"5min":0.433,"15min":0.39100000000000001},"max":{"1min":9.7289999999999992,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":0.52900000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.524,"15min":3.0259999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.648999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.3050000000000002,"5min":2.2850000000000001,"15min":1.704},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.36699999999999999},"max":{"1min":9.8239999999999998,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.56000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019349,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":606536,"kb_used_data":2688,"kb_used_omap":0,"kb_used_meta":603840,"kb_avail":93138616,"statfs":{"total":95995035648,"available":95373942784,"internally_reserved":0,"allocated":2752512,"data_stored":2192678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":618332160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":4.1260000000000003,"15min":7.899},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.503,"5min":234.203,"15min":234.203},"last":0.28799999999999998},{"interface":"front","average":{"1min":3.2770000000000001,"5min":4.3730000000000002,"15min":7.8220000000000001},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":37.399999999999999,"5min":234.19,"15min":234.19},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":8.7840000000000007,"5min":3.121,"15min":3.468},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":88.700999999999993,"5min":88.700999999999993,"15min":182.035},"last":0.51100000000000001},{"interface":"front","average":{"1min":8.6110000000000007,"5min":3.3130000000000002,"15min":3.375},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":87.853999999999999,"5min":87.853999999999999,"15min":181.73099999999999},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":7.6150000000000002,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.308999999999997,"5min":255.386,"15min":255.386},"last":0.35799999999999998},{"interface":"front","average":{"1min":6.4720000000000004,"5min":7.5499999999999998,"15min":3.6850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":85.322000000000003,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":17.739000000000001,"5min":7.5629999999999997,"15min":4.1150000000000002},"min":{"1min":0.57699999999999996,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":102.94,"5min":234.91900000000001,"15min":234.91900000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":12.029999999999999,"5min":8.1590000000000007,"15min":4.0819999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":90.489999999999995,"5min":235.05099999999999,"15min":235.05099999999999},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.379,"15min":0.99299999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.54300000000000004,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.437},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.45700000000000002,"15min":1.004},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.45500000000000002,"5min":5.25,"15min":11.904999999999999},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.43099999999999999,"15min":1.0249999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.53900000000000003,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.432,"5min":0.46600000000000003,"15min":1.0900000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.59499999999999997,"5min":1.71,"15min":13.845000000000001},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.77500000000000002,"15min":1.2789999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.38},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.79600000000000004,"15min":1.2210000000000001},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.57199999999999995,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.48099999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823121,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":353501184,"data_stored":353501184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":260308992,"data_stored":260308992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:35:59.346 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:35:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:59.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:35:59.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:35:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:35:59.452 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:35:59.483 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:35:59.529 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:35:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:35:59.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:35:59.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:35:59.666 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:00.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:00.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:00.195 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:00.217 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:00.217 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:00.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:00.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: osdmap e310: 8 total, 8 up, 6 in 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[180297]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:00.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: osdmap e310: 8 total, 8 up, 6 in 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:00 smithi064 ceph-mon[186706]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:00.281 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2690535815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: osdmap e310: 8 total, 8 up, 6 in 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.310 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: failed to encode map e310 with expected crc 2023-06-10T21:36:00.311 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:00 smithi152 ceph-mon[156819]: pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:00.312 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:00.361 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:00.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:00.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:00.496 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:00.541 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:00.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:00.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:00.646 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:00.674 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:00.716 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:00.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:00.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:00.848 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:00.890 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:00.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:01.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:01.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:01.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:01.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:01.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:01.149 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:36:01.245 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:01.275 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:01.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:36:01.376 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:01.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:01 smithi064 ceph-mon[186706]: failed to encode map e310 with expected crc 2023-06-10T21:36:01.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:01 smithi064 ceph-mon[180297]: failed to encode map e310 with expected crc 2023-06-10T21:36:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:01.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:01 smithi152 ceph-mon[156819]: failed to encode map e310 with expected crc 2023-06-10T21:36:01.482 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:01.488 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:01.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:01.554 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:01.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:01.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:01.684 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:01.731 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:01.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:01.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:01.864 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:01.961 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:01.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:02.054 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:02.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:02.097 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:02.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:36:02.114 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:02 smithi152 ceph-mon[156819]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:02.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:02.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:02.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:02.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:02.257 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:02 smithi064 ceph-mon[180297]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:02.258 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:02 smithi064 ceph-mon[186706]: pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:02.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:02.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:36:02.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:02.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:02.652 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:02.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:02.890 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:02.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:02.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:03.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:03.072 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:03.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:02] "GET /metrics HTTP/1.1" 200 224034 "" "Prometheus/2.33.4" 2023-06-10T21:36:03.172 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:03.173 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:03.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:03.208 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:03.274 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_7' created 2023-06-10T21:36:03.353 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:03.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:03.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:03.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:03.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-10T21:36:03.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:03.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:03.486 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:03.530 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:03.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:03.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-10T21:36:03.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:03.535 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:03.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-10T21:36:03.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:03.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:03.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:03.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:03.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:36:03.999 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:04.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:04.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:04.085 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-06-10T21:36:04.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:04.125 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:04.146 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:04.159 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:04.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:04.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:04.369 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:04.401 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-10T21:36:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: osdmap e311: 8 total, 8 up, 6 in 2023-06-10T21:36:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:04.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: pgmap v1152: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:04.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[186706]: osdmap e312: 8 total, 8 up, 6 in 2023-06-10T21:36:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-10T21:36:04.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: osdmap e311: 8 total, 8 up, 6 in 2023-06-10T21:36:04.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:04.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: pgmap v1152: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:04.414 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:04 smithi064 ceph-mon[180297]: osdmap e312: 8 total, 8 up, 6 in 2023-06-10T21:36:04.443 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:04.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:04.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:04.478 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:04.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:36:04.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:36:04.537 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:36:04.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:04.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3480381849' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-10T21:36:04.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: osdmap e311: 8 total, 8 up, 6 in 2023-06-10T21:36:04.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.551 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: failed to encode map e311 with expected crc 2023-06-10T21:36:04.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:04.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: pgmap v1152: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:04.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:04 smithi152 ceph-mon[156819]: osdmap e312: 8 total, 8 up, 6 in 2023-06-10T21:36:04.557 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:36:04.579 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:36:04.592 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:04.603 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:36:04.631 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:36:04.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:04.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:04.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:04.656 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.659 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:36:04.757 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:04.791 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:05.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[180297]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[180297]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/105607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:05.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[186706]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[186706]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/105607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:05.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:05 smithi152 ceph-mon[156819]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.497 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:05 smithi152 ceph-mon[156819]: failed to encode map e312 with expected crc 2023-06-10T21:36:05.498 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/105607' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:05.689 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:05.689 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:36:05.689 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:36:05.689 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:36:05.689 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:05.985 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:05.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:06.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:06.120 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:06.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:36:06.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: osdmap e313: 8 total, 8 up, 6 in 2023-06-10T21:36:06.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[180297]: pgmap v1155: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: osdmap e313: 8 total, 8 up, 6 in 2023-06-10T21:36:06.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:06 smithi064 ceph-mon[186706]: pgmap v1155: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:06.663 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:06.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:06.683 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:06.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: osdmap e313: 8 total, 8 up, 6 in 2023-06-10T21:36:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: failed to encode map e313 with expected crc 2023-06-10T21:36:06.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:06 smithi152 ceph-mon[156819]: pgmap v1155: 89 pgs: 2 creating+peering, 9 unknown, 78 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:06.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:06.789 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:06.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:06.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:06.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:07.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:07.044 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:07.238 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:07.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:07.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:07.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:07.379 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:07.404 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:36:07.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:07.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3859763608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:07.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3859763608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:07 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:07.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:07.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:07.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3859763608' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:07 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:07 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:07.774 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:07.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:07.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:07.810 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:07.811 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:36:07.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:07.908 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:08.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:36:08.003 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:08.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:08.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:08.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:08.174 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(10.f 227/220 [PushOp(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head, version: 200'380, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:fd43a91d:::benchmark_data_smithi064_312713_object6281:head@200'380, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:36:08.175 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 528.65631615400002, 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001, 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514152+0000", 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514151+0000", 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5096999999999998e-05 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514206+0000", 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5574e-05 2023-06-10T21:36:08.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514222+0000", 2023-06-10T21:36:08.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0900000000000005e-06 2023-06-10T21:36:08.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.514230+0000", 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0036878129999999999 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517918+0000", 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0910000000000003e-06 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.517922+0000", 2023-06-10T21:36:08.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.231370187 2023-06-10T21:36:08.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.749292+0000", 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23514043400000001 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:96995 10.f e227/220 10:f95a89c6:::benchmark_data_smithi064_312713_object6239:head v 227'5921, mlcod=200'373)", 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 528.65200126100001, 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001, 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.185 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518466+0000", 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1389999999999998e-06 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518473+0000", 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0293e-05 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518483+0000", 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6609999999999997e-06 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.518489+0000", 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23332091099999999 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751810+0000", 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9099999999999999e-05 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.751839+0000", 2023-06-10T21:36:08.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027593894000000001 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779433+0000", 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1985999999999999e-05 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779455+0000", 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.26098824100000001 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.35769.0:97001 10.f e227/220 10:fd7a027e:::benchmark_data_smithi064_312713_object6245:head v 227'5922, mlcod=200'373)", 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 528.647971762, 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002, 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522496+0000", 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522495+0000", 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5910000000000001e-06 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.522502+0000", 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22114999699999999 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743652+0000", 2023-06-10T21:36:08.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4776999999999999e-05 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.743667+0000", 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013856040999999999 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757523+0000", 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8875999999999999e-05 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.757551+0000", 2023-06-10T21:36:08.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021906202999999999 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779458+0000", 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9106e-05 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:19.779477+0000", 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25698084100000002 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111479 10.6 10:610f9587:::benchmark_data_smithi064_312713_object20723:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.85811490399999, 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999, 2023-06-10T21:36:08.201 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111479 2023-06-10T21:36:08.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312353+0000", 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312351+0000", 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1720000000000004e-06 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312358+0000", 2023-06-10T21:36:08.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312370+0000", 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4580000000000004e-06 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312378+0000", 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024553309999999998 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314833+0000", 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.314842+0000", 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0343133339999999 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.349155+0000", 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001791864 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350947+0000", 2023-06-10T21:36:08.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0385939259999999 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111480 10.d 10:bb138473:::benchmark_data_smithi064_312713_object20724:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e232)", 2023-06-10T21:36:08.210 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.85808160099998, 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999, 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.211 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.212 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.212 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111480 2023-06-10T21:36:08.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.212 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312386+0000", 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.802e-06 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312388+0000", 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9600000000000002e-06 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312392+0000", 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7259999999999998e-06 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312395+0000", 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7097000000000001e-05 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312432+0000", 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5759999999999996e-06 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.312438+0000", 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.019375897 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331814+0000", 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6889000000000003e-05 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331881+0000", 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0194944969999999 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.219 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111483 10.d 10:b3a132fc:::benchmark_data_smithi064_312713_object20727:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.84915516299998, 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817, 2023-06-10T21:36:08.220 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111483 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.221 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321313+0000", 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321311+0000", 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1680000000000003e-06 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321316+0000", 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2384999999999999e-05 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321329+0000", 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9739999999999999e-06 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.321335+0000", 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069082699999999998 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322026+0000", 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2101e-05 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.322038+0000", 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.009867013 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331905+0000", 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2904000000000005e-05 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331978+0000", 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00039655499999999999 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 6", 2023-06-10T21:36:08.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.332374+0000", 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011245184 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343619+0000", 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7795999999999998e-05 2023-06-10T21:36:08.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343707+0000", 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0745000000000003e-05 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343758+0000", 2023-06-10T21:36:08.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9522e-05 2023-06-10T21:36:08.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.343857+0000", 2023-06-10T21:36:08.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.022544817 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.233 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111488 10.4 10.41ecfd74 (undecoded) ondisk+write+known_if_redirected e232)", 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.83873470100002, 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999, 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.234 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111488 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331733+0000", 2023-06-10T21:36:08.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:36:08.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331731+0000", 2023-06-10T21:36:08.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7428e-05 2023-06-10T21:36:08.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:36:08.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4331999999999999e-05 2023-06-10T21:36:08.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331763+0000", 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5490000000000003e-06 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331769+0000", 2023-06-10T21:36:08.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9897e-05 2023-06-10T21:36:08.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331819+0000", 2023-06-10T21:36:08.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0000000000000002e-06 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331825+0000", 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0023298279999999 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334155+0000", 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0281e-05 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.334185+0000", 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0024523359999999 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.245 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111491 10.6 10:69475ff3:::benchmark_data_smithi064_312713_object20735:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:36:08.245 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:36:08.245 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.82052245, 2023-06-10T21:36:08.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693, 2023-06-10T21:36:08.246 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.246 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.246 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.246 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.247 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.247 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111491 2023-06-10T21:36:08.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.247 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:36:08.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349945+0000", 2023-06-10T21:36:08.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:36:08.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349943+0000", 2023-06-10T21:36:08.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5360000000000004e-06 2023-06-10T21:36:08.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349951+0000", 2023-06-10T21:36:08.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4100999999999999e-05 2023-06-10T21:36:08.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349965+0000", 2023-06-10T21:36:08.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2029999999999997e-06 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.349971+0000", 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4274000000000002e-05 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350035+0000", 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9600000000000006e-06 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.350044+0000", 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.000890976 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.350935+0000", 2023-06-10T21:36:08.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051541900000000003 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351451+0000", 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029125 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.351742+0000", 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098297039999999999 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.361571+0000", 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025176170000000002 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364089+0000", 2023-06-10T21:36:08.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5660000000000001e-05 2023-06-10T21:36:08.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364125+0000", 2023-06-10T21:36:08.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4188000000000001e-05 2023-06-10T21:36:08.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364209+0000", 2023-06-10T21:36:08.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.014263693 2023-06-10T21:36:08.263 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.263 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.263 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111494 10.6 10:60c1325c:::benchmark_data_smithi064_312713_object20738:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 506.80429485799999, 2023-06-10T21:36:08.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013, 2023-06-10T21:36:08.265 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.265 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.265 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:08.265 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:08.265 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111494 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:36:08.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366173+0000", 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366171+0000", 2023-06-10T21:36:08.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9200000000000001e-06 2023-06-10T21:36:08.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366177+0000", 2023-06-10T21:36:08.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2636000000000001e-05 2023-06-10T21:36:08.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366190+0000", 2023-06-10T21:36:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9599999999999997e-06 2023-06-10T21:36:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366196+0000", 2023-06-10T21:36:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6909000000000001e-05 2023-06-10T21:36:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366243+0000", 2023-06-10T21:36:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4109999999999996e-06 2023-06-10T21:36:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.366251+0000", 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.989107077 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355358+0000", 2023-06-10T21:36:08.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102029 2023-06-10T21:36:08.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355460+0000", 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2402000000000001e-05 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1", 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.355553+0000", 2023-06-10T21:36:08.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086816740000000003 2023-06-10T21:36:08.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:08.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.364234+0000", 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003252605 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367487+0000", 2023-06-10T21:36:08.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8502999999999999e-05 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367546+0000", 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7332999999999998e-05 2023-06-10T21:36:08.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.367643+0000", 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.001470013 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.281 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.282 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.282 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:112647 11.d e298/269 11:b76bb90c:::benchmark_data_smithi064_385725_object28374:head v 298'7032, mlcod=298'7027)", 2023-06-10T21:36:08.282 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:48.118486+0000", 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 140.05198199399999, 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010239988, 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118486+0000", 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118486+0000", 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118485+0000", 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4390000000000002e-06 2023-06-10T21:36:08.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118490+0000", 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4190000000000001e-06 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118495+0000", 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3269999999999999e-06 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.118497+0000", 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034164199999999999 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.121914+0000", 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5386000000000001e-05 2023-06-10T21:36:08.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.121939+0000", 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067360529999999997 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.128675+0000", 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0337999999999998e-05 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.128726+0000", 2023-06-10T21:36:08.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010239988 2023-06-10T21:36:08.291 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.291 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.291 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.292 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:112960 11.7 e298/269 11:e5139eac:::benchmark_data_smithi064_385725_object28687:head v 298'7073, mlcod=298'7069)", 2023-06-10T21:36:08.292 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:36:08.292 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 139.45869905800001, 2023-06-10T21:36:08.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010463623999999999, 2023-06-10T21:36:08.293 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.293 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.293 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711768+0000", 2023-06-10T21:36:08.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.305e-06 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711774+0000", 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5260000000000004e-06 2023-06-10T21:36:08.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711778+0000", 2023-06-10T21:36:08.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8289999999999999e-06 2023-06-10T21:36:08.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711781+0000", 2023-06-10T21:36:08.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026061560000000001 2023-06-10T21:36:08.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.714387+0000", 2023-06-10T21:36:08.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4856999999999999e-05 2023-06-10T21:36:08.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.714412+0000", 2023-06-10T21:36:08.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077718650000000002 2023-06-10T21:36:08.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.722184+0000", 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8520999999999998e-05 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.722232+0000", 2023-06-10T21:36:08.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010463623999999999 2023-06-10T21:36:08.303 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.303 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.303 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.304 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:123854 11.c e299/269 11:3352ddb2:::benchmark_data_smithi064_385725_object39581:head v 299'7719, mlcod=299'7715)", 2023-06-10T21:36:08.305 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:09.091142+0000", 2023-06-10T21:36:08.305 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 119.079325274, 2023-06-10T21:36:08.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098067099999999997, 2023-06-10T21:36:08.305 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091142+0000", 2023-06-10T21:36:08.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091142+0000", 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091142+0000", 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3540000000000002e-06 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091149+0000", 2023-06-10T21:36:08.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7769999999999999e-06 2023-06-10T21:36:08.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091155+0000", 2023-06-10T21:36:08.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3699999999999999e-06 2023-06-10T21:36:08.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.091159+0000", 2023-06-10T21:36:08.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064627360000000002 2023-06-10T21:36:08.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.097621+0000", 2023-06-10T21:36:08.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4076000000000001e-05 2023-06-10T21:36:08.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.097645+0000", 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003267692 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.100913+0000", 2023-06-10T21:36:08.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5982000000000002e-05 2023-06-10T21:36:08.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:09.100949+0000", 2023-06-10T21:36:08.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098067099999999997 2023-06-10T21:36:08.315 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.315 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.315 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:128227 11.c e300/269 11:351f71d2:::benchmark_data_smithi064_385725_object43954:head v 300'8002, mlcod=300'7999)", 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:17.364160+0000", 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 110.80630734, 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098295220000000003, 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.316 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.317 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364160+0000", 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364160+0000", 2023-06-10T21:36:08.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364160+0000", 2023-06-10T21:36:08.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.129e-06 2023-06-10T21:36:08.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364163+0000", 2023-06-10T21:36:08.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.344e-06 2023-06-10T21:36:08.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364169+0000", 2023-06-10T21:36:08.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6769999999999999e-06 2023-06-10T21:36:08.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.364171+0000", 2023-06-10T21:36:08.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003126088 2023-06-10T21:36:08.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.367297+0000", 2023-06-10T21:36:08.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5415000000000001e-05 2023-06-10T21:36:08.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.367313+0000", 2023-06-10T21:36:08.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066323889999999998 2023-06-10T21:36:08.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.373945+0000", 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4740000000000002e-05 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:17.373990+0000", 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0098295220000000003 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.330 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:129479 11.c e300/269 11:3234d96d:::benchmark_data_smithi064_385725_object45206:head v 300'8087, mlcod=300'8084)", 2023-06-10T21:36:08.330 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:20.085874+0000", 2023-06-10T21:36:08.330 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 108.084593764, 2023-06-10T21:36:08.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097292200000000002, 2023-06-10T21:36:08.330 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.331 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.331 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085874+0000", 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085874+0000", 2023-06-10T21:36:08.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085873+0000", 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9039999999999998e-06 2023-06-10T21:36:08.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085882+0000", 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6385999999999999e-05 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085898+0000", 2023-06-10T21:36:08.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0487e-05 2023-06-10T21:36:08.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.085909+0000", 2023-06-10T21:36:08.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034998260000000002 2023-06-10T21:36:08.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.089409+0000", 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4270999999999999e-05 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.089433+0000", 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006133566 2023-06-10T21:36:08.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.095567+0000", 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6544000000000001e-05 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:20.095603+0000", 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0097292200000000002 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.339 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:141854 11.c e302/269 11:377c3135:::benchmark_data_smithi064_385725_object57581:head v 302'8837, mlcod=302'8834)", 2023-06-10T21:36:08.340 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:36:08.340 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 83.017568037999993, 2023-06-10T21:36:08.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012712886, 2023-06-10T21:36:08.340 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.340 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.341 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:36:08.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:36:08.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152899+0000", 2023-06-10T21:36:08.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6380000000000003e-06 2023-06-10T21:36:08.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152904+0000", 2023-06-10T21:36:08.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.19e-06 2023-06-10T21:36:08.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152910+0000", 2023-06-10T21:36:08.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8850000000000001e-06 2023-06-10T21:36:08.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152914+0000", 2023-06-10T21:36:08.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032875410000000002 2023-06-10T21:36:08.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.156202+0000", 2023-06-10T21:36:08.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5582000000000001e-05 2023-06-10T21:36:08.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.156217+0000", 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093518679999999993 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.165569+0000", 2023-06-10T21:36:08.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3519999999999997e-05 2023-06-10T21:36:08.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.165613+0000", 2023-06-10T21:36:08.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012712886 2023-06-10T21:36:08.352 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.352 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.353 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.353 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:143254 11.d e302/269 11:b20b535e:::benchmark_data_smithi064_385725_object58981:head v 302'8903, mlcod=302'8900)", 2023-06-10T21:36:08.354 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:36:08.354 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 80.338053629000001, 2023-06-10T21:36:08.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010447279, 2023-06-10T21:36:08.354 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.355 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.355 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:36:08.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:36:08.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:36:08.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:36:08.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832419+0000", 2023-06-10T21:36:08.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9190000000000004e-06 2023-06-10T21:36:08.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832425+0000", 2023-06-10T21:36:08.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8449999999999999e-06 2023-06-10T21:36:08.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832428+0000", 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041750249999999997 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.836603+0000", 2023-06-10T21:36:08.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8873e-05 2023-06-10T21:36:08.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.836622+0000", 2023-06-10T21:36:08.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061937030000000001 2023-06-10T21:36:08.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.842815+0000", 2023-06-10T21:36:08.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6097000000000003e-05 2023-06-10T21:36:08.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.842861+0000", 2023-06-10T21:36:08.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010447279 2023-06-10T21:36:08.366 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:144255 11.c e302/269 11:36e1695b:::benchmark_data_smithi064_385725_object59982:head v 302'8986, mlcod=302'8983)", 2023-06-10T21:36:08.367 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:36:08.368 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 78.154818542000001, 2023-06-10T21:36:08.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013049181999999999, 2023-06-10T21:36:08.368 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.368 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.369 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:36:08.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:36:08.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:36:08.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5439999999999999e-06 2023-06-10T21:36:08.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015657+0000", 2023-06-10T21:36:08.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1619999999999996e-06 2023-06-10T21:36:08.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015664+0000", 2023-06-10T21:36:08.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9319999999999998e-06 2023-06-10T21:36:08.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015667+0000", 2023-06-10T21:36:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044353730000000003 2023-06-10T21:36:08.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.020103+0000", 2023-06-10T21:36:08.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4808999999999999e-05 2023-06-10T21:36:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.020128+0000", 2023-06-10T21:36:08.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085162720000000001 2023-06-10T21:36:08.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.028644+0000", 2023-06-10T21:36:08.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4398000000000003e-05 2023-06-10T21:36:08.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.028698+0000", 2023-06-10T21:36:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013049181999999999 2023-06-10T21:36:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.380 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:151200 11.c e303/269 11:333792e6:::benchmark_data_smithi064_385725_object66927:head v 303'9446, mlcod=303'9444)", 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:36:08.381 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 64.045547948999996, 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013186708, 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.382 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:36:08.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:08.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124919+0000", 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1994000000000001e-05 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124931+0000", 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1949e-05 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124943+0000", 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8090000000000004e-06 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124949+0000", 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2096999999999999e-05 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.125001+0000", 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0669000000000001e-05 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.125022+0000", 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013036641999999999 2023-06-10T21:36:08.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138059+0000", 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7722000000000002e-05 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138106+0000", 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013186708 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:151201 11.c e303/269 11:3f124850:::benchmark_data_smithi064_385725_object66928:head v 303'9447, mlcod=303'9444)", 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 64.044312519000002, 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011978391999999999, 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.391 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.392 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:36:08.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.394 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.394 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:36:08.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:08.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126154+0000", 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2140000000000001e-05 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126166+0000", 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4511e-05 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126181+0000", 2023-06-10T21:36:08.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3780000000000003e-06 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126188+0000", 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048361719999999997 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.131024+0000", 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9647e-05 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.131044+0000", 2023-06-10T21:36:08.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070684169999999996 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138113+0000", 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.105e-05 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138134+0000", 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011978391999999999 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:151928 11.7 e303/269 11:e9886661:::benchmark_data_smithi064_385725_object67655:head v 303'9497, mlcod=303'9493)", 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:05.653954+0000", 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 62.516514188000002, 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010018495000000001, 2023-06-10T21:36:08.401 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653954+0000", 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653954+0000", 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:08.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653953+0000", 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9359999999999998e-06 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653958+0000", 2023-06-10T21:36:08.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2150000000000004e-06 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653965+0000", 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2590000000000002e-06 2023-06-10T21:36:08.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.653969+0000", 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027518210000000002 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:08.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.656721+0000", 2023-06-10T21:36:08.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2025999999999999e-05 2023-06-10T21:36:08.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:08.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.656743+0000", 2023-06-10T21:36:08.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071819900000000001 2023-06-10T21:36:08.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.663925+0000", 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6768999999999999e-05 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:08.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:08.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:08.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:05.663972+0000", 2023-06-10T21:36:08.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010018495000000001 2023-06-10T21:36:08.410 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.411 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.411 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.411 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:08.411 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:08.411 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:08.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:08.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:08.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:08.566 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: osdmap e314: 8 total, 8 up, 6 in 2023-06-10T21:36:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: pgmap v1157: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[180297]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: osdmap e314: 8 total, 8 up, 6 in 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: pgmap v1157: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:08 smithi064 ceph-mon[186706]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: osdmap e314: 8 total, 8 up, 6 in 2023-06-10T21:36:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: from='client.46535 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: pgmap v1157: 89 pgs: 2 creating+peering, 4 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:08 smithi152 ceph-mon[156819]: failed to encode map e314 with expected crc 2023-06-10T21:36:08.861 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:36:08.922 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:08.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:08.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:08.988 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1156,"stamp":"2023-06-10T21:36:07.394423+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":1777362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":255,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":237,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3444648,"kb_used_data":573496,"kb_used_omap":30,"kb_used_meta":2871073,"kb_avail":559026264,"statfs":{"total":575970213888,"available":572442894336,"internally_reserved":0,"allocated":587259904,"data_stored":582050301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2939978790},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.477135"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":306,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.274366+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:36:04.274366+0000","last_peered":"2023-06-10T21:36:04.274366+0000","last_clean":"2023-06-10T21:36:04.274366+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:36:04.274366+0000","last_undegraded":"2023-06-10T21:36:04.274366+0000","last_fullsized":"2023-06-10T21:36:04.274366+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":247,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287103+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:36:04.287103+0000","last_peered":"2023-06-10T21:36:04.287103+0000","last_clean":"2023-06-10T21:36:04.287103+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:36:04.287103+0000","last_undegraded":"2023-06-10T21:36:04.287103+0000","last_fullsized":"2023-06-10T21:36:04.287103+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":222,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286669+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:36:04.286669+0000","last_peered":"2023-06-10T21:36:04.286669+0000","last_clean":"2023-06-10T21:36:04.286669+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:36:04.286669+0000","last_undegraded":"2023-06-10T21:36:04.286669+0000","last_fullsized":"2023-06-10T21:36:04.286669+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":167,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171366+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:35:55.171366+0000","last_peered":"2023-06-10T21:35:55.171366+0000","last_clean":"2023-06-10T21:35:55.171366+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:35:55.171366+0000","last_undegraded":"2023-06-10T21:35:55.171366+0000","last_fullsized":"2023-06-10T21:35:55.171366+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":287,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286012+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:36:04.286012+0000","last_peered":"2023-06-10T21:36:04.286012+0000","last_clean":"2023-06-10T21:36:04.286012+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:36:04.286012+0000","last_undegraded":"2023-06-10T21:36:04.286012+0000","last_fullsized":"2023-06-10T21:36:04.286012+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":277,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171106+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:35:55.171106+0000","last_peered":"2023-06-10T21:35:55.171106+0000","last_clean":"2023-06-10T21:35:55.171106+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:35:55.171106+0000","last_undegraded":"2023-06-10T21:35:55.171106+0000","last_fullsized":"2023-06-10T21:35:55.171106+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":167,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171468+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:35:55.171468+0000","last_peered":"2023-06-10T21:35:55.171468+0000","last_clean":"2023-06-10T21:35:55.171468+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:35:55.171468+0000","last_undegraded":"2023-06-10T21:35:55.171468+0000","last_fullsized":"2023-06-10T21:35:55.171468+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":262,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.285521+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:36:04.285521+0000","last_peered":"2023-06-10T21:36:04.285521+0000","last_clean":"2023-06-10T21:36:04.285521+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:36:04.285521+0000","last_undegraded":"2023-06-10T21:36:04.285521+0000","last_fullsized":"2023-06-10T21:36:04.285521+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":32,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382508+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:35:59.382508+0000","last_peered":"2023-06-10T21:35:59.382508+0000","last_clean":"2023-06-10T21:35:59.382508+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:35:59.382508+0000","last_undegraded":"2023-06-10T21:35:59.382508+0000","last_fullsized":"2023-06-10T21:35:59.382508+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":36,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286329+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:36:04.286329+0000","last_peered":"2023-06-10T21:36:04.286329+0000","last_clean":"2023-06-10T21:36:04.286329+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:36:04.286329+0000","last_undegraded":"2023-06-10T21:36:04.286329+0000","last_fullsized":"2023-06-10T21:36:04.286329+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":38,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382390+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:35:59.382390+0000","last_peered":"2023-06-10T21:35:59.382390+0000","last_clean":"2023-06-10T21:35:59.382390+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:35:59.382390+0000","last_undegraded":"2023-06-10T21:35:59.382390+0000","last_fullsized":"2023-06-10T21:35:59.382390+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":275,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.280491+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:36:03.280491+0000","last_peered":"2023-06-10T21:36:03.280491+0000","last_clean":"2023-06-10T21:36:03.280491+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:36:03.280491+0000","last_undegraded":"2023-06-10T21:36:03.280491+0000","last_fullsized":"2023-06-10T21:36:03.280491+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":181,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382481+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:35:59.382481+0000","last_peered":"2023-06-10T21:35:59.382481+0000","last_clean":"2023-06-10T21:35:59.382481+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:35:59.382481+0000","last_undegraded":"2023-06-10T21:35:59.382481+0000","last_fullsized":"2023-06-10T21:35:59.382481+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":42,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382285+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:35:59.382285+0000","last_peered":"2023-06-10T21:35:59.382285+0000","last_clean":"2023-06-10T21:35:59.382285+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:35:59.382285+0000","last_undegraded":"2023-06-10T21:35:59.382285+0000","last_fullsized":"2023-06-10T21:35:59.382285+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":310,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382302+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:35:59.382302+0000","last_peered":"2023-06-10T21:35:59.382302+0000","last_clean":"2023-06-10T21:35:59.382302+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:35:59.382302+0000","last_undegraded":"2023-06-10T21:35:59.382302+0000","last_fullsized":"2023-06-10T21:35:59.382302+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":248,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287595+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:36:04.287595+0000","last_peered":"2023-06-10T21:36:04.287595+0000","last_clean":"2023-06-10T21:36:04.287595+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:36:04.287595+0000","last_undegraded":"2023-06-10T21:36:04.287595+0000","last_fullsized":"2023-06-10T21:36:04.287595+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":167,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170884+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:35:55.170884+0000","last_peered":"2023-06-10T21:35:55.170884+0000","last_clean":"2023-06-10T21:35:55.170884+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:35:55.170884+0000","last_undegraded":"2023-06-10T21:35:55.170884+0000","last_fullsized":"2023-06-10T21:35:55.170884+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":239,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382615+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:35:59.382615+0000","last_peered":"2023-06-10T21:35:59.382615+0000","last_clean":"2023-06-10T21:35:59.382615+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:35:59.382615+0000","last_undegraded":"2023-06-10T21:35:59.382615+0000","last_fullsized":"2023-06-10T21:35:59.382615+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":287,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382112+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:35:59.382112+0000","last_peered":"2023-06-10T21:35:59.382112+0000","last_clean":"2023-06-10T21:35:59.382112+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:35:59.382112+0000","last_undegraded":"2023-06-10T21:35:59.382112+0000","last_fullsized":"2023-06-10T21:35:59.382112+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171194+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:35:55.171194+0000","last_peered":"2023-06-10T21:35:55.171194+0000","last_clean":"2023-06-10T21:35:55.171194+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:35:55.171194+0000","last_undegraded":"2023-06-10T21:35:55.171194+0000","last_fullsized":"2023-06-10T21:35:55.171194+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":286,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170118+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:35:55.170118+0000","last_peered":"2023-06-10T21:35:55.170118+0000","last_clean":"2023-06-10T21:35:55.170118+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:35:55.170118+0000","last_undegraded":"2023-06-10T21:35:55.170118+0000","last_fullsized":"2023-06-10T21:35:55.170118+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":265,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287258+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:36:04.287258+0000","last_peered":"2023-06-10T21:36:04.287258+0000","last_clean":"2023-06-10T21:36:04.287258+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:36:04.287258+0000","last_undegraded":"2023-06-10T21:36:04.287258+0000","last_fullsized":"2023-06-10T21:36:04.287258+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":10,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.326058+0000","last_change":"2023-06-10T21:36:04.325572+0000","last_active":"2023-06-10T21:36:04.326058+0000","last_peered":"2023-06-10T21:36:04.326058+0000","last_clean":"2023-06-10T21:36:04.326058+0000","last_became_active":"2023-06-10T21:36:04.325154+0000","last_became_peered":"2023-06-10T21:36:04.325154+0000","last_unstale":"2023-06-10T21:36:04.326058+0000","last_undegraded":"2023-06-10T21:36:04.326058+0000","last_fullsized":"2023-06-10T21:36:04.326058+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:53.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":36,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382671+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:35:59.382671+0000","last_peered":"2023-06-10T21:35:59.382671+0000","last_clean":"2023-06-10T21:35:59.382671+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:35:59.382671+0000","last_undegraded":"2023-06-10T21:35:59.382671+0000","last_fullsized":"2023-06-10T21:35:59.382671+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":32,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.381887+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:35:59.381887+0000","last_peered":"2023-06-10T21:35:59.381887+0000","last_clean":"2023-06-10T21:35:59.381887+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:35:59.381887+0000","last_undegraded":"2023-06-10T21:35:59.381887+0000","last_fullsized":"2023-06-10T21:35:59.381887+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":11,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-06-10T21:36:05.281203+0000","last_change":"2023-06-10T21:36:04.292881+0000","last_active":"2023-06-10T21:36:05.281203+0000","last_peered":"2023-06-10T21:36:05.281203+0000","last_clean":"2023-06-10T21:36:05.281203+0000","last_became_active":"2023-06-10T21:36:04.292606+0000","last_became_peered":"2023-06-10T21:36:04.292606+0000","last_unstale":"2023-06-10T21:36:05.281203+0000","last_undegraded":"2023-06-10T21:36:05.281203+0000","last_fullsized":"2023-06-10T21:36:05.281203+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:21.334865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":36,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382565+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:35:59.382565+0000","last_peered":"2023-06-10T21:35:59.382565+0000","last_clean":"2023-06-10T21:35:59.382565+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:35:59.382565+0000","last_undegraded":"2023-06-10T21:35:59.382565+0000","last_fullsized":"2023-06-10T21:35:59.382565+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286779+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:36:04.286779+0000","last_peered":"2023-06-10T21:36:04.286779+0000","last_clean":"2023-06-10T21:36:04.286779+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:36:04.286779+0000","last_undegraded":"2023-06-10T21:36:04.286779+0000","last_fullsized":"2023-06-10T21:36:04.286779+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"0'0","reported_seq":11,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-06-10T21:36:05.281149+0000","last_change":"2023-06-10T21:36:04.325459+0000","last_active":"2023-06-10T21:36:05.281149+0000","last_peered":"2023-06-10T21:36:05.281149+0000","last_clean":"2023-06-10T21:36:05.281149+0000","last_became_active":"2023-06-10T21:36:04.325111+0000","last_became_peered":"2023-06-10T21:36:04.325111+0000","last_unstale":"2023-06-10T21:36:05.281149+0000","last_undegraded":"2023-06-10T21:36:05.281149+0000","last_fullsized":"2023-06-10T21:36:05.281149+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:49:06.786091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":318,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382466+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:35:59.382466+0000","last_peered":"2023-06-10T21:35:59.382466+0000","last_clean":"2023-06-10T21:35:59.382466+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:35:59.382466+0000","last_undegraded":"2023-06-10T21:35:59.382466+0000","last_fullsized":"2023-06-10T21:35:59.382466+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":32,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382102+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:35:59.382102+0000","last_peered":"2023-06-10T21:35:59.382102+0000","last_clean":"2023-06-10T21:35:59.382102+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:35:59.382102+0000","last_undegraded":"2023-06-10T21:35:59.382102+0000","last_fullsized":"2023-06-10T21:35:59.382102+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.262449+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":195,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287678+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:36:04.287678+0000","last_peered":"2023-06-10T21:36:04.287678+0000","last_clean":"2023-06-10T21:36:04.287678+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:36:04.287678+0000","last_undegraded":"2023-06-10T21:36:04.287678+0000","last_fullsized":"2023-06-10T21:36:04.287678+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":134,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170236+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:35:55.170236+0000","last_peered":"2023-06-10T21:35:55.170236+0000","last_clean":"2023-06-10T21:35:55.170236+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:35:55.170236+0000","last_undegraded":"2023-06-10T21:35:55.170236+0000","last_fullsized":"2023-06-10T21:35:55.170236+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":11,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-06-10T21:36:05.281478+0000","last_change":"2023-06-10T21:36:04.288539+0000","last_active":"2023-06-10T21:36:05.281478+0000","last_peered":"2023-06-10T21:36:05.281478+0000","last_clean":"2023-06-10T21:36:05.281478+0000","last_became_active":"2023-06-10T21:36:04.288222+0000","last_became_peered":"2023-06-10T21:36:04.288222+0000","last_unstale":"2023-06-10T21:36:05.281478+0000","last_undegraded":"2023-06-10T21:36:05.281478+0000","last_fullsized":"2023-06-10T21:36:05.281478+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:53.556650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":265,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.288693+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:36:04.288693+0000","last_peered":"2023-06-10T21:36:04.288693+0000","last_clean":"2023-06-10T21:36:04.288693+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:36:04.288693+0000","last_undegraded":"2023-06-10T21:36:04.288693+0000","last_fullsized":"2023-06-10T21:36:04.288693+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":185,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287733+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:36:04.287733+0000","last_peered":"2023-06-10T21:36:04.287733+0000","last_clean":"2023-06-10T21:36:04.287733+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:36:04.287733+0000","last_undegraded":"2023-06-10T21:36:04.287733+0000","last_fullsized":"2023-06-10T21:36:04.287733+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":10,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.292388+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:36:04.292388+0000","last_peered":"2023-06-10T21:36:04.292388+0000","last_clean":"2023-06-10T21:36:04.292388+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:36:04.292388+0000","last_undegraded":"2023-06-10T21:36:04.292388+0000","last_fullsized":"2023-06-10T21:36:04.292388+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286525+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:36:04.286525+0000","last_peered":"2023-06-10T21:36:04.286525+0000","last_clean":"2023-06-10T21:36:04.286525+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:36:04.286525+0000","last_undegraded":"2023-06-10T21:36:04.286525+0000","last_fullsized":"2023-06-10T21:36:04.286525+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":304,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.278639+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:36:03.278639+0000","last_peered":"2023-06-10T21:36:03.278639+0000","last_clean":"2023-06-10T21:36:03.278639+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:36:03.278639+0000","last_undegraded":"2023-06-10T21:36:03.278639+0000","last_fullsized":"2023-06-10T21:36:03.278639+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.262449+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":290,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.278415+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:36:03.278415+0000","last_peered":"2023-06-10T21:36:03.278415+0000","last_clean":"2023-06-10T21:36:03.278415+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:36:03.278415+0000","last_undegraded":"2023-06-10T21:36:03.278415+0000","last_fullsized":"2023-06-10T21:36:03.278415+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287185+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:36:04.287185+0000","last_peered":"2023-06-10T21:36:04.287185+0000","last_clean":"2023-06-10T21:36:04.287185+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:36:04.287185+0000","last_undegraded":"2023-06-10T21:36:04.287185+0000","last_fullsized":"2023-06-10T21:36:04.287185+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.283615+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":268,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.285868+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:36:04.285868+0000","last_peered":"2023-06-10T21:36:04.285868+0000","last_clean":"2023-06-10T21:36:04.285868+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:36:04.285868+0000","last_undegraded":"2023-06-10T21:36:04.285868+0000","last_fullsized":"2023-06-10T21:36:04.285868+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":224,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287496+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:36:04.287496+0000","last_peered":"2023-06-10T21:36:04.287496+0000","last_clean":"2023-06-10T21:36:04.287496+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:36:04.287496+0000","last_undegraded":"2023-06-10T21:36:04.287496+0000","last_fullsized":"2023-06-10T21:36:04.287496+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":10,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.292009+0000","last_change":"2023-06-10T21:36:04.291075+0000","last_active":"2023-06-10T21:36:04.292009+0000","last_peered":"2023-06-10T21:36:04.292009+0000","last_clean":"2023-06-10T21:36:04.292009+0000","last_became_active":"2023-06-10T21:36:04.290808+0000","last_became_peered":"2023-06-10T21:36:04.290808+0000","last_unstale":"2023-06-10T21:36:04.292009+0000","last_undegraded":"2023-06-10T21:36:04.292009+0000","last_fullsized":"2023-06-10T21:36:04.292009+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:13:41.537032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170973+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:35:55.170973+0000","last_peered":"2023-06-10T21:35:55.170973+0000","last_clean":"2023-06-10T21:35:55.170973+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:35:55.170973+0000","last_undegraded":"2023-06-10T21:35:55.170973+0000","last_fullsized":"2023-06-10T21:35:55.170973+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":317,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287217+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:36:04.287217+0000","last_peered":"2023-06-10T21:36:04.287217+0000","last_clean":"2023-06-10T21:36:04.287217+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:36:04.287217+0000","last_undegraded":"2023-06-10T21:36:04.287217+0000","last_fullsized":"2023-06-10T21:36:04.287217+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":154,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170702+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:35:55.170702+0000","last_peered":"2023-06-10T21:35:55.170702+0000","last_clean":"2023-06-10T21:35:55.170702+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:35:55.170702+0000","last_undegraded":"2023-06-10T21:35:55.170702+0000","last_fullsized":"2023-06-10T21:35:55.170702+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.262449+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":180,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170797+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:35:55.170797+0000","last_peered":"2023-06-10T21:35:55.170797+0000","last_clean":"2023-06-10T21:35:55.170797+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:35:55.170797+0000","last_undegraded":"2023-06-10T21:35:55.170797+0000","last_fullsized":"2023-06-10T21:35:55.170797+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170404+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:35:55.170404+0000","last_peered":"2023-06-10T21:35:55.170404+0000","last_clean":"2023-06-10T21:35:55.170404+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:35:55.170404+0000","last_undegraded":"2023-06-10T21:35:55.170404+0000","last_fullsized":"2023-06-10T21:35:55.170404+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":146,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.278858+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:36:03.278858+0000","last_peered":"2023-06-10T21:36:03.278858+0000","last_clean":"2023-06-10T21:36:03.278858+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:36:03.278858+0000","last_undegraded":"2023-06-10T21:36:03.278858+0000","last_fullsized":"2023-06-10T21:36:03.278858+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":10,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.325911+0000","last_change":"2023-06-10T21:36:04.325488+0000","last_active":"2023-06-10T21:36:04.325911+0000","last_peered":"2023-06-10T21:36:04.325911+0000","last_clean":"2023-06-10T21:36:04.325911+0000","last_became_active":"2023-06-10T21:36:04.325162+0000","last_became_peered":"2023-06-10T21:36:04.325162+0000","last_unstale":"2023-06-10T21:36:04.325911+0000","last_undegraded":"2023-06-10T21:36:04.325911+0000","last_fullsized":"2023-06-10T21:36:04.325911+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:41:33.200826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":157,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.278484+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:36:03.278484+0000","last_peered":"2023-06-10T21:36:03.278484+0000","last_clean":"2023-06-10T21:36:03.278484+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:36:03.278484+0000","last_undegraded":"2023-06-10T21:36:03.278484+0000","last_fullsized":"2023-06-10T21:36:03.278484+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":155,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.280129+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:36:03.280129+0000","last_peered":"2023-06-10T21:36:03.280129+0000","last_clean":"2023-06-10T21:36:03.280129+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:36:03.280129+0000","last_undegraded":"2023-06-10T21:36:03.280129+0000","last_fullsized":"2023-06-10T21:36:03.280129+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171508+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:35:55.171508+0000","last_peered":"2023-06-10T21:35:55.171508+0000","last_clean":"2023-06-10T21:35:55.171508+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:35:55.171508+0000","last_undegraded":"2023-06-10T21:35:55.171508+0000","last_fullsized":"2023-06-10T21:35:55.171508+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.283567+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":336,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.277904+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:36:03.277904+0000","last_peered":"2023-06-10T21:36:03.277904+0000","last_clean":"2023-06-10T21:36:03.277904+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:36:03.277904+0000","last_undegraded":"2023-06-10T21:36:03.277904+0000","last_fullsized":"2023-06-10T21:36:03.277904+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":474,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.381917+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:35:59.381917+0000","last_peered":"2023-06-10T21:35:59.381917+0000","last_clean":"2023-06-10T21:35:59.381917+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:35:59.381917+0000","last_undegraded":"2023-06-10T21:35:59.381917+0000","last_fullsized":"2023-06-10T21:35:59.381917+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":251,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286704+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:36:04.286704+0000","last_peered":"2023-06-10T21:36:04.286704+0000","last_clean":"2023-06-10T21:36:04.286704+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:36:04.286704+0000","last_undegraded":"2023-06-10T21:36:04.286704+0000","last_fullsized":"2023-06-10T21:36:04.286704+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382578+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:35:59.382578+0000","last_peered":"2023-06-10T21:35:59.382578+0000","last_clean":"2023-06-10T21:35:59.382578+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:35:59.382578+0000","last_undegraded":"2023-06-10T21:35:59.382578+0000","last_fullsized":"2023-06-10T21:35:59.382578+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:36:03.262449+0000","last_change":"2023-06-10T21:36:03.262449+0000","last_active":"2023-06-10T21:36:03.262449+0000","last_peered":"2023-06-10T21:36:03.262449+0000","last_clean":"2023-06-10T21:36:03.262449+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:36:03.262449+0000","last_undegraded":"2023-06-10T21:36:03.262449+0000","last_fullsized":"2023-06-10T21:36:03.262449+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":330,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382019+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:35:59.382019+0000","last_peered":"2023-06-10T21:35:59.382019+0000","last_clean":"2023-06-10T21:35:59.382019+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:35:59.382019+0000","last_undegraded":"2023-06-10T21:35:59.382019+0000","last_fullsized":"2023-06-10T21:35:59.382019+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":149,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170498+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:35:55.170498+0000","last_peered":"2023-06-10T21:35:55.170498+0000","last_clean":"2023-06-10T21:35:55.170498+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:35:55.170498+0000","last_undegraded":"2023-06-10T21:35:55.170498+0000","last_fullsized":"2023-06-10T21:35:55.170498+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":134,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171117+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:35:55.171117+0000","last_peered":"2023-06-10T21:35:55.171117+0000","last_clean":"2023-06-10T21:35:55.171117+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:35:55.171117+0000","last_undegraded":"2023-06-10T21:35:55.171117+0000","last_fullsized":"2023-06-10T21:35:55.171117+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"0'0","reported_seq":11,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-06-10T21:36:05.281235+0000","last_change":"2023-06-10T21:36:04.325466+0000","last_active":"2023-06-10T21:36:05.281235+0000","last_peered":"2023-06-10T21:36:05.281235+0000","last_clean":"2023-06-10T21:36:05.281235+0000","last_became_active":"2023-06-10T21:36:04.325178+0000","last_became_peered":"2023-06-10T21:36:04.325178+0000","last_unstale":"2023-06-10T21:36:05.281235+0000","last_undegraded":"2023-06-10T21:36:05.281235+0000","last_fullsized":"2023-06-10T21:36:05.281235+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:03.551659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":315,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382505+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:35:59.382505+0000","last_peered":"2023-06-10T21:35:59.382505+0000","last_clean":"2023-06-10T21:35:59.382505+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:35:59.382505+0000","last_undegraded":"2023-06-10T21:35:59.382505+0000","last_fullsized":"2023-06-10T21:35:59.382505+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":155,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.278937+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:36:03.278937+0000","last_peered":"2023-06-10T21:36:03.278937+0000","last_clean":"2023-06-10T21:36:03.278937+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:36:03.278937+0000","last_undegraded":"2023-06-10T21:36:03.278937+0000","last_fullsized":"2023-06-10T21:36:03.278937+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":130,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170667+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:35:55.170667+0000","last_peered":"2023-06-10T21:35:55.170667+0000","last_clean":"2023-06-10T21:35:55.170667+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:35:55.170667+0000","last_undegraded":"2023-06-10T21:35:55.170667+0000","last_fullsized":"2023-06-10T21:35:55.170667+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"0'0","reported_seq":11,"reported_epoch":313,"state":"active+clean","last_fresh":"2023-06-10T21:36:05.281002+0000","last_change":"2023-06-10T21:36:04.292180+0000","last_active":"2023-06-10T21:36:05.281002+0000","last_peered":"2023-06-10T21:36:05.281002+0000","last_clean":"2023-06-10T21:36:05.281002+0000","last_became_active":"2023-06-10T21:36:04.291901+0000","last_became_peered":"2023-06-10T21:36:04.291901+0000","last_unstale":"2023-06-10T21:36:05.281002+0000","last_undegraded":"2023-06-10T21:36:05.281002+0000","last_fullsized":"2023-06-10T21:36:05.281002+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:45.649615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":291,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287328+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:36:04.287328+0000","last_peered":"2023-06-10T21:36:04.287328+0000","last_clean":"2023-06-10T21:36:04.287328+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:36:04.287328+0000","last_undegraded":"2023-06-10T21:36:04.287328+0000","last_fullsized":"2023-06-10T21:36:04.287328+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":88,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.287062+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:36:04.287062+0000","last_peered":"2023-06-10T21:36:04.287062+0000","last_clean":"2023-06-10T21:36:04.287062+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:36:04.287062+0000","last_undegraded":"2023-06-10T21:36:04.287062+0000","last_fullsized":"2023-06-10T21:36:04.287062+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":272,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.285784+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:36:04.285784+0000","last_peered":"2023-06-10T21:36:04.285784+0000","last_clean":"2023-06-10T21:36:04.285784+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:36:04.285784+0000","last_undegraded":"2023-06-10T21:36:04.285784+0000","last_fullsized":"2023-06-10T21:36:04.285784+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":10,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.297440+0000","last_change":"2023-06-10T21:36:04.296839+0000","last_active":"2023-06-10T21:36:04.297440+0000","last_peered":"2023-06-10T21:36:04.297440+0000","last_clean":"2023-06-10T21:36:04.297440+0000","last_became_active":"2023-06-10T21:36:04.294647+0000","last_became_peered":"2023-06-10T21:36:04.294647+0000","last_unstale":"2023-06-10T21:36:04.297440+0000","last_undegraded":"2023-06-10T21:36:04.297440+0000","last_fullsized":"2023-06-10T21:36:04.297440+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:16.069639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":32,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382617+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:35:59.382617+0000","last_peered":"2023-06-10T21:35:59.382617+0000","last_clean":"2023-06-10T21:35:59.382617+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:35:59.382617+0000","last_undegraded":"2023-06-10T21:35:59.382617+0000","last_fullsized":"2023-06-10T21:35:59.382617+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":138,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171282+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:35:55.171282+0000","last_peered":"2023-06-10T21:35:55.171282+0000","last_clean":"2023-06-10T21:35:55.171282+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:35:55.171282+0000","last_undegraded":"2023-06-10T21:35:55.171282+0000","last_fullsized":"2023-06-10T21:35:55.171282+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":266,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.170069+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:35:55.170069+0000","last_peered":"2023-06-10T21:35:55.170069+0000","last_clean":"2023-06-10T21:35:55.170069+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:35:55.170069+0000","last_undegraded":"2023-06-10T21:35:55.170069+0000","last_fullsized":"2023-06-10T21:35:55.170069+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":134,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.378559+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:35:59.378559+0000","last_peered":"2023-06-10T21:35:59.378559+0000","last_clean":"2023-06-10T21:35:59.378559+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:35:59.378559+0000","last_undegraded":"2023-06-10T21:35:59.378559+0000","last_fullsized":"2023-06-10T21:35:59.378559+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":154,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.279012+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:36:03.279012+0000","last_peered":"2023-06-10T21:36:03.279012+0000","last_clean":"2023-06-10T21:36:03.279012+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:36:03.279012+0000","last_undegraded":"2023-06-10T21:36:03.279012+0000","last_fullsized":"2023-06-10T21:36:03.279012+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":36,"reported_epoch":310,"state":"active+clean","last_fresh":"2023-06-10T21:35:59.382329+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:35:59.382329+0000","last_peered":"2023-06-10T21:35:59.382329+0000","last_clean":"2023-06-10T21:35:59.382329+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:35:59.382329+0000","last_undegraded":"2023-06-10T21:35:59.382329+0000","last_fullsized":"2023-06-10T21:35:59.382329+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.288530+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:36:04.288530+0000","last_peered":"2023-06-10T21:36:04.288530+0000","last_clean":"2023-06-10T21:36:04.288530+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:36:04.288530+0000","last_undegraded":"2023-06-10T21:36:04.288530+0000","last_fullsized":"2023-06-10T21:36:04.288530+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":91,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286938+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:36:04.286938+0000","last_peered":"2023-06-10T21:36:04.286938+0000","last_clean":"2023-06-10T21:36:04.286938+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:36:04.286938+0000","last_undegraded":"2023-06-10T21:36:04.286938+0000","last_fullsized":"2023-06-10T21:36:04.286938+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286609+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:36:04.286609+0000","last_peered":"2023-06-10T21:36:04.286609+0000","last_clean":"2023-06-10T21:36:04.286609+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:36:04.286609+0000","last_undegraded":"2023-06-10T21:36:04.286609+0000","last_fullsized":"2023-06-10T21:36:04.286609+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":297,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286795+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:36:04.286795+0000","last_peered":"2023-06-10T21:36:04.286795+0000","last_clean":"2023-06-10T21:36:04.286795+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:36:04.286795+0000","last_undegraded":"2023-06-10T21:36:04.286795+0000","last_fullsized":"2023-06-10T21:36:04.286795+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":300,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-06-10T21:36:04.286472+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:36:04.286472+0000","last_peered":"2023-06-10T21:36:04.286472+0000","last_clean":"2023-06-10T21:36:04.286472+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:36:04.286472+0000","last_undegraded":"2023-06-10T21:36:04.286472+0000","last_fullsized":"2023-06-10T21:36:04.286472+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":199,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-06-10T21:35:55.171551+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:35:55.171551+0000","last_peered":"2023-06-10T21:35:55.171551+0000","last_clean":"2023-06-10T21:35:55.171551+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:35:55.171551+0000","last_undegraded":"2023-06-10T21:35:55.171551+0000","last_fullsized":"2023-06-10T21:35:55.171551+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":45,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-06-10T21:36:03.279679+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:36:03.279679+0000","last_peered":"2023-06-10T21:36:03.279679+0000","last_clean":"2023-06-10T21:36:03.279679+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:36:03.279679+0000","last_undegraded":"2023-06-10T21:36:03.279679+0000","last_fullsized":"2023-06-10T21:36:03.279679+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":36,"acting":36,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430668,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":608804,"kb_used_data":3292,"kb_used_omap":0,"kb_used_meta":605504,"kb_avail":93136348,"statfs":{"total":95995035648,"available":95371620352,"internally_reserved":0,"allocated":3371008,"data_stored":2799369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":620036096},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":4.5199999999999996,"15min":7.4340000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":28.875,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":3.734,"5min":4.7830000000000004,"15min":7.7130000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":28.943000000000001,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":3.2210000000000001,"15min":3.2930000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":4.4669999999999996,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.0649999999999999,"5min":3.6539999999999999,"15min":3.8570000000000002},"min":{"1min":0.35599999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":8.1020000000000003,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":6.5759999999999996,"5min":7.1050000000000004,"15min":4.5890000000000004},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":103.866,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":7.5999999999999996,"5min":7.4850000000000003,"15min":5.3200000000000003},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":103.705,"5min":172.185,"15min":212.13399999999999},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.2229999999999999,"5min":4.218,"15min":3.0720000000000001},"min":{"1min":0.375,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":23.611999999999998,"5min":99.266000000000005,"15min":99.266000000000005},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.6800000000000002,"5min":3.883,"15min":2.516},"min":{"1min":0.36399999999999999,"5min":0.31,"15min":0.31},"max":{"1min":23.591999999999999,"5min":99.138000000000005,"15min":99.138000000000005},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39000000000000001,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.59499999999999997,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.436,"5min":0.438,"15min":1.0209999999999999},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.66600000000000004,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":1.8120000000000001},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.68700000000000006,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.45400000000000001,"15min":1.607},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.29299999999999998},"max":{"1min":0.68600000000000005,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.60899999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.50900000000000001,"15min":1.069},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":3.2559999999999998,"15min":13.073},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.48499999999999999,"15min":0.92900000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":1.1879999999999999,"15min":10.874000000000001},"last":0.63700000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626896,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566339,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990848,"kb_used_data":329848,"kb_used_omap":4,"kb_used_meta":660987,"kb_avail":92754304,"statfs":{"total":95995035648,"available":94980407296,"internally_reserved":0,"allocated":337764352,"data_stored":336778218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":676851598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:48 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":4.9550000000000001,"15min":10.315},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":28.507999999999999,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":3.714,"5min":4.5700000000000003,"15min":10.029},"min":{"1min":0.27300000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":28.045999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":11.297000000000001,"5min":9.1150000000000002,"15min":5.7110000000000003},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":134.79900000000001,"5min":191.101,"15min":191.101},"last":0.29699999999999999},{"interface":"front","average":{"1min":4.8479999999999999,"5min":7.702,"15min":5.0469999999999997},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.218},"max":{"1min":15.923,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":3.9079999999999999,"5min":4.7329999999999997,"15min":2.895},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.23799999999999999},"max":{"1min":15.130000000000001,"5min":80.584999999999994,"15min":80.584999999999994},"last":0.439},{"interface":"front","average":{"1min":3.593,"5min":5.3479999999999999,"15min":3.9039999999999999},"min":{"1min":0.35199999999999998,"5min":0.24299999999999999,"15min":0.23999999999999999},"max":{"1min":10.961,"5min":80.873999999999995,"15min":80.873999999999995},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":1.355,"15min":1.548},"min":{"1min":0.35799999999999998,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":9.3659999999999997,"5min":10.036,"15min":16.059999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.0680000000000001,"5min":1.589,"15min":1.452},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":8.8629999999999995,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0720000000000001,"5min":2.0550000000000002,"15min":4.2169999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.035,"5min":10.035,"15min":383.83100000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.3680000000000001,"15min":3.8050000000000002},"min":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":9.3160000000000007,"5min":10.116,"15min":383.84800000000001},"last":0.61399999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.069,"15min":2.1139999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":9.9960000000000004,"5min":9.9960000000000004,"15min":14.858000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.4990000000000001,"15min":1.7210000000000001},"min":{"1min":0.41699999999999998,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":9.3849999999999998,"5min":10.066000000000001,"15min":13.48},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:35:14 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":2.0579999999999998,"15min":2.661},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":9.343,"5min":47.844000000000001,"15min":162.399},"last":0.64100000000000001},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.4369999999999998,"15min":2.476},"min":{"1min":0.36199999999999999,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":8.8360000000000003,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.629}]}]},{"osd":0,"up_from":244,"seq":1047972020315,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265580,"kb_used_data":3812,"kb_used_omap":10,"kb_used_meta":261749,"kb_avail":93479572,"statfs":{"total":95995035648,"available":95723081728,"internally_reserved":0,"allocated":3903488,"data_stored":2853038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":268031028},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":4.0869999999999997,"15min":4.391},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":4.6500000000000004,"5min":139.68000000000001,"15min":139.68000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":4.5839999999999996,"5min":4.5140000000000002,"15min":5.4619999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":15.403,"5min":141.79499999999999,"15min":141.79499999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2569999999999997,"5min":7.1600000000000001,"15min":6.7930000000000001},"min":{"1min":0.31,"5min":0.216,"15min":0.216},"max":{"1min":24.166,"5min":132.245,"15min":205.10900000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":4.0499999999999998,"5min":6.7140000000000004,"15min":6.2770000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":13.06,"5min":132.26599999999999,"15min":208.34299999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:35:12 2023","interfaces":[{"interface":"back","average":{"1min":4.2629999999999999,"5min":4.1340000000000003,"15min":5.569},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.934999999999999,"5min":111.745,"15min":111.745},"last":0.376},{"interface":"front","average":{"1min":4.29,"5min":4.9589999999999996,"15min":6.5090000000000003},"min":{"1min":0.34300000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":12.438000000000001,"5min":114.205,"15min":114.205},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.9710000000000001,"15min":3.9460000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":8.9550000000000001,"5min":10.744999999999999,"15min":29.709},"last":0.68000000000000005},{"interface":"front","average":{"1min":3.4049999999999998,"5min":2.2440000000000002,"15min":3.5110000000000001},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":24.222999999999999,"5min":24.222999999999999,"15min":24.222999999999999},"last":0.69899999999999995}]},{"osd":5,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":2.2639999999999998,"15min":3.5859999999999999},"min":{"1min":0.438,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":24.288,"5min":24.288,"15min":24.288},"last":0.77600000000000002},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.8169999999999999,"15min":3.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":6.9870000000000001,"5min":8.0079999999999991,"15min":19.846},"last":0.80000000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.056,"15min":4.1100000000000003},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":9,"5min":10.771000000000001,"15min":29.806000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.4430000000000001,"5min":2.3839999999999999,"15min":3.6509999999999998},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":24.260999999999999,"5min":24.260999999999999,"15min":24.260999999999999},"last":0.72899999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.0449999999999999,"15min":3.3839999999999999},"min":{"1min":0.48199999999999998,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.0149999999999997,"5min":17.523,"15min":19.866},"last":0.75800000000000001},{"interface":"front","average":{"1min":2.843,"5min":2.2120000000000002,"15min":4.1609999999999996},"min":{"1min":0.44800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0259999999999998,"5min":17.792999999999999,"15min":29.77},"last":0.70999999999999996}]}]},{"osd":2,"up_from":120,"seq":515396075727,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494628,"kb_used_data":3164,"kb_used_omap":7,"kb_used_meta":491448,"kb_avail":93250524,"statfs":{"total":95995035648,"available":95488536576,"internally_reserved":0,"allocated":3239936,"data_stored":2265442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":503242833},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7000000000000002,"5min":4.6079999999999997,"15min":4.694},"min":{"1min":0.217,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.9,"5min":70.510999999999996,"15min":70.510999999999996},"last":0.432},{"interface":"front","average":{"1min":5.0209999999999999,"5min":4.4329999999999998,"15min":5.1479999999999997},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":11.733000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":5.0979999999999999,"15min":3.2799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.19},"max":{"1min":7.7910000000000004,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":4.3780000000000001,"5min":4.968,"15min":3.0059999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.214},"max":{"1min":9.4619999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:35:07 2023","interfaces":[{"interface":"back","average":{"1min":13.648,"5min":6.2619999999999996,"15min":5.5810000000000004},"min":{"1min":0.314,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":97.296000000000006,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":12.912000000000001,"5min":6.1219999999999999,"15min":6.2359999999999998},"min":{"1min":0.28999999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":97.308999999999997,"5min":138.095,"15min":138.095},"last":0.376}]},{"osd":4,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6879999999999999,"15min":2.976},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":224.00999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.831,"5min":2.3029999999999999,"15min":3.3260000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":7.7030000000000003,"5min":29.486999999999998,"15min":223.28399999999999},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.681,"15min":1.671},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":5.4470000000000001,"5min":10.554,"15min":14.180999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.7879999999999998,"5min":1.73,"15min":1.8939999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":7.3200000000000003,"5min":7.3200000000000003,"15min":16.125},"last":0.501}]},{"osd":6,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8140000000000001,"5min":1.802,"15min":1.966},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":7.4180000000000001,"5min":7.4180000000000001,"15min":16.183},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.1339999999999999,"5min":2.4239999999999999,"15min":2.3500000000000001},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.7510000000000003,"5min":29.538,"15min":29.538},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":3.4630000000000001,"15min":2.6429999999999998},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.8399999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.4340000000000002,"15min":2.9390000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.3620000000000001,"5min":73.475999999999999,"15min":113.937},"last":0.56000000000000005}]}]},{"osd":3,"up_from":250,"seq":1073741824087,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476816,"kb_used_data":230664,"kb_used_omap":7,"kb_used_meta":246136,"kb_avail":93268336,"statfs":{"total":95995035648,"available":95506776064,"internally_reserved":0,"allocated":236199936,"data_stored":235138394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":252043283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":8.3510000000000009,"5min":4.8239999999999998,"15min":5.3070000000000004},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":105.09,"5min":105.09,"15min":105.09},"last":0.36899999999999999},{"interface":"front","average":{"1min":8.1940000000000008,"5min":4.4980000000000002,"15min":5.0540000000000003},"min":{"1min":0.51200000000000001,"5min":0.246,"15min":0.216},"max":{"1min":99.852999999999994,"5min":99.852999999999994,"15min":99.852999999999994},"last":0.33600000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.0459999999999994,"5min":5.194,"15min":4.157},"min":{"1min":0.25800000000000001,"5min":0.248,"15min":0.23100000000000001},"max":{"1min":94.766000000000005,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.27200000000000002},{"interface":"front","average":{"1min":10.343,"5min":4.6310000000000002,"15min":3.867},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":94.691000000000003,"5min":94.691000000000003,"15min":94.691000000000003},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":9.5120000000000005,"5min":3.992,"15min":9.9450000000000003},"min":{"1min":0.33400000000000002,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":100.676,"5min":100.676,"15min":100.676},"last":0.38800000000000001},{"interface":"front","average":{"1min":10.388,"5min":5.7539999999999996,"15min":11.48},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":103.032,"5min":103.032,"15min":103.032},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.575,"15min":1.4410000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":5.0170000000000003,"5min":13.417,"15min":13.417},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.8260000000000001,"15min":2.7679999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.616,"5min":19.728000000000002,"15min":19.728000000000002},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.266,"5min":1.5569999999999999,"15min":1.4470000000000001},"min":{"1min":0.35699999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.7889999999999997,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.71,"15min":1.5229999999999999},"min":{"1min":0.38800000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.0629999999999997,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.67300000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":2.847,"15min":2.7909999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":10.537000000000001,"5min":19.805,"15min":19.805},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6140000000000001,"15min":1.498},"min":{"1min":0.433,"5min":0.433,"15min":0.39100000000000001},"max":{"1min":9.7289999999999992,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:35:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.524,"15min":3.0259999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.648999999999999,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.3050000000000002,"5min":2.2850000000000001,"15min":1.704},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.36699999999999999},"max":{"1min":9.8239999999999998,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.56499999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019352,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607972,"kb_used_data":2716,"kb_used_omap":0,"kb_used_meta":605248,"kb_avail":93137180,"statfs":{"total":95995035648,"available":95372472320,"internally_reserved":0,"allocated":2781184,"data_stored":2215840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":619773952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":4.1260000000000003,"15min":7.899},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.503,"5min":234.203,"15min":234.203},"last":0.77200000000000002},{"interface":"front","average":{"1min":3.2770000000000001,"5min":4.3730000000000002,"15min":7.8220000000000001},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":37.399999999999999,"5min":234.19,"15min":234.19},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":8.7840000000000007,"5min":3.121,"15min":3.468},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":88.700999999999993,"5min":88.700999999999993,"15min":182.035},"last":0.69699999999999995},{"interface":"front","average":{"1min":8.6110000000000007,"5min":3.3130000000000002,"15min":3.375},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":87.853999999999999,"5min":87.853999999999999,"15min":181.73099999999999},"last":0.624}]},{"osd":2,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":7.6150000000000002,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.308999999999997,"5min":255.386,"15min":255.386},"last":0.749},{"interface":"front","average":{"1min":6.4720000000000004,"5min":7.5499999999999998,"15min":3.6850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":85.322000000000003,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.67000000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":17.739000000000001,"5min":7.5629999999999997,"15min":4.1150000000000002},"min":{"1min":0.57699999999999996,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":102.94,"5min":234.91900000000001,"15min":234.91900000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":12.029999999999999,"5min":8.1590000000000007,"15min":4.0819999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":90.489999999999995,"5min":235.05099999999999,"15min":235.05099999999999},"last":0.74099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.379,"15min":0.99299999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.54300000000000004,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.45700000000000002,"15min":1.004},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.45500000000000002,"5min":5.25,"15min":11.904999999999999},"last":0.56699999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.43099999999999999,"15min":1.0249999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25600000000000001},"max":{"1min":0.53900000000000003,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.432,"5min":0.46600000000000003,"15min":1.0900000000000001},"min":{"1min":0.35199999999999998,"5min":0.34000000000000002,"15min":0.309},"max":{"1min":0.59499999999999997,"5min":1.71,"15min":13.845000000000001},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.77500000000000002,"15min":1.2789999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":0.629,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.79600000000000004,"15min":1.2210000000000001},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.57199999999999995,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.68500000000000005}]}]},{"osd":5,"up_from":0,"seq":154618823124,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:36:08.995 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:36:09.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:09.067 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:09.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:09.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:09.213 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:09.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:09.248 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:09.532 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:09.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:09 smithi064 ceph-mon[186706]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:09 smithi064 ceph-mon[186706]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:09 smithi064 ceph-mon[180297]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:09 smithi064 ceph-mon[180297]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:09.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:09.637 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:09.642 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:09.661 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:09 smithi152 ceph-mon[156819]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:09 smithi152 ceph-mon[156819]: failed to encode map e314 with expected crc 2023-06-10T21:36:09.664 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:09.668 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:09.809 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:09.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:09.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:09.939 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:10.169 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:10.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:10.274 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:10.309 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:10.512 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:10 smithi152 ceph-mon[156819]: pgmap v1158: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:10 smithi064 ceph-mon[180297]: pgmap v1158: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:10 smithi064 ceph-mon[186706]: pgmap v1158: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:10.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:36:10.688 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:36:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:36:10.689 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:11.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:36:11.423 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-06-10T21:36:11.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/525189005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/525189005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.642 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:11.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:11.673 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/525189005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:36:11.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:11.774 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:11.867 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:11.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:11.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:11.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:12.002 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:12.385 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:12.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:12.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:12.489 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:12.518 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[186706]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[186706]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[186706]: osdmap e315: 8 total, 8 up, 6 in 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[186706]: pgmap v1160: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[180297]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[180297]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:36:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[180297]: osdmap e315: 8 total, 8 up, 6 in 2023-06-10T21:36:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:12 smithi064 ceph-mon[180297]: pgmap v1160: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:12 smithi152 ceph-mon[156819]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:12 smithi152 ceph-mon[156819]: failed to encode map e315 with expected crc 2023-06-10T21:36:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:36:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:12 smithi152 ceph-mon[156819]: osdmap e315: 8 total, 8 up, 6 in 2023-06-10T21:36:12.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:12 smithi152 ceph-mon[156819]: pgmap v1160: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:12.729 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:12.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:12.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:12.798 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:36:12.815 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:12.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:36:12.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:12.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:36:12.859 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:36:12.865 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:12.883 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:36:12.903 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:36:12.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:12.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:12.928 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:36:12.951 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:36:12.952 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:36:12.953 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:36:13.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:13.058 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:12] "GET /metrics HTTP/1.1" 200 221595 "" "Prometheus/2.33.4" 2023-06-10T21:36:13.157 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:13.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:13.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:13.212 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:13.213 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:13.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:13.294 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:13.390 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:13.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:13.430 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1950185234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:13.430 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:13 smithi064 ceph-mon[186706]: failed to encode map e315 with expected crc 2023-06-10T21:36:13.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:13.498 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:13.498 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1646,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":315,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":24,"data_bytes":595854,"bytes_used":3507937280,"bytes_avail":572462276608,"bytes_total":575970213888},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":25,"modified":"2023-06-10T21:34:59.873283+0000","services":{}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:36:13.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:13.624 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-06-10T21:36:13.627 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:36:13.628 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:36:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1950185234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:13 smithi152 ceph-mon[156819]: failed to encode map e315 with expected crc 2023-06-10T21:36:13.733 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:36:13.733 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:36:13.733 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_442287 2023-06-10T21:36:13.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:36:13.734 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:36:13.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1950185234' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:13.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:13 smithi064 ceph-mon[180297]: failed to encode map e315 with expected crc 2023-06-10T21:36:13.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:36:14.385 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2444384856' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[180297]: pgmap v1161: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:36:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2444384856' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[186706]: pgmap v1161: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:36:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:14 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2444384856' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:14 smithi152 ceph-mon[156819]: pgmap v1161: 89 pgs: 89 active+clean; 582 KiB data, 3.3 GiB used, 533 GiB / 536 GiB avail 2023-06-10T21:36:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:36:14.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:14 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:14.736 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 2932 2916 181.991 182.25 0.00496477 0.00547291 2023-06-10T21:36:15.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:15.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 4923 4907 153.228 124.438 0.00426281 0.00651684 2023-06-10T21:36:15.775 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:15.776 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:16.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:16.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:16.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:16.320 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:16.321 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:16.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:16.564 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:16 smithi064 ceph-mon[186706]: pgmap v1162: 89 pgs: 89 active+clean; 40 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:36:16.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:16 smithi064 ceph-mon[180297]: pgmap v1162: 89 pgs: 89 active+clean; 40 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:36:16.573 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:16.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:16 smithi152 ceph-mon[156819]: pgmap v1162: 89 pgs: 89 active+clean; 40 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 4.7 MiB/s wr, 74 op/s 2023-06-10T21:36:16.735 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 7611 7595 158.142 168 0.00607907 0.00631401 2023-06-10T21:36:17.229 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:17.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:17.243 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-06-10T21:36:17.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:36:17.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:17.380 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:17.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:17.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:17.502 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:36:17.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:17.646 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:17.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 9507 9491 148.113 118.5 0.00592851 0.00674146 2023-06-10T21:36:17.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:36:17.904 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:17.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:18.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:18.101 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:18.127 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:18.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:17 smithi064 ceph-mon[186706]: pgmap v1163: 89 pgs: 89 active+clean; 185 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:36:18.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:17 smithi064 ceph-mon[180297]: pgmap v1163: 89 pgs: 89 active+clean; 185 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:36:18.258 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:18.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:18.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:18.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:18.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:17 smithi152 ceph-mon[156819]: pgmap v1163: 89 pgs: 89 active+clean; 185 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 18 MiB/s wr, 294 op/s 2023-06-10T21:36:18.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:18.460 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:18.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 11217 11201 139.87 106.875 0.00600968 0.00713997 2023-06-10T21:36:18.772 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:18.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:18.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:18.941 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:19.031 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:36:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:36:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:36:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:19.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:36:19.440 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:19.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:19.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:19.598 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:19.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 12895 12879 134.04 104.875 0.00642506 0.00744853 2023-06-10T21:36:20.119 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:20.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:36:20.209 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: osdmap e316: 8 total, 8 up, 6 in 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.210 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[180297]: pgmap v1165: 89 pgs: 89 active+clean; 635 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: osdmap e316: 8 total, 8 up, 6 in 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.211 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:20 smithi064 ceph-mon[186706]: pgmap v1165: 89 pgs: 89 active+clean; 635 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:36:20.308 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:20.308 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:20.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:20.339 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:20.340 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/818981919' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:36:20.340 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: osdmap e316: 8 total, 8 up, 6 in 2023-06-10T21:36:20.340 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.341 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.341 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.341 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.341 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:20.341 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:20 smithi152 ceph-mon[156819]: pgmap v1165: 89 pgs: 89 active+clean; 635 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-10T21:36:20.352 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:20.738 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 15 15168 15153 135.192 142.125 0.0106058 0.00738959 2023-06-10T21:36:21.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:21 smithi064 ceph-mon[186706]: failed to encode map e316 with expected crc 2023-06-10T21:36:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:21 smithi064 ceph-mon[180297]: failed to encode map e316 with expected crc 2023-06-10T21:36:21.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:21 smithi152 ceph-mon[156819]: failed to encode map e316 with expected crc 2023-06-10T21:36:21.510 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:21.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:21.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:21.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:21.650 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:21.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:36:21.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 16962 16946 132.298 112.062 0.00503656 0.00755172 2023-06-10T21:36:21.764 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:21.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:21.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:21.907 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:21.982 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:21.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:22.027 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:22 smithi152 ceph-mon[156819]: pgmap v1166: 89 pgs: 89 active+clean; 821 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:22.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:22.114 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:22.216 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:22.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:22.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:22.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:22 smithi064 ceph-mon[180297]: pgmap v1166: 89 pgs: 89 active+clean; 821 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:22.333 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:22 smithi064 ceph-mon[186706]: pgmap v1166: 89 pgs: 89 active+clean; 821 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:22.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:22.401 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:22.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 18507 18491 128.328 96.5625 0.002416 0.00777334 2023-06-10T21:36:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:22] "GET /metrics HTTP/1.1" 200 224038 "" "Prometheus/2.33.4" 2023-06-10T21:36:23.305 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:23.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:23.336 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:23.413 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:23.502 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:23.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:23.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 19913 19897 124.284 87.875 0.0142905 0.0080377 2023-06-10T21:36:23.759 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:23.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:23.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:23.872 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:23.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:24.155 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:24.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:24.268 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:24.277 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:24.278 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:24.332 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:24 smithi064 ceph-mon[180297]: pgmap v1167: 89 pgs: 89 active+clean; 959 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-10T21:36:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:24 smithi064 ceph-mon[186706]: pgmap v1167: 89 pgs: 89 active+clean; 959 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-10T21:36:24.673 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:24.674 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:36:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:24 smithi152 ceph-mon[156819]: pgmap v1167: 89 pgs: 89 active+clean; 959 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-10T21:36:24.740 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 21315 21299 120.952 87.625 0.00181062 0.0081821 2023-06-10T21:36:25.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:36:25.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:36:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:25 smithi152 ceph-mon[156819]: from='client.36486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:25.591 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:25 smithi064 ceph-mon[186706]: from='client.36486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:25 smithi064 ceph-mon[180297]: from='client.36486 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:25.740 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 23066 23050 119.992 109.438 0.00597215 0.00832622 2023-06-10T21:36:25.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:25.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:25.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:25.969 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:26.003 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:26.612 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:26.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:26.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:26.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:26 smithi064 ceph-mon[180297]: pgmap v1168: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-10T21:36:26.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:26 smithi064 ceph-mon[186706]: pgmap v1168: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-10T21:36:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:26 smithi152 ceph-mon[156819]: pgmap v1168: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-10T21:36:26.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:26.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 24355 24339 116.959 80.5625 0.00671813 0.00854206 2023-06-10T21:36:26.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:26.897 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:26.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:27.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:27.045 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:27.101 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:27.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:27.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:27.137 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1167,"stamp":"2023-06-10T21:36:23.923646+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1005131662,"num_objects":15352,"num_object_clones":0,"num_object_copies":46056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15352,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":15589,"num_write_kb":983438,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":1777362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15523,"ondisk_log_size":15523,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6004008,"kb_used_data":3076088,"kb_used_omap":30,"kb_used_meta":2927841,"kb_avail":556466904,"statfs":{"total":575970213888,"available":569822109696,"internally_reserved":0,"allocated":3149914112,"data_stored":3144645393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":2998109222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1004535808,"num_objects":15328,"num_object_clones":0,"num_object_copies":45984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15328,"num_write_kb":980992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003577"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":310,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301417+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:36:12.301417+0000","last_peered":"2023-06-10T21:36:12.301417+0000","last_clean":"2023-06-10T21:36:12.301417+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:36:12.301417+0000","last_undegraded":"2023-06-10T21:36:12.301417+0000","last_fullsized":"2023-06-10T21:36:12.301417+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":250,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305191+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:36:13.305191+0000","last_peered":"2023-06-10T21:36:13.305191+0000","last_clean":"2023-06-10T21:36:13.305191+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:36:13.305191+0000","last_undegraded":"2023-06-10T21:36:13.305191+0000","last_fullsized":"2023-06-10T21:36:13.305191+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":226,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304788+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:36:13.304788+0000","last_peered":"2023-06-10T21:36:13.304788+0000","last_clean":"2023-06-10T21:36:13.304788+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:36:13.304788+0000","last_undegraded":"2023-06-10T21:36:13.304788+0000","last_fullsized":"2023-06-10T21:36:13.304788+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328680+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:36:12.328680+0000","last_peered":"2023-06-10T21:36:12.328680+0000","last_clean":"2023-06-10T21:36:12.328680+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:36:12.328680+0000","last_undegraded":"2023-06-10T21:36:12.328680+0000","last_fullsized":"2023-06-10T21:36:12.328680+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":291,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304680+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:36:13.304680+0000","last_peered":"2023-06-10T21:36:13.304680+0000","last_clean":"2023-06-10T21:36:13.304680+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:36:13.304680+0000","last_undegraded":"2023-06-10T21:36:13.304680+0000","last_fullsized":"2023-06-10T21:36:13.304680+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":283,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328696+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:36:12.328696+0000","last_peered":"2023-06-10T21:36:12.328696+0000","last_clean":"2023-06-10T21:36:12.328696+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:36:12.328696+0000","last_undegraded":"2023-06-10T21:36:12.328696+0000","last_fullsized":"2023-06-10T21:36:12.328696+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":173,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328720+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:36:12.328720+0000","last_peered":"2023-06-10T21:36:12.328720+0000","last_clean":"2023-06-10T21:36:12.328720+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:36:12.328720+0000","last_undegraded":"2023-06-10T21:36:12.328720+0000","last_fullsized":"2023-06-10T21:36:12.328720+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":265,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304293+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:36:13.304293+0000","last_peered":"2023-06-10T21:36:13.304293+0000","last_clean":"2023-06-10T21:36:13.304293+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:36:13.304293+0000","last_undegraded":"2023-06-10T21:36:13.304293+0000","last_fullsized":"2023-06-10T21:36:13.304293+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":36,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082215+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:36:08.082215+0000","last_peered":"2023-06-10T21:36:08.082215+0000","last_clean":"2023-06-10T21:36:08.082215+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:36:08.082215+0000","last_undegraded":"2023-06-10T21:36:08.082215+0000","last_fullsized":"2023-06-10T21:36:08.082215+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304506+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:36:13.304506+0000","last_peered":"2023-06-10T21:36:13.304506+0000","last_clean":"2023-06-10T21:36:13.304506+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:36:13.304506+0000","last_undegraded":"2023-06-10T21:36:13.304506+0000","last_fullsized":"2023-06-10T21:36:13.304506+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":42,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082215+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:36:08.082215+0000","last_peered":"2023-06-10T21:36:08.082215+0000","last_clean":"2023-06-10T21:36:08.082215+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:36:08.082215+0000","last_undegraded":"2023-06-10T21:36:08.082215+0000","last_fullsized":"2023-06-10T21:36:08.082215+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":279,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301274+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:36:12.301274+0000","last_peered":"2023-06-10T21:36:12.301274+0000","last_clean":"2023-06-10T21:36:12.301274+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:36:12.301274+0000","last_undegraded":"2023-06-10T21:36:12.301274+0000","last_fullsized":"2023-06-10T21:36:12.301274+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":185,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082674+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:36:08.082674+0000","last_peered":"2023-06-10T21:36:08.082674+0000","last_clean":"2023-06-10T21:36:08.082674+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:36:08.082674+0000","last_undegraded":"2023-06-10T21:36:08.082674+0000","last_fullsized":"2023-06-10T21:36:08.082674+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":46,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082617+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:36:08.082617+0000","last_peered":"2023-06-10T21:36:08.082617+0000","last_clean":"2023-06-10T21:36:08.082617+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:36:08.082617+0000","last_undegraded":"2023-06-10T21:36:08.082617+0000","last_fullsized":"2023-06-10T21:36:08.082617+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":314,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082567+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:36:08.082567+0000","last_peered":"2023-06-10T21:36:08.082567+0000","last_clean":"2023-06-10T21:36:08.082567+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:36:08.082567+0000","last_undegraded":"2023-06-10T21:36:08.082567+0000","last_fullsized":"2023-06-10T21:36:08.082567+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305454+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:36:13.305454+0000","last_peered":"2023-06-10T21:36:13.305454+0000","last_clean":"2023-06-10T21:36:13.305454+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:36:13.305454+0000","last_undegraded":"2023-06-10T21:36:13.305454+0000","last_fullsized":"2023-06-10T21:36:13.305454+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":173,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328419+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:36:12.328419+0000","last_peered":"2023-06-10T21:36:12.328419+0000","last_clean":"2023-06-10T21:36:12.328419+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:36:12.328419+0000","last_undegraded":"2023-06-10T21:36:12.328419+0000","last_fullsized":"2023-06-10T21:36:12.328419+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":243,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.083074+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:36:08.083074+0000","last_peered":"2023-06-10T21:36:08.083074+0000","last_clean":"2023-06-10T21:36:08.083074+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:36:08.083074+0000","last_undegraded":"2023-06-10T21:36:08.083074+0000","last_fullsized":"2023-06-10T21:36:08.083074+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":291,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082474+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:36:08.082474+0000","last_peered":"2023-06-10T21:36:08.082474+0000","last_clean":"2023-06-10T21:36:08.082474+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:36:08.082474+0000","last_undegraded":"2023-06-10T21:36:08.082474+0000","last_fullsized":"2023-06-10T21:36:08.082474+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":173,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328852+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:36:12.328852+0000","last_peered":"2023-06-10T21:36:12.328852+0000","last_clean":"2023-06-10T21:36:12.328852+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:36:12.328852+0000","last_undegraded":"2023-06-10T21:36:12.328852+0000","last_fullsized":"2023-06-10T21:36:12.328852+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":292,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328297+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:36:12.328297+0000","last_peered":"2023-06-10T21:36:12.328297+0000","last_clean":"2023-06-10T21:36:12.328297+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:36:12.328297+0000","last_undegraded":"2023-06-10T21:36:12.328297+0000","last_fullsized":"2023-06-10T21:36:12.328297+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":269,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305021+0000","last_change":"2023-06-10T21:32:55.516118+0000","last_active":"2023-06-10T21:36:13.305021+0000","last_peered":"2023-06-10T21:36:13.305021+0000","last_clean":"2023-06-10T21:36:13.305021+0000","last_became_active":"2023-06-10T21:32:55.515740+0000","last_became_peered":"2023-06-10T21:32:55.515740+0000","last_unstale":"2023-06-10T21:36:13.305021+0000","last_undegraded":"2023-06-10T21:36:13.305021+0000","last_fullsized":"2023-06-10T21:36:13.305021+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:43:41.549439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"316'736","reported_seq":750,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.355068+0000","last_change":"2023-06-10T21:36:04.325572+0000","last_active":"2023-06-10T21:36:19.355068+0000","last_peered":"2023-06-10T21:36:19.355068+0000","last_clean":"2023-06-10T21:36:19.355068+0000","last_became_active":"2023-06-10T21:36:04.325154+0000","last_became_peered":"2023-06-10T21:36:04.325154+0000","last_unstale":"2023-06-10T21:36:19.355068+0000","last_undegraded":"2023-06-10T21:36:19.355068+0000","last_fullsized":"2023-06-10T21:36:19.355068+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":736,"log_dups_size":0,"ondisk_log_size":736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:53.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48234496,"num_objects":736,"num_object_clones":0,"num_object_copies":2208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":736,"num_write_kb":47104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":40,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.083076+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:36:08.083076+0000","last_peered":"2023-06-10T21:36:08.083076+0000","last_clean":"2023-06-10T21:36:08.083076+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:36:08.083076+0000","last_undegraded":"2023-06-10T21:36:08.083076+0000","last_fullsized":"2023-06-10T21:36:08.083076+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":36,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082316+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:36:08.082316+0000","last_peered":"2023-06-10T21:36:08.082316+0000","last_clean":"2023-06-10T21:36:08.082316+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:36:08.082316+0000","last_undegraded":"2023-06-10T21:36:08.082316+0000","last_fullsized":"2023-06-10T21:36:08.082316+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"316'1002","reported_seq":1016,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:21.707152+0000","last_change":"2023-06-10T21:36:04.292881+0000","last_active":"2023-06-10T21:36:21.707152+0000","last_peered":"2023-06-10T21:36:21.707152+0000","last_clean":"2023-06-10T21:36:21.707152+0000","last_became_active":"2023-06-10T21:36:04.292606+0000","last_became_peered":"2023-06-10T21:36:04.292606+0000","last_unstale":"2023-06-10T21:36:21.707152+0000","last_undegraded":"2023-06-10T21:36:21.707152+0000","last_fullsized":"2023-06-10T21:36:21.707152+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1002,"log_dups_size":0,"ondisk_log_size":1002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:21.334865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1002,"num_write_kb":64128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":40,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.083031+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:36:08.083031+0000","last_peered":"2023-06-10T21:36:08.083031+0000","last_clean":"2023-06-10T21:36:08.083031+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:36:08.083031+0000","last_undegraded":"2023-06-10T21:36:08.083031+0000","last_fullsized":"2023-06-10T21:36:08.083031+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304965+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:36:13.304965+0000","last_peered":"2023-06-10T21:36:13.304965+0000","last_clean":"2023-06-10T21:36:13.304965+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:36:13.304965+0000","last_undegraded":"2023-06-10T21:36:13.304965+0000","last_fullsized":"2023-06-10T21:36:13.304965+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"316'1094","reported_seq":1108,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:21.724298+0000","last_change":"2023-06-10T21:36:04.325459+0000","last_active":"2023-06-10T21:36:21.724298+0000","last_peered":"2023-06-10T21:36:21.724298+0000","last_clean":"2023-06-10T21:36:21.724298+0000","last_became_active":"2023-06-10T21:36:04.325111+0000","last_became_peered":"2023-06-10T21:36:04.325111+0000","last_unstale":"2023-06-10T21:36:21.724298+0000","last_undegraded":"2023-06-10T21:36:21.724298+0000","last_fullsized":"2023-06-10T21:36:21.724298+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1094,"log_dups_size":0,"ondisk_log_size":1094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:49:06.786091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71696384,"num_objects":1094,"num_object_clones":0,"num_object_copies":3282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1094,"num_write_kb":70016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":322,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082908+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:36:08.082908+0000","last_peered":"2023-06-10T21:36:08.082908+0000","last_clean":"2023-06-10T21:36:08.082908+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:36:08.082908+0000","last_undegraded":"2023-06-10T21:36:08.082908+0000","last_fullsized":"2023-06-10T21:36:08.082908+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":36,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082517+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:36:08.082517+0000","last_peered":"2023-06-10T21:36:08.082517+0000","last_clean":"2023-06-10T21:36:08.082517+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:36:08.082517+0000","last_undegraded":"2023-06-10T21:36:08.082517+0000","last_fullsized":"2023-06-10T21:36:08.082517+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"316'1220","reported_seq":1234,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:23.212226+0000","last_change":"2023-06-10T21:36:04.289584+0000","last_active":"2023-06-10T21:36:23.212226+0000","last_peered":"2023-06-10T21:36:23.212226+0000","last_clean":"2023-06-10T21:36:23.212226+0000","last_became_active":"2023-06-10T21:36:04.289267+0000","last_became_peered":"2023-06-10T21:36:04.289267+0000","last_unstale":"2023-06-10T21:36:23.212226+0000","last_undegraded":"2023-06-10T21:36:23.212226+0000","last_fullsized":"2023-06-10T21:36:23.212226+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1220,"log_dups_size":0,"ondisk_log_size":1220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:34.840732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79953920,"num_objects":1220,"num_object_clones":0,"num_object_copies":3660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1220,"num_write_kb":78080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":198,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305690+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:36:13.305690+0000","last_peered":"2023-06-10T21:36:13.305690+0000","last_clean":"2023-06-10T21:36:13.305690+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:36:13.305690+0000","last_undegraded":"2023-06-10T21:36:13.305690+0000","last_fullsized":"2023-06-10T21:36:13.305690+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":140,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328339+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:36:12.328339+0000","last_peered":"2023-06-10T21:36:12.328339+0000","last_clean":"2023-06-10T21:36:12.328339+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:36:12.328339+0000","last_undegraded":"2023-06-10T21:36:12.328339+0000","last_fullsized":"2023-06-10T21:36:12.328339+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"316'1102","reported_seq":1116,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:21.724009+0000","last_change":"2023-06-10T21:36:04.288539+0000","last_active":"2023-06-10T21:36:21.724009+0000","last_peered":"2023-06-10T21:36:21.724009+0000","last_clean":"2023-06-10T21:36:21.724009+0000","last_became_active":"2023-06-10T21:36:04.288222+0000","last_became_peered":"2023-06-10T21:36:04.288222+0000","last_unstale":"2023-06-10T21:36:21.724009+0000","last_undegraded":"2023-06-10T21:36:21.724009+0000","last_fullsized":"2023-06-10T21:36:21.724009+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1102,"log_dups_size":0,"ondisk_log_size":1102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:53.556650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72220672,"num_objects":1102,"num_object_clones":0,"num_object_copies":3306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1102,"num_write_kb":70528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":268,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.306181+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:36:13.306181+0000","last_peered":"2023-06-10T21:36:13.306181+0000","last_clean":"2023-06-10T21:36:13.306181+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:36:13.306181+0000","last_undegraded":"2023-06-10T21:36:13.306181+0000","last_fullsized":"2023-06-10T21:36:13.306181+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305767+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:36:13.305767+0000","last_peered":"2023-06-10T21:36:13.305767+0000","last_clean":"2023-06-10T21:36:13.305767+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:36:13.305767+0000","last_undegraded":"2023-06-10T21:36:13.305767+0000","last_fullsized":"2023-06-10T21:36:13.305767+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"316'824","reported_seq":838,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.349230+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:36:19.349230+0000","last_peered":"2023-06-10T21:36:19.349230+0000","last_clean":"2023-06-10T21:36:19.349230+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:36:19.349230+0000","last_undegraded":"2023-06-10T21:36:19.349230+0000","last_fullsized":"2023-06-10T21:36:19.349230+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":824,"log_dups_size":0,"ondisk_log_size":824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":824,"num_write_kb":52736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304661+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:36:13.304661+0000","last_peered":"2023-06-10T21:36:13.304661+0000","last_clean":"2023-06-10T21:36:13.304661+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:36:13.304661+0000","last_undegraded":"2023-06-10T21:36:13.304661+0000","last_fullsized":"2023-06-10T21:36:13.304661+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":308,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301534+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:36:12.301534+0000","last_peered":"2023-06-10T21:36:12.301534+0000","last_clean":"2023-06-10T21:36:12.301534+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:36:12.301534+0000","last_undegraded":"2023-06-10T21:36:12.301534+0000","last_fullsized":"2023-06-10T21:36:12.301534+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"316'1135","reported_seq":1149,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:23.235323+0000","last_change":"2023-06-10T21:36:04.281931+0000","last_active":"2023-06-10T21:36:23.235323+0000","last_peered":"2023-06-10T21:36:23.235323+0000","last_clean":"2023-06-10T21:36:23.235323+0000","last_became_active":"2023-06-10T21:36:04.281632+0000","last_became_peered":"2023-06-10T21:36:04.281632+0000","last_unstale":"2023-06-10T21:36:23.235323+0000","last_undegraded":"2023-06-10T21:36:23.235323+0000","last_fullsized":"2023-06-10T21:36:23.235323+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1135,"log_dups_size":0,"ondisk_log_size":1135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:57.076427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74383360,"num_objects":1135,"num_object_clones":0,"num_object_copies":3405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1135,"num_write_kb":72640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":294,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301256+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:36:12.301256+0000","last_peered":"2023-06-10T21:36:12.301256+0000","last_clean":"2023-06-10T21:36:12.301256+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:36:12.301256+0000","last_undegraded":"2023-06-10T21:36:12.301256+0000","last_fullsized":"2023-06-10T21:36:12.301256+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305277+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:36:13.305277+0000","last_peered":"2023-06-10T21:36:13.305277+0000","last_clean":"2023-06-10T21:36:13.305277+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:36:13.305277+0000","last_undegraded":"2023-06-10T21:36:13.305277+0000","last_fullsized":"2023-06-10T21:36:13.305277+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"315'718","reported_seq":730,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:18.932109+0000","last_change":"2023-06-10T21:36:04.289902+0000","last_active":"2023-06-10T21:36:18.932109+0000","last_peered":"2023-06-10T21:36:18.932109+0000","last_clean":"2023-06-10T21:36:18.932109+0000","last_became_active":"2023-06-10T21:36:04.289647+0000","last_became_peered":"2023-06-10T21:36:04.289647+0000","last_unstale":"2023-06-10T21:36:18.932109+0000","last_undegraded":"2023-06-10T21:36:18.932109+0000","last_fullsized":"2023-06-10T21:36:18.932109+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":718,"log_dups_size":0,"ondisk_log_size":718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47054848,"num_objects":718,"num_object_clones":0,"num_object_copies":2154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":718,"num_write_kb":45952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":271,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.306324+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:36:13.306324+0000","last_peered":"2023-06-10T21:36:13.306324+0000","last_clean":"2023-06-10T21:36:13.306324+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:36:13.306324+0000","last_undegraded":"2023-06-10T21:36:13.306324+0000","last_fullsized":"2023-06-10T21:36:13.306324+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":227,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305209+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:36:13.305209+0000","last_peered":"2023-06-10T21:36:13.305209+0000","last_clean":"2023-06-10T21:36:13.305209+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:36:13.305209+0000","last_undegraded":"2023-06-10T21:36:13.305209+0000","last_fullsized":"2023-06-10T21:36:13.305209+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"316'773","reported_seq":786,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.357829+0000","last_change":"2023-06-10T21:36:04.291075+0000","last_active":"2023-06-10T21:36:19.357829+0000","last_peered":"2023-06-10T21:36:19.357829+0000","last_clean":"2023-06-10T21:36:19.357829+0000","last_became_active":"2023-06-10T21:36:04.290808+0000","last_became_peered":"2023-06-10T21:36:04.290808+0000","last_unstale":"2023-06-10T21:36:19.357829+0000","last_undegraded":"2023-06-10T21:36:19.357829+0000","last_fullsized":"2023-06-10T21:36:19.357829+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":773,"log_dups_size":0,"ondisk_log_size":773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:13:41.537032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50659328,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":773,"num_write_kb":49472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":140,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328546+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:36:12.328546+0000","last_peered":"2023-06-10T21:36:12.328546+0000","last_clean":"2023-06-10T21:36:12.328546+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:36:12.328546+0000","last_undegraded":"2023-06-10T21:36:12.328546+0000","last_fullsized":"2023-06-10T21:36:12.328546+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305285+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:36:13.305285+0000","last_peered":"2023-06-10T21:36:13.305285+0000","last_clean":"2023-06-10T21:36:13.305285+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:36:13.305285+0000","last_undegraded":"2023-06-10T21:36:13.305285+0000","last_fullsized":"2023-06-10T21:36:13.305285+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":160,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328302+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:36:12.328302+0000","last_peered":"2023-06-10T21:36:12.328302+0000","last_clean":"2023-06-10T21:36:12.328302+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:36:12.328302+0000","last_undegraded":"2023-06-10T21:36:12.328302+0000","last_fullsized":"2023-06-10T21:36:12.328302+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"316'1287","reported_seq":1301,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:23.229603+0000","last_change":"2023-06-10T21:36:04.286731+0000","last_active":"2023-06-10T21:36:23.229603+0000","last_peered":"2023-06-10T21:36:23.229603+0000","last_clean":"2023-06-10T21:36:23.229603+0000","last_became_active":"2023-06-10T21:36:04.286427+0000","last_became_peered":"2023-06-10T21:36:04.286427+0000","last_unstale":"2023-06-10T21:36:23.229603+0000","last_undegraded":"2023-06-10T21:36:23.229603+0000","last_fullsized":"2023-06-10T21:36:23.229603+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1287,"log_dups_size":0,"ondisk_log_size":1287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:26:20.203365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84344832,"num_objects":1287,"num_object_clones":0,"num_object_copies":3861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1287,"num_write_kb":82368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":186,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328379+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:36:12.328379+0000","last_peered":"2023-06-10T21:36:12.328379+0000","last_clean":"2023-06-10T21:36:12.328379+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:36:12.328379+0000","last_undegraded":"2023-06-10T21:36:12.328379+0000","last_fullsized":"2023-06-10T21:36:12.328379+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":260,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328540+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:36:12.328540+0000","last_peered":"2023-06-10T21:36:12.328540+0000","last_clean":"2023-06-10T21:36:12.328540+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:36:12.328540+0000","last_undegraded":"2023-06-10T21:36:12.328540+0000","last_fullsized":"2023-06-10T21:36:12.328540+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301431+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:36:12.301431+0000","last_peered":"2023-06-10T21:36:12.301431+0000","last_clean":"2023-06-10T21:36:12.301431+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:36:12.301431+0000","last_undegraded":"2023-06-10T21:36:12.301431+0000","last_fullsized":"2023-06-10T21:36:12.301431+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"316'738","reported_seq":751,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.354790+0000","last_change":"2023-06-10T21:36:04.325488+0000","last_active":"2023-06-10T21:36:19.354790+0000","last_peered":"2023-06-10T21:36:19.354790+0000","last_clean":"2023-06-10T21:36:19.354790+0000","last_became_active":"2023-06-10T21:36:04.325162+0000","last_became_peered":"2023-06-10T21:36:04.325162+0000","last_unstale":"2023-06-10T21:36:19.354790+0000","last_undegraded":"2023-06-10T21:36:19.354790+0000","last_fullsized":"2023-06-10T21:36:19.354790+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":738,"log_dups_size":0,"ondisk_log_size":738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:41:33.200826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48365568,"num_objects":738,"num_object_clones":0,"num_object_copies":2214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":738,"num_write_kb":47232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":161,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301329+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:36:12.301329+0000","last_peered":"2023-06-10T21:36:12.301329+0000","last_clean":"2023-06-10T21:36:12.301329+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:36:12.301329+0000","last_undegraded":"2023-06-10T21:36:12.301329+0000","last_fullsized":"2023-06-10T21:36:12.301329+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301304+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:36:12.301304+0000","last_peered":"2023-06-10T21:36:12.301304+0000","last_clean":"2023-06-10T21:36:12.301304+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:36:12.301304+0000","last_undegraded":"2023-06-10T21:36:12.301304+0000","last_fullsized":"2023-06-10T21:36:12.301304+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":209,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328747+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:36:12.328747+0000","last_peered":"2023-06-10T21:36:12.328747+0000","last_clean":"2023-06-10T21:36:12.328747+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:36:12.328747+0000","last_undegraded":"2023-06-10T21:36:12.328747+0000","last_fullsized":"2023-06-10T21:36:12.328747+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"315'691","reported_seq":703,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:18.925847+0000","last_change":"2023-06-10T21:36:04.289917+0000","last_active":"2023-06-10T21:36:18.925847+0000","last_peered":"2023-06-10T21:36:18.925847+0000","last_clean":"2023-06-10T21:36:18.925847+0000","last_became_active":"2023-06-10T21:36:04.289698+0000","last_became_peered":"2023-06-10T21:36:04.289698+0000","last_unstale":"2023-06-10T21:36:18.925847+0000","last_undegraded":"2023-06-10T21:36:18.925847+0000","last_fullsized":"2023-06-10T21:36:18.925847+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":691,"log_dups_size":0,"ondisk_log_size":691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45285376,"num_objects":691,"num_object_clones":0,"num_object_copies":2073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":691,"num_write_kb":44224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":341,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:14.356931+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:36:14.356931+0000","last_peered":"2023-06-10T21:36:14.356931+0000","last_clean":"2023-06-10T21:36:14.356931+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:36:14.356931+0000","last_undegraded":"2023-06-10T21:36:14.356931+0000","last_fullsized":"2023-06-10T21:36:14.356931+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":478,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082627+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:36:08.082627+0000","last_peered":"2023-06-10T21:36:08.082627+0000","last_clean":"2023-06-10T21:36:08.082627+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:36:08.082627+0000","last_undegraded":"2023-06-10T21:36:08.082627+0000","last_fullsized":"2023-06-10T21:36:08.082627+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304794+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:36:13.304794+0000","last_peered":"2023-06-10T21:36:13.304794+0000","last_clean":"2023-06-10T21:36:13.304794+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:36:13.304794+0000","last_undegraded":"2023-06-10T21:36:13.304794+0000","last_fullsized":"2023-06-10T21:36:13.304794+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.083023+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:36:08.083023+0000","last_peered":"2023-06-10T21:36:08.083023+0000","last_clean":"2023-06-10T21:36:08.083023+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:36:08.083023+0000","last_undegraded":"2023-06-10T21:36:08.083023+0000","last_fullsized":"2023-06-10T21:36:08.083023+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"316'1199","reported_seq":1213,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:23.226886+0000","last_change":"2023-06-10T21:36:04.325644+0000","last_active":"2023-06-10T21:36:23.226886+0000","last_peered":"2023-06-10T21:36:23.226886+0000","last_clean":"2023-06-10T21:36:23.226886+0000","last_became_active":"2023-06-10T21:36:04.325301+0000","last_became_peered":"2023-06-10T21:36:04.325301+0000","last_unstale":"2023-06-10T21:36:23.226886+0000","last_undegraded":"2023-06-10T21:36:23.226886+0000","last_fullsized":"2023-06-10T21:36:23.226886+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1199,"log_dups_size":0,"ondisk_log_size":1199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:28:03.018219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78577664,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1199,"num_write_kb":76736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":334,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082720+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:36:08.082720+0000","last_peered":"2023-06-10T21:36:08.082720+0000","last_clean":"2023-06-10T21:36:08.082720+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:36:08.082720+0000","last_undegraded":"2023-06-10T21:36:08.082720+0000","last_fullsized":"2023-06-10T21:36:08.082720+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":155,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328290+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:36:12.328290+0000","last_peered":"2023-06-10T21:36:12.328290+0000","last_clean":"2023-06-10T21:36:12.328290+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:36:12.328290+0000","last_undegraded":"2023-06-10T21:36:12.328290+0000","last_fullsized":"2023-06-10T21:36:12.328290+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":140,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328898+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:36:12.328898+0000","last_peered":"2023-06-10T21:36:12.328898+0000","last_clean":"2023-06-10T21:36:12.328898+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:36:12.328898+0000","last_undegraded":"2023-06-10T21:36:12.328898+0000","last_fullsized":"2023-06-10T21:36:12.328898+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"316'999","reported_seq":1013,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:21.718638+0000","last_change":"2023-06-10T21:36:04.325466+0000","last_active":"2023-06-10T21:36:21.718638+0000","last_peered":"2023-06-10T21:36:21.718638+0000","last_clean":"2023-06-10T21:36:21.718638+0000","last_became_active":"2023-06-10T21:36:04.325178+0000","last_became_peered":"2023-06-10T21:36:04.325178+0000","last_unstale":"2023-06-10T21:36:21.718638+0000","last_undegraded":"2023-06-10T21:36:21.718638+0000","last_fullsized":"2023-06-10T21:36:21.718638+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":999,"log_dups_size":0,"ondisk_log_size":999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:03.551659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65470464,"num_objects":999,"num_object_clones":0,"num_object_copies":2997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":999,"num_write_kb":63936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":319,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082902+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:36:08.082902+0000","last_peered":"2023-06-10T21:36:08.082902+0000","last_clean":"2023-06-10T21:36:08.082902+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:36:08.082902+0000","last_undegraded":"2023-06-10T21:36:08.082902+0000","last_fullsized":"2023-06-10T21:36:08.082902+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":159,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301461+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:36:12.301461+0000","last_peered":"2023-06-10T21:36:12.301461+0000","last_clean":"2023-06-10T21:36:12.301461+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:36:12.301461+0000","last_undegraded":"2023-06-10T21:36:12.301461+0000","last_fullsized":"2023-06-10T21:36:12.301461+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":136,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328550+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:36:12.328550+0000","last_peered":"2023-06-10T21:36:12.328550+0000","last_clean":"2023-06-10T21:36:12.328550+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:36:12.328550+0000","last_undegraded":"2023-06-10T21:36:12.328550+0000","last_fullsized":"2023-06-10T21:36:12.328550+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"316'1081","reported_seq":1095,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:21.726509+0000","last_change":"2023-06-10T21:36:04.292180+0000","last_active":"2023-06-10T21:36:21.726509+0000","last_peered":"2023-06-10T21:36:21.726509+0000","last_clean":"2023-06-10T21:36:21.726509+0000","last_became_active":"2023-06-10T21:36:04.291901+0000","last_became_peered":"2023-06-10T21:36:04.291901+0000","last_unstale":"2023-06-10T21:36:21.726509+0000","last_undegraded":"2023-06-10T21:36:21.726509+0000","last_fullsized":"2023-06-10T21:36:21.726509+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1081,"log_dups_size":0,"ondisk_log_size":1081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:45.649615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1081,"num_write_kb":69184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":294,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305096+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:36:13.305096+0000","last_peered":"2023-06-10T21:36:13.305096+0000","last_clean":"2023-06-10T21:36:13.305096+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:36:13.305096+0000","last_undegraded":"2023-06-10T21:36:13.305096+0000","last_fullsized":"2023-06-10T21:36:13.305096+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305132+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:36:13.305132+0000","last_peered":"2023-06-10T21:36:13.305132+0000","last_clean":"2023-06-10T21:36:13.305132+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:36:13.305132+0000","last_undegraded":"2023-06-10T21:36:13.305132+0000","last_fullsized":"2023-06-10T21:36:13.305132+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":275,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304514+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:36:13.304514+0000","last_peered":"2023-06-10T21:36:13.304514+0000","last_clean":"2023-06-10T21:36:13.304514+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:36:13.304514+0000","last_undegraded":"2023-06-10T21:36:13.304514+0000","last_fullsized":"2023-06-10T21:36:13.304514+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"316'729","reported_seq":743,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.353295+0000","last_change":"2023-06-10T21:36:04.296839+0000","last_active":"2023-06-10T21:36:19.353295+0000","last_peered":"2023-06-10T21:36:19.353295+0000","last_clean":"2023-06-10T21:36:19.353295+0000","last_became_active":"2023-06-10T21:36:04.294647+0000","last_became_peered":"2023-06-10T21:36:04.294647+0000","last_unstale":"2023-06-10T21:36:19.353295+0000","last_undegraded":"2023-06-10T21:36:19.353295+0000","last_fullsized":"2023-06-10T21:36:19.353295+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":729,"log_dups_size":0,"ondisk_log_size":729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:16.069639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":729,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":36,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082343+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:36:08.082343+0000","last_peered":"2023-06-10T21:36:08.082343+0000","last_clean":"2023-06-10T21:36:08.082343+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:36:08.082343+0000","last_undegraded":"2023-06-10T21:36:08.082343+0000","last_fullsized":"2023-06-10T21:36:08.082343+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":144,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328778+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:36:12.328778+0000","last_peered":"2023-06-10T21:36:12.328778+0000","last_clean":"2023-06-10T21:36:12.328778+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:36:12.328778+0000","last_undegraded":"2023-06-10T21:36:12.328778+0000","last_fullsized":"2023-06-10T21:36:12.328778+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":272,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328184+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:36:12.328184+0000","last_peered":"2023-06-10T21:36:12.328184+0000","last_clean":"2023-06-10T21:36:12.328184+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:36:12.328184+0000","last_undegraded":"2023-06-10T21:36:12.328184+0000","last_fullsized":"2023-06-10T21:36:12.328184+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":140,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328637+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:36:12.328637+0000","last_peered":"2023-06-10T21:36:12.328637+0000","last_clean":"2023-06-10T21:36:12.328637+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:36:12.328637+0000","last_undegraded":"2023-06-10T21:36:12.328637+0000","last_fullsized":"2023-06-10T21:36:12.328637+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":158,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301527+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:36:12.301527+0000","last_peered":"2023-06-10T21:36:12.301527+0000","last_clean":"2023-06-10T21:36:12.301527+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:36:12.301527+0000","last_undegraded":"2023-06-10T21:36:12.301527+0000","last_fullsized":"2023-06-10T21:36:12.301527+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":40,"reported_epoch":314,"state":"active+clean","last_fresh":"2023-06-10T21:36:08.082804+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:36:08.082804+0000","last_peered":"2023-06-10T21:36:08.082804+0000","last_clean":"2023-06-10T21:36:08.082804+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:36:08.082804+0000","last_undegraded":"2023-06-10T21:36:08.082804+0000","last_fullsized":"2023-06-10T21:36:08.082804+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305939+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:36:13.305939+0000","last_peered":"2023-06-10T21:36:13.305939+0000","last_clean":"2023-06-10T21:36:13.305939+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:36:13.305939+0000","last_undegraded":"2023-06-10T21:36:13.305939+0000","last_fullsized":"2023-06-10T21:36:13.305939+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305032+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:36:13.305032+0000","last_peered":"2023-06-10T21:36:13.305032+0000","last_clean":"2023-06-10T21:36:13.305032+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:36:13.305032+0000","last_undegraded":"2023-06-10T21:36:13.305032+0000","last_fullsized":"2023-06-10T21:36:13.305032+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":254,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304679+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:36:13.304679+0000","last_peered":"2023-06-10T21:36:13.304679+0000","last_clean":"2023-06-10T21:36:13.304679+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:36:13.304679+0000","last_undegraded":"2023-06-10T21:36:13.304679+0000","last_fullsized":"2023-06-10T21:36:13.304679+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":300,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304943+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:36:13.304943+0000","last_peered":"2023-06-10T21:36:13.304943+0000","last_clean":"2023-06-10T21:36:13.304943+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:36:13.304943+0000","last_undegraded":"2023-06-10T21:36:13.304943+0000","last_fullsized":"2023-06-10T21:36:13.304943+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":304,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304643+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:36:13.304643+0000","last_peered":"2023-06-10T21:36:13.304643+0000","last_clean":"2023-06-10T21:36:13.304643+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:36:13.304643+0000","last_undegraded":"2023-06-10T21:36:13.304643+0000","last_fullsized":"2023-06-10T21:36:13.304643+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":205,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.328813+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:36:12.328813+0000","last_peered":"2023-06-10T21:36:12.328813+0000","last_clean":"2023-06-10T21:36:12.328813+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:36:12.328813+0000","last_undegraded":"2023-06-10T21:36:12.328813+0000","last_fullsized":"2023-06-10T21:36:12.328813+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:12.301135+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:36:12.301135+0000","last_peered":"2023-06-10T21:36:12.301135+0000","last_clean":"2023-06-10T21:36:12.301135+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:36:12.301135+0000","last_undegraded":"2023-06-10T21:36:12.301135+0000","last_fullsized":"2023-06-10T21:36:12.301135+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":1004535808,"num_objects":15328,"num_object_clones":0,"num_object_copies":45984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15328,"num_write_kb":980992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2829975552,"data_stored":2829975552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15328,"ondisk_log_size":15328,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430671,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1043148,"kb_used_data":438532,"kb_used_omap":0,"kb_used_meta":604608,"kb_avail":92702004,"statfs":{"total":95995035648,"available":94926852096,"internally_reserved":0,"allocated":449056768,"data_stored":448483337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":619118592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":4.5199999999999996,"15min":7.4340000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":28.875,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":3.734,"5min":4.7830000000000004,"15min":7.7130000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":28.943000000000001,"5min":185.58000000000001,"15min":185.58000000000001},"last":1.248}]},{"osd":1,"last update":"Sat Jun 10 21:36:15 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":3.681,"15min":3.4830000000000001},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":54.43,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.1029999999999998,"15min":4.0469999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":54.564,"5min":87.438000000000002,"15min":212.20599999999999},"last":1.0740000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":6.5759999999999996,"5min":7.1050000000000004,"15min":4.5890000000000004},"min":{"1min":0.309,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":103.866,"5min":172.33600000000001,"15min":172.33600000000001},"last":1.704},{"interface":"front","average":{"1min":7.5999999999999996,"5min":7.4850000000000003,"15min":5.3200000000000003},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":103.705,"5min":172.185,"15min":212.13399999999999},"last":3.6949999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:36:14 2023","interfaces":[{"interface":"back","average":{"1min":11.154,"5min":6.0650000000000004,"15min":3.649},"min":{"1min":0.38100000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.00700000000001,"5min":147.00700000000001,"15min":147.00700000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":11.568,"5min":5.8860000000000001,"15min":3.165},"min":{"1min":0.41399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":146.93899999999999,"5min":146.93899999999999,"15min":146.93899999999999},"last":0.433}]},{"osd":4,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39000000000000001,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.59499999999999997,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.438,"15min":1.0209999999999999},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.66600000000000004,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.32400000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":1.8120000000000001},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.68700000000000006,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.45400000000000001,"15min":1.607},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.29299999999999998},"max":{"1min":0.68600000000000005,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.50900000000000001,"15min":1.069},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":3.2559999999999998,"15min":13.073},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.48499999999999999,"15min":0.92900000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":1.1879999999999999,"15min":10.874000000000001},"last":0.52600000000000002}]}]},{"osd":6,"up_from":0,"seq":180388626899,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566342,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1392932,"kb_used_data":727836,"kb_used_omap":4,"kb_used_meta":665083,"kb_avail":92352220,"statfs":{"total":95995035648,"available":94568673280,"internally_reserved":0,"allocated":745304064,"data_stored":744306080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":681045902},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:48 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":4.9550000000000001,"15min":10.315},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":28.507999999999999,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.714,"5min":4.5700000000000003,"15min":10.029},"min":{"1min":0.27300000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":28.045999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":11.297000000000001,"5min":9.1150000000000002,"15min":5.7110000000000003},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":134.79900000000001,"5min":191.101,"15min":191.101},"last":0.437},{"interface":"front","average":{"1min":4.8479999999999999,"5min":7.702,"15min":5.0469999999999997},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.218},"max":{"1min":15.923,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":7.6280000000000001,"5min":5.734,"15min":3.2719999999999998},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":83.810000000000002,"5min":83.810000000000002,"15min":83.810000000000002},"last":0.45100000000000001},{"interface":"front","average":{"1min":7.7050000000000001,"5min":5.8680000000000003,"15min":4.2050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":92.757000000000005,"5min":92.757000000000005,"15min":92.757000000000005},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.24,"15min":1.5920000000000001},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":7.609,"5min":9.3659999999999997,"15min":16.059999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.5289999999999999,"15min":1.48},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":5.5389999999999997,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.74299999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.1099999999999999,"15min":4.3200000000000003},"min":{"1min":0.434,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.959,"5min":11.959,"15min":383.83100000000002},"last":0.82299999999999995},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.2509999999999999,"15min":3.847},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":7.6429999999999998,"5min":9.3160000000000007,"15min":383.84800000000001},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.1469999999999998,"15min":2.226},"min":{"1min":0.40699999999999997,"5min":0.316,"15min":0.316},"max":{"1min":11.923,"5min":11.923,"15min":14.858000000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.411,"5min":1.4019999999999999,"15min":1.772},"min":{"1min":0.45500000000000002,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":7.5700000000000003,"5min":9.3849999999999998,"15min":13.48},"last":0.83599999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.9470000000000001,"15min":2.7090000000000001},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":7.4749999999999996,"5min":47.844000000000001,"15min":162.399},"last":63.249000000000002},{"interface":"front","average":{"1min":1.115,"5min":2.3490000000000002,"15min":2.5059999999999998},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":5.5650000000000004,"5min":47.753999999999998,"15min":162.27199999999999},"last":62.975999999999999}]}]},{"osd":0,"up_from":244,"seq":1047972020319,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":726192,"kb_used_data":442280,"kb_used_omap":10,"kb_used_meta":283893,"kb_avail":93018960,"statfs":{"total":95995035648,"available":95251415040,"internally_reserved":0,"allocated":452894720,"data_stored":451824302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":290706484},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":4.0869999999999997,"15min":4.391},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":4.6500000000000004,"5min":139.68000000000001,"15min":139.68000000000001},"last":0.22800000000000001},{"interface":"front","average":{"1min":4.5839999999999996,"5min":4.5140000000000002,"15min":5.4619999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":15.403,"5min":141.79499999999999,"15min":141.79499999999999},"last":2.2410000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2569999999999997,"5min":7.1600000000000001,"15min":6.7930000000000001},"min":{"1min":0.31,"5min":0.216,"15min":0.216},"max":{"1min":24.166,"5min":132.245,"15min":205.10900000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":4.0499999999999998,"5min":6.7140000000000004,"15min":6.2770000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":13.06,"5min":132.26599999999999,"15min":208.34299999999999},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:12 2023","interfaces":[{"interface":"back","average":{"1min":15.250999999999999,"5min":6.3860000000000001,"15min":6.1669999999999998},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":211.864,"5min":211.864,"15min":211.864},"last":0.57999999999999996},{"interface":"front","average":{"1min":14.010999999999999,"5min":6.7969999999999997,"15min":6.9569999999999999},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":207.517,"5min":207.517,"15min":207.517},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.9710000000000001,"15min":3.9460000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":8.9550000000000001,"5min":10.744999999999999,"15min":29.709},"last":2.2839999999999998},{"interface":"front","average":{"1min":3.4049999999999998,"5min":2.2440000000000002,"15min":3.5110000000000001},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":24.222999999999999,"5min":24.222999999999999,"15min":24.222999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":2.2639999999999998,"15min":3.5859999999999999},"min":{"1min":0.438,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":24.288,"5min":24.288,"15min":24.288},"last":0.379},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.8169999999999999,"15min":3.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":6.9870000000000001,"5min":8.0079999999999991,"15min":19.846},"last":0.435}]},{"osd":6,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.056,"15min":4.1100000000000003},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":9,"5min":10.771000000000001,"15min":29.806000000000001},"last":3.2650000000000001},{"interface":"front","average":{"1min":3.4430000000000001,"5min":2.3839999999999999,"15min":3.6509999999999998},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":24.260999999999999,"5min":24.260999999999999,"15min":24.260999999999999},"last":0.90800000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.0449999999999999,"15min":3.3839999999999999},"min":{"1min":0.48199999999999998,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.0149999999999997,"5min":17.523,"15min":19.866},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.843,"5min":2.2120000000000002,"15min":4.1609999999999996},"min":{"1min":0.44800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0259999999999998,"5min":17.792999999999999,"15min":29.77},"last":3.306}]}]},{"osd":2,"up_from":120,"seq":515396075730,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":969680,"kb_used_data":474120,"kb_used_omap":7,"kb_used_meta":495544,"kb_avail":92775472,"statfs":{"total":95995035648,"available":95002083328,"internally_reserved":0,"allocated":485498880,"data_stored":484512177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":507437137},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7000000000000002,"5min":4.6079999999999997,"15min":4.694},"min":{"1min":0.217,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.9,"5min":70.510999999999996,"15min":70.510999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":5.0209999999999999,"5min":4.4329999999999998,"15min":5.1479999999999997},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":11.733000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":5.0979999999999999,"15min":3.2799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.19},"max":{"1min":7.7910000000000004,"5min":147.20099999999999,"15min":147.20099999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.3780000000000001,"5min":4.968,"15min":3.0059999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.214},"max":{"1min":9.4619999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.502}]},{"osd":3,"last update":"Sat Jun 10 21:36:11 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":6.0970000000000004,"15min":5.3760000000000003},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.15,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.7360000000000002,"5min":6.3019999999999996,"15min":6.0659999999999998},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":17.553999999999998,"5min":138.095,"15min":138.095},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6879999999999999,"15min":2.976},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":224.00999999999999},"last":5.9470000000000001},{"interface":"front","average":{"1min":1.831,"5min":2.3029999999999999,"15min":3.3260000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":7.7030000000000003,"5min":29.486999999999998,"15min":223.28399999999999},"last":5.7869999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.681,"15min":1.671},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":5.4470000000000001,"5min":10.554,"15min":14.180999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.7879999999999998,"5min":1.73,"15min":1.8939999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":7.3200000000000003,"5min":7.3200000000000003,"15min":16.125},"last":0.73499999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8140000000000001,"5min":1.802,"15min":1.966},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":7.4180000000000001,"5min":7.4180000000000001,"15min":16.183},"last":0.81000000000000005},{"interface":"front","average":{"1min":2.1339999999999999,"5min":2.4239999999999999,"15min":2.3500000000000001},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.7510000000000003,"5min":29.538,"15min":29.538},"last":0.82999999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":3.4630000000000001,"15min":2.6429999999999998},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.8399999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.84499999999999997},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.4340000000000002,"15min":2.9390000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.3620000000000001,"5min":73.475999999999999,"15min":113.937},"last":0.85399999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824090,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1053876,"kb_used_data":779820,"kb_used_omap":7,"kb_used_meta":274040,"kb_avail":92691276,"statfs":{"total":95995035648,"available":94915866624,"internally_reserved":0,"allocated":798535680,"data_stored":797462288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":280616979},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":4.4569999999999999,"15min":4.9820000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.1890000000000001,"5min":105.09,"15min":105.09},"last":3.5169999999999999},{"interface":"front","average":{"1min":1.472,"5min":4.4160000000000004,"15min":4.7969999999999997},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":99.852999999999994,"15min":99.852999999999994},"last":0.80400000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":4.766,"15min":4.0380000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":10.419,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.4650000000000001,"5min":4.6639999999999997,"15min":3.7320000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.9629999999999992,"5min":94.691000000000003,"15min":94.691000000000003},"last":3.649}]},{"osd":2,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":3.7690000000000001,"15min":9.1449999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7480000000000002,"5min":100.676,"15min":100.676},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.2400000000000002,"15min":10.669},"min":{"1min":0.30399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.835000000000001,"5min":103.032,"15min":103.032},"last":0.88500000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.446,"15min":1.405},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":3.7770000000000001,"5min":13.417,"15min":13.417},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.244,"5min":2.3780000000000001,"15min":2.6680000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.2030000000000003,"5min":14.384,"15min":19.728000000000002},"last":0.92600000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.5169999999999999,"15min":1.397},"min":{"1min":0.377,"5min":0.35699999999999998,"15min":0.32400000000000001},"max":{"1min":1.4079999999999999,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":3.46},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.6060000000000001,"15min":1.4870000000000001},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.7999999999999998,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.3919999999999999,"15min":2.6899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.1740000000000004,"5min":14.269,"15min":19.805},"last":1.835},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.552,"15min":1.4470000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.39100000000000001},"max":{"1min":1.4379999999999999,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":3.4950000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0760000000000001,"15min":2.9260000000000002},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.1449999999999996,"5min":79.984999999999999,"15min":79.984999999999999},"last":1.7929999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":2.2360000000000002,"15min":1.6579999999999999},"min":{"1min":0.44,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":1.6899999999999999,"5min":80.272999999999996,"15min":80.272999999999996},"last":3.5470000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019355,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818180,"kb_used_data":213500,"kb_used_omap":0,"kb_used_meta":604672,"kb_avail":92926972,"statfs":{"total":95995035648,"available":95157219328,"internally_reserved":0,"allocated":218624000,"data_stored":218057209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":619184128},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":4.1260000000000003,"15min":7.899},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.503,"5min":234.203,"15min":234.203},"last":0.96699999999999997},{"interface":"front","average":{"1min":3.2770000000000001,"5min":4.3730000000000002,"15min":7.8220000000000001},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":37.399999999999999,"5min":234.19,"15min":234.19},"last":1.24}]},{"osd":1,"last update":"Sat Jun 10 21:35:27 2023","interfaces":[{"interface":"back","average":{"1min":8.7840000000000007,"5min":3.121,"15min":3.468},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":88.700999999999993,"5min":88.700999999999993,"15min":182.035},"last":0.40899999999999997},{"interface":"front","average":{"1min":8.6110000000000007,"5min":3.3130000000000002,"15min":3.375},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.25900000000000001},"max":{"1min":87.853999999999999,"5min":87.853999999999999,"15min":181.73099999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":7.6150000000000002,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.308999999999997,"5min":255.386,"15min":255.386},"last":0.40000000000000002},{"interface":"front","average":{"1min":6.4720000000000004,"5min":7.5499999999999998,"15min":3.6850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":85.322000000000003,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3959999999999999,"5min":8.0939999999999994,"15min":4.2489999999999997},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.28399999999999997},"max":{"1min":36.366999999999997,"5min":234.91900000000001,"15min":234.91900000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":3.6619999999999999,"5min":8.4410000000000007,"15min":4.1900000000000004},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":38.659999999999997,"5min":235.05099999999999,"15min":235.05099999999999},"last":0.55600000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.379,"15min":0.99299999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.54300000000000004,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.45700000000000002,"15min":1.004},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.45500000000000002,"5min":5.25,"15min":11.904999999999999},"last":0.31}]},{"osd":6,"last update":"Sat Jun 10 21:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42899999999999999,"15min":1.0149999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.67700000000000005,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46600000000000003,"15min":1.079},"min":{"1min":0.35499999999999998,"5min":0.34699999999999998,"15min":0.309},"max":{"1min":0.63400000000000001,"5min":1.71,"15min":13.845000000000001},"last":0.376}]},{"osd":7,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.77900000000000003,"15min":1.2669999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":0.65300000000000002,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.79200000000000004,"15min":1.21},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.68500000000000005,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.41899999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823127,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":477167616,"data_stored":477167616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":467927040,"data_stored":467927040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":496959488,"data_stored":496959488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":629342208,"data_stored":629342208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":234160128,"data_stored":234160128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":524419072,"data_stored":524419072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:36:27.206 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:27.241 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:27.403 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:27.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:27.457 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:27.458 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:27.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:27.545 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:27.598 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:27.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:27.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:27.728 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:36:27.740 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 25841 25825 115.239 92.875 0.0109477 0.00866759 2023-06-10T21:36:27.759 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:36:27.778 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:36:27.797 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:36:27.817 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:36:27.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:27.840 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:36:27.857 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:27.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3528923214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:27.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3528923214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:27.864 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:36:27.904 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:27.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:27.915 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:36:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:36:27.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:36:28.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:28.036 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3528923214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:28.039 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:28.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:36:28.313 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:28.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:28.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:28.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:28.487 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:28.736 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:28.739 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 27176 27160 113.119 83.4375 0.00954777 0.00883323 2023-06-10T21:36:29.010 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:28 smithi152 ceph-mon[156819]: pgmap v1169: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:36:29.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:28 smithi064 ceph-mon[180297]: pgmap v1169: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:36:29.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:28 smithi064 ceph-mon[186706]: pgmap v1169: 89 pgs: 89 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:36:29.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:29.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:29.588 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:29.617 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:29.675 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:29.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:29.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:29.741 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 28461 28445 111.061 80.3125 0.00762751 0.00899823 2023-06-10T21:36:29.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:29.844 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:30.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:36:30.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:30 smithi064 ceph-mon[186706]: pgmap v1170: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-06-10T21:36:30.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:30 smithi064 ceph-mon[180297]: pgmap v1170: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-06-10T21:36:30.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:30 smithi152 ceph-mon[156819]: pgmap v1170: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-06-10T21:36:30.460 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:30.460 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:36:30.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.460 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:29203 12.d 12:bcdfc615:::benchmark_data_smithi064_442287_object29202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:30.308310+0000", 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.14942033399999999, 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.149450312, 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:30.461 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 29203 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308310+0000", 2023-06-10T21:36:30.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308310+0000", 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308309+0000", 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3759999999999998e-05 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308392+0000", 2023-06-10T21:36:30.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6639999999999993e-06 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308402+0000", 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.374e-06 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308407+0000", 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000449426 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308856+0000", 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010602700000000001 2023-06-10T21:36:30.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308962+0000", 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018572599999999999 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.467 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.309148+0000", 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019627220000000001 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.311110+0000", 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028425350000000002 2023-06-10T21:36:30.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.313953+0000", 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056427450000000002 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:29204 12.d 12:bf270004:::benchmark_data_smithi064_442287_object29203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:30.308824+0000", 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.148906803, 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14899832499999999, 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:36:30.470 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 29204 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308824+0000", 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308824+0000", 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:30.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308823+0000", 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6863999999999999e-05 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:30.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308850+0000", 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8639999999999998e-06 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308857+0000", 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5590000000000001e-06 2023-06-10T21:36:30.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.308860+0000", 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012595670000000001 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:30.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310120+0000", 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111267 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310231+0000", 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119251 2023-06-10T21:36:30.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310351+0000", 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001892063 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:30.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.312243+0000", 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017598430000000001 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.314002+0000", 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051787120000000002 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.478 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:29205 12.d 12:b6d4b875:::benchmark_data_smithi064_442287_object29204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e316)", 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:30.310545+0000", 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.147185127, 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.147328405, 2023-06-10T21:36:30.479 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 29205 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.480 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310545+0000", 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:30.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310545+0000", 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310544+0000", 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010183600000000001 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310646+0000", 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2713e-05 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310659+0000", 2023-06-10T21:36:30.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4099999999999996e-06 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.310665+0000", 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00047551899999999999 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.311141+0000", 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010901800000000001 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.311250+0000", 2023-06-10T21:36:30.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000145699 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.311396+0000", 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001004811 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.312400+0000", 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033189460000000001 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:30.315719+0000", 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00517396 2023-06-10T21:36:30.487 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 3 2023-06-10T21:36:30.488 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:30.741 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 29442 29426 108.136 61.3125 0.00183489 0.00923799 2023-06-10T21:36:31.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:31.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:31.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:31.268 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:31.277 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:36:31.322 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:31.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:31.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:31.514 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:31.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 30683 30667 106.433 77.5625 0.00392085 0.00938821 2023-06-10T21:36:31.749 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:31.789 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:31.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:31.855 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:31.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:31.928 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:31.988 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:31.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:32.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:32.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:31 smithi064 ceph-mon[180297]: pgmap v1171: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-06-10T21:36:32.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:31 smithi064 ceph-mon[186706]: pgmap v1171: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-06-10T21:36:32.176 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:32.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:31 smithi152 ceph-mon[156819]: pgmap v1171: 89 pgs: 89 active+clean; 1.7 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-06-10T21:36:32.311 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:32.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:32.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:32.458 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:32.502 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:32.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:32.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:32.683 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:32.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 31837 31821 104.626 72.125 0.00697626 0.00954865 2023-06-10T21:36:33.033 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:36:33.062 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:32] "GET /metrics HTTP/1.1" 200 224038 "" "Prometheus/2.33.4" 2023-06-10T21:36:33.062 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3269006995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.063 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3269006995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.063 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:36:33.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:33.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:33.223 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3269006995' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.224 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:36:33.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:33.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:33.423 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:33.424 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:36:33.424 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:36:33.424 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:36:33.425 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:33.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:36:33.740551+0000 min lat: 0.00116179 max lat: 0.261793 avg lat: 0.00966466 2023-06-10T21:36:33.742 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:36:33.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 33046 33030 103.173 75.5625 0.00189757 0.00966466 2023-06-10T21:36:34.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:36:34.259 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:36:34.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:36:34.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: osdmap e317: 8 total, 8 up, 6 in 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.261 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:34 smithi152 ceph-mon[156819]: pgmap v1173: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:36:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:36:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:36:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: osdmap e317: 8 total, 8 up, 6 in 2023-06-10T21:36:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:34.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[180297]: pgmap v1173: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:36:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:36:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: failed to encode map e317 with expected crc 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: osdmap e317: 8 total, 8 up, 6 in 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-06-10T21:36:34.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:34 smithi064 ceph-mon[186706]: pgmap v1173: 89 pgs: 89 active+clean; 1.8 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-10T21:36:34.605 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:36:34.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 33636 33620 100.013 36.875 0.00303536 0.00999317 2023-06-10T21:36:34.787 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:34.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:34.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:34.903 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:34.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:35.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:35.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2023-06-10T21:36:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-06-10T21:36:35.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: osdmap e318: 8 total, 8 up, 6 in 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2023-06-10T21:36:35.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-06-10T21:36:35.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: osdmap e318: 8 total, 8 up, 6 in 2023-06-10T21:36:35.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.440 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.8", "id": [3, 2]}]': finished 2023-06-10T21:36:35.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-06-10T21:36:35.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: osdmap e318: 8 total, 8 up, 6 in 2023-06-10T21:36:35.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: failed to encode map e318 with expected crc 2023-06-10T21:36:35.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:35.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:35.623 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:35.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 34642 34626 98.325 62.875 0.0113556 0.0101623 2023-06-10T21:36:35.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:35.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:35.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:35.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:36.139 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:36.202 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:36.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:36.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:36.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:36.321 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111471 10.1 10:85cc9677:::benchmark_data_smithi064_312713_object20715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e231)", 2023-06-10T21:36:36.322 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 535.02712668799995, 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999, 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:36.323 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:36.324 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111471 2023-06-10T21:36:36.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.324 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289585+0000", 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289583+0000", 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0890000000000002e-06 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289590+0000", 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6314e-05 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289607+0000", 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.267e-06 2023-06-10T21:36:36.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289614+0000", 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3952000000000001e-05 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289668+0000", 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0001314 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289799+0000", 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9462999999999994e-05 2023-06-10T21:36:36.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,4", 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.289889+0000", 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029815369999999998 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.292870+0000", 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00065814400000000005 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.293528+0000", 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.030991312 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324520+0000", 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3043e-05 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.324533+0000", 2023-06-10T21:36:36.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.00200007 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326533+0000", 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6407000000000003e-05 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326599+0000", 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0370145049999999 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.35769.0:111487 10.1 10:8be62f36:::benchmark_data_smithi064_312713_object20731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e232)", 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 534.98498546200005, 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997, 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.335 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.35769", 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 111487 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:36:36.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331726+0000", 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331724+0000", 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7449999999999995e-06 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331732+0000", 2023-06-10T21:36:36.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5709000000000002e-05 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331748+0000", 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0659999999999998e-06 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331755+0000", 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1113999999999999e-05 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331806+0000", 2023-06-10T21:36:36.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0156999999999999e-05 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:41.331816+0000", 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99503633400000002 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326852+0000", 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9278999999999996e-05 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.326952+0000", 2023-06-10T21:36:36.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015410900000000001 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 4", 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.327106+0000", 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035978350000000002 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.330703+0000", 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00051919100000000001 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331223+0000", 2023-06-10T21:36:36.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1488999999999994e-05 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331284+0000", 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112025 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:27:42.331396+0000", 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99967027799999997 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.346 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85133 11.3 11.3e76a33 (undecoded) ondisk+retry+write+known_if_redirected e288)", 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.79588636099999, 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995, 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.347 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85133 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520824+0000", 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.792e-06 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520829+0000", 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0971999999999999e-05 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520840+0000", 2023-06-10T21:36:36.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9069999999999998e-06 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520845+0000", 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032454789999999999 2023-06-10T21:36:36.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524090+0000", 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.843e-06 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524095+0000", 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98421531600000001 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508311+0000", 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7920000000000004e-06 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508320+0000", 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.355 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.356 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.356 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85152 11.5 11.363bf865 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.356 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.77519859899999, 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995, 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.357 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85152 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:36:36.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541512+0000", 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9899999999999999e-06 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541516+0000", 2023-06-10T21:36:36.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0390000000000005e-06 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541524+0000", 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2599999999999999e-06 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541528+0000", 2023-06-10T21:36:36.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4838000000000001e-05 2023-06-10T21:36:36.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541563+0000", 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.95e-06 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541568+0000", 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96671271599999997 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508281+0000", 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1725999999999999e-05 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508293+0000", 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995 2023-06-10T21:36:36.365 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85160 11.3 11.f67e3553 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.75568854799999, 2023-06-10T21:36:36.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94735323500000002, 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85160 2023-06-10T21:36:36.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561023+0000", 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561022+0000", 2023-06-10T21:36:36.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1909999999999997e-06 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561028+0000", 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3774999999999999e-05 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.370 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561042+0000", 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1970000000000004e-06 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561047+0000", 2023-06-10T21:36:36.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1782000000000001e-05 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561089+0000", 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4460000000000007e-06 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.561098+0000", 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94724917200000003 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508347+0000", 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8926000000000001e-05 2023-06-10T21:36:36.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508376+0000", 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94735323500000002 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85191 11.5 11.7bbb6cb5 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.71851527999999, 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91013171800000003, 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.375 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85191 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598196+0000", 2023-06-10T21:36:36.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598195+0000", 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5589999999999999e-06 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598200+0000", 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3920000000000003e-06 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598208+0000", 2023-06-10T21:36:36.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2100000000000001e-06 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598213+0000", 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6866999999999998e-05 2023-06-10T21:36:36.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598250+0000", 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6789999999999996e-06 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.598255+0000", 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91006529300000005 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508320+0000", 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5769999999999996e-06 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508328+0000", 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.91013171800000003 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.383 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.384 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.384 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85195 11.3 11.919bb373 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.384 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.71588039400001, 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758057299999995, 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.385 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85195 2023-06-10T21:36:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:36:36.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600831+0000", 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8479999999999999e-06 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600833+0000", 2023-06-10T21:36:36.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2070000000000003e-06 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600838+0000", 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0210000000000001e-06 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600842+0000", 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2150000000000001e-05 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600884+0000", 2023-06-10T21:36:36.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8730000000000004e-06 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600890+0000", 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90751452799999999 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508404+0000", 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1849999999999998e-06 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508412+0000", 2023-06-10T21:36:36.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758057299999995 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85196 11.3 11.67cdda23 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:36:36.394 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.71585538299999, 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758501300000005, 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85196 2023-06-10T21:36:36.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600856+0000", 2023-06-10T21:36:36.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7930000000000002e-06 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600858+0000", 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6449999999999999e-06 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600865+0000", 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0100000000000003e-06 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.600870+0000", 2023-06-10T21:36:36.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00046476800000000001 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.601335+0000", 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3800000000000002e-06 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.601340+0000", 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90709262499999999 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508433+0000", 2023-06-10T21:36:36.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2449999999999998e-06 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508441+0000", 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90758501300000005 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.402 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85222 11.3 11.2dc991d3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.68138615500001, 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314513999999999, 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.403 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85222 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635325+0000", 2023-06-10T21:36:36.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635324+0000", 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5709999999999999e-06 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635329+0000", 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1164e-05 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635340+0000", 2023-06-10T21:36:36.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6739999999999996e-06 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635345+0000", 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3868000000000005e-05 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635418+0000", 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4380000000000003e-06 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635424+0000", 2023-06-10T21:36:36.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87303943799999995 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508463+0000", 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0779999999999998e-06 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508470+0000", 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314513999999999 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.411 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85223 11.3 11.8bf7f803 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.681356497, 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314413300000004, 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.412 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85223 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:36:36.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635355+0000", 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635354+0000", 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1090000000000001e-06 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635357+0000", 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3224e-05 2023-06-10T21:36:36.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635380+0000", 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7760000000000002e-06 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635383+0000", 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00044493000000000001 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635828+0000", 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3069999999999996e-06 2023-06-10T21:36:36.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635832+0000", 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87265963000000002 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508491+0000", 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.542e-06 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508499+0000", 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87314413300000004 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.419 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85227 11.3 11.edeb40b3 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 221.68129636699999, 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87315411499999995, 2023-06-10T21:36:36.420 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85227 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.421 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635415+0000", 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0999999999999998e-06 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.423 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635417+0000", 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4450000000000001e-06 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635420+0000", 2023-06-10T21:36:36.424 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2230000000000001e-06 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.635422+0000", 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001176156 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.425 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636599+0000", 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.481e-06 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.636601+0000", 2023-06-10T21:36:36.426 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87194879999999997 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508550+0000", 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9221000000000001e-05 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.427 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508569+0000", 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87315411499999995 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.428 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155021 11.3 11:c69a6bdd:::benchmark_data_smithi064_385725_object70748:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.229104402000004, 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99854388599999999, 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.429 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155021 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.430 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.431 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087606+0000", 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0050000000000001e-06 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.432 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087612+0000", 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6149999999999997e-06 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087620+0000", 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2990000000000002e-06 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.433 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087627+0000", 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6254999999999998e-05 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087663+0000", 2023-06-10T21:36:36.434 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5990000000000002e-06 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087672+0000", 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95657830200000005 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.435 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044250+0000", 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115345 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044365+0000", 2023-06-10T21:36:36.436 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053313099999999995 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044898+0000", 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017003877000000001 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.437 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.061902+0000", 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018346986999999999 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080249+0000", 2023-06-10T21:36:36.438 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058076330000000004 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086057+0000", 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4727e-05 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.439 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086112+0000", 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9286000000000003e-05 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086151+0000", 2023-06-10T21:36:36.440 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99854388599999999 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155026 11.3 11:cb0acfcd:::benchmark_data_smithi064_385725_object70753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:36:36.441 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.215960803000002, 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98550636899999999, 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155026 2023-06-10T21:36:36.442 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.443 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100749+0000", 2023-06-10T21:36:36.444 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2400000000000004e-06 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100755+0000", 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1558e-05 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.445 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100767+0000", 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8810000000000004e-06 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100772+0000", 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034783600000000002 2023-06-10T21:36:36.446 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.101120+0000", 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6319999999999997e-06 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.447 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.101128+0000", 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95303523300000004 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054164+0000", 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013795699999999999 2023-06-10T21:36:36.448 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054302+0000", 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015307100000000001 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.449 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054455+0000", 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074513979999999997 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.061906+0000", 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018387883000000001 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.450 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080294+0000", 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059022859999999996 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.451 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086196+0000", 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5917000000000002e-05 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086232+0000", 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4827000000000001e-05 2023-06-10T21:36:36.452 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086257+0000", 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98550636899999999 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.453 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155031 11.3 11:c81286d5:::benchmark_data_smithi064_385725_object70758:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.210161106000001, 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.979807235, 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.454 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155031 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.455 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:36:36.456 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106549+0000", 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.64e-06 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.457 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106555+0000", 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0412e-05 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106566+0000", 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2620000000000004e-06 2023-06-10T21:36:36.458 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106572+0000", 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9200000000000003e-05 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.459 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106621+0000", 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7770000000000002e-06 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106629+0000", 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95023215400000005 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.460 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.056861+0000", 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.996e-05 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.461 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.056941+0000", 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8929000000000001e-05 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.057040+0000", 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.008220148 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.462 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065260+0000", 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0150592 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080319+0000", 2023-06-10T21:36:36.463 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059690949999999998 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086288+0000", 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7549999999999998e-05 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.464 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086326+0000", 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1538000000000002e-05 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086358+0000", 2023-06-10T21:36:36.465 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.979807235 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155045 11.3 11:cf5ff534:::benchmark_data_smithi064_385725_object70772:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:36:36.466 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.188900656000001, 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96627055699999997, 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155045 2023-06-10T21:36:36.467 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.468 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127810+0000", 2023-06-10T21:36:36.469 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.54e-06 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127814+0000", 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2520000000000002e-06 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.470 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127821+0000", 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3760000000000001e-06 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127825+0000", 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0646999999999997e-05 2023-06-10T21:36:36.471 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127886+0000", 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9740000000000008e-06 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.472 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127895+0000", 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93078737700000003 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058682+0000", 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104113 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.473 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058786+0000", 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2851999999999998e-05 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.474 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058879+0000", 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063842059999999999 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065264+0000", 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015182453 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.475 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080446+0000", 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013573333999999999 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094019+0000", 2023-06-10T21:36:36.476 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5673000000000002e-05 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094055+0000", 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6285999999999999e-05 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.477 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094081+0000", 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96627055699999997 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.478 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155052 11.3 11:ce2c9903:::benchmark_data_smithi064_385725_object70779:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.139451+0000", 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.177260266999994, 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95475026799999996, 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.479 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155052 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.480 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139451+0000", 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139451+0000", 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.481 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139450+0000", 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0679999999999999e-06 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139457+0000", 2023-06-10T21:36:36.482 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7790000000000007e-06 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139467+0000", 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.481e-06 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.483 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139471+0000", 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1235000000000006e-05 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139532+0000", 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2761e-05 2023-06-10T21:36:36.484 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.139545+0000", 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.92079873599999995 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.485 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.060344+0000", 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8479999999999995e-05 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.060412+0000", 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7884000000000001e-05 2023-06-10T21:36:36.486 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.060500+0000", 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078886949999999994 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.487 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.068389+0000", 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015592979 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.083982+0000", 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010145464999999999 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.488 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094127+0000", 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8699999999999999e-05 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094166+0000", 2023-06-10T21:36:36.489 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5252000000000001e-05 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094201+0000", 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95475026799999996 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.490 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155074 11.3 11:ca9e9776:::benchmark_data_smithi064_385725_object70801:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.167787+0000", 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.148924597000004, 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93434221100000003, 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.491 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155074 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.492 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167787+0000", 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167787+0000", 2023-06-10T21:36:36.493 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167786+0000", 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.176e-06 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.494 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167789+0000", 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6729999999999998e-06 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167797+0000", 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8339999999999999e-06 2023-06-10T21:36:36.495 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167800+0000", 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5373e-05 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.496 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167856+0000", 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0200999999999999e-05 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.167866+0000", 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89408275299999995 2023-06-10T21:36:36.497 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.061948+0000", 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113188 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.498 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.062062+0000", 2023-06-10T21:36:36.499 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3595000000000002e-05 2023-06-10T21:36:36.499 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.499 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.501 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.501 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.062145+0000", 2023-06-10T21:36:36.501 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062468920000000004 2023-06-10T21:36:36.501 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.068392+0000", 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.025847143999999999 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.502 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094239+0000", 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078294290000000006 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.102069+0000", 2023-06-10T21:36:36.503 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4635000000000003e-05 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.102103+0000", 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5647999999999999e-05 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.504 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.102129+0000", 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93434221100000003 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.505 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155083 11.3 11:ca90d1ea:::benchmark_data_smithi064_385725_object70810:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.183948+0000", 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.132763675999996, 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.932014228, 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.506 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155083 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183948+0000", 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183948+0000", 2023-06-10T21:36:36.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183946+0000", 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8359999999999997e-06 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183952+0000", 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2456999999999999e-05 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183965+0000", 2023-06-10T21:36:36.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9289999999999997e-06 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.183970+0000", 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4792e-05 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.184024+0000", 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.148e-06 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.184032+0000", 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87960687400000004 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.063639+0000", 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0488999999999995e-05 2023-06-10T21:36:36.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.063719+0000", 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2385000000000004e-05 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.063811+0000", 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011146231 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.074958+0000", 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.019308832000000001 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094266+0000", 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.021641341000000001 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.115908+0000", 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9244e-05 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.115937+0000", 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4905e-05 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.115962+0000", 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.932014228 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.519 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155105 11.3 11:c901e82a:::benchmark_data_smithi064_385725_object70832:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.519 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.222734+0000", 2023-06-10T21:36:36.519 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.093977643000002, 2023-06-10T21:36:36.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89337408399999996, 2023-06-10T21:36:36.520 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.520 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.520 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.520 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.521 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.521 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155105 2023-06-10T21:36:36.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.522 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222734+0000", 2023-06-10T21:36:36.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222734+0000", 2023-06-10T21:36:36.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:36:36.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222733+0000", 2023-06-10T21:36:36.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.63e-06 2023-06-10T21:36:36.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222736+0000", 2023-06-10T21:36:36.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2199999999999992e-06 2023-06-10T21:36:36.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222745+0000", 2023-06-10T21:36:36.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9199999999999997e-06 2023-06-10T21:36:36.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222748+0000", 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2327999999999998e-05 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222801+0000", 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0692e-05 2023-06-10T21:36:36.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.222811+0000", 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84249649800000004 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.531 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065308+0000", 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101797 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065410+0000", 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6218000000000006e-05 2023-06-10T21:36:36.532 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065506+0000", 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094555560000000004 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.074962+0000", 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.029573522000000001 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.104535+0000", 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011483808999999999 2023-06-10T21:36:36.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116019+0000", 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4863000000000001e-05 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116064+0000", 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4087e-05 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116108+0000", 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89337408399999996 2023-06-10T21:36:36.536 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155110 11.3 11:ceb5879d:::benchmark_data_smithi064_385725_object70837:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.235212+0000", 2023-06-10T21:36:36.537 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 84.081499123, 2023-06-10T21:36:36.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88097805299999998, 2023-06-10T21:36:36.538 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:36:36.538 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:36:36.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:36:36.538 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155110 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:36:36.539 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235212+0000", 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235212+0000", 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:36:36.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235211+0000", 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7940000000000008e-06 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:36:36.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235219+0000", 2023-06-10T21:36:36.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9907e-05 2023-06-10T21:36:36.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:36:36.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235249+0000", 2023-06-10T21:36:36.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1689999999999992e-06 2023-06-10T21:36:36.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:36:36.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235258+0000", 2023-06-10T21:36:36.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5141000000000005e-05 2023-06-10T21:36:36.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235323+0000", 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3579e-05 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.235336+0000", 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83151302800000004 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.066849+0000", 2023-06-10T21:36:36.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1060000000000001e-05 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.066920+0000", 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7013000000000001e-05 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.067017+0000", 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011252592 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.078270+0000", 2023-06-10T21:36:36.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.026308186000000001 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.104578+0000", 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011564723000000001 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116143+0000", 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6631e-05 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116169+0000", 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0845000000000001e-05 2023-06-10T21:36:36.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.116190+0000", 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88097805299999998 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.551 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:36:36.552 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:36:36.552 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: osdmap e319: 8 total, 8 up, 6 in 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:36:36.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:36.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:36 smithi152 ceph-mon[156819]: pgmap v1176: 89 pgs: 1 peering, 88 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s 2023-06-10T21:36:36.554 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:36.554 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: osdmap e319: 8 total, 8 up, 6 in 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:36.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[180297]: pgmap v1176: 89 pgs: 1 peering, 88 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: osdmap e319: 8 total, 8 up, 6 in 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: failed to encode map e319 with expected crc 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:36.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:36 smithi064 ceph-mon[186706]: pgmap v1176: 89 pgs: 1 peering, 88 active+clean; 2.0 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s 2023-06-10T21:36:36.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:36.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:36.666 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:36.695 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:36.742 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:36.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 35831 35815 97.281 74.3125 0.00161131 0.0102335 2023-06-10T21:36:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:36.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:36.773 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:36.796 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:36:36.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:36.921 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:37.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:36:37.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:37 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:36:37.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:37 smithi152 ceph-mon[156819]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:37.534 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:36:37.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:37 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:36:37.586 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:37 smithi064 ceph-mon[186706]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:37.586 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:37 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-10T21:36:37.587 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:37 smithi064 ceph-mon[180297]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:36:37.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 37166 37150 96.7041 83.4375 0.0219693 0.0103309 2023-06-10T21:36:38.169 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:38.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:38.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:38.273 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:38.307 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:38.358 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:38.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:38.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:38.465 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:38 smithi064 ceph-mon[180297]: pgmap v1177: 89 pgs: 1 peering, 88 active+clean; 2.1 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 57 MiB/s wr, 911 op/s 2023-06-10T21:36:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:38 smithi064 ceph-mon[186706]: pgmap v1177: 89 pgs: 1 peering, 88 active+clean; 2.1 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 57 MiB/s wr, 911 op/s 2023-06-10T21:36:38.645 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:38 smithi152 ceph-mon[156819]: pgmap v1177: 89 pgs: 1 peering, 88 active+clean; 2.1 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 57 MiB/s wr, 911 op/s 2023-06-10T21:36:38.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 37981 37965 94.8738 50.9375 0.00913465 0.0105333 2023-06-10T21:36:38.987 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:39.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:39.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:39.021 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1176,"stamp":"2023-06-10T21:36:35.932419+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2114328462,"num_objects":32277,"num_object_clones":0,"num_object_copies":96831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32277,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":32514,"num_write_kb":2066638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":1777362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32448,"ondisk_log_size":32448,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9600604,"kb_used_data":6551084,"kb_used_omap":30,"kb_used_meta":3049377,"kb_avail":552870308,"statfs":{"total":575970213888,"available":566139195392,"internally_reserved":0,"allocated":6708310016,"data_stored":6703035478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":3122562086},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":379781120,"num_objects":5795,"num_object_clones":0,"num_object_copies":17385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5795,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002899"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":312,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.042942+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:36:33.042942+0000","last_peered":"2023-06-10T21:36:33.042942+0000","last_clean":"2023-06-10T21:36:33.042942+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:36:33.042942+0000","last_undegraded":"2023-06-10T21:36:33.042942+0000","last_fullsized":"2023-06-10T21:36:33.042942+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":251,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.024458+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:36:19.024458+0000","last_peered":"2023-06-10T21:36:19.024458+0000","last_clean":"2023-06-10T21:36:19.024458+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:36:19.024458+0000","last_undegraded":"2023-06-10T21:36:19.024458+0000","last_fullsized":"2023-06-10T21:36:19.024458+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":226,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304788+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:36:13.304788+0000","last_peered":"2023-06-10T21:36:13.304788+0000","last_clean":"2023-06-10T21:36:13.304788+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:36:13.304788+0000","last_undegraded":"2023-06-10T21:36:13.304788+0000","last_fullsized":"2023-06-10T21:36:13.304788+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":175,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.049258+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:36:33.049258+0000","last_peered":"2023-06-10T21:36:33.049258+0000","last_clean":"2023-06-10T21:36:33.049258+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:36:33.049258+0000","last_undegraded":"2023-06-10T21:36:33.049258+0000","last_fullsized":"2023-06-10T21:36:33.049258+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":291,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304680+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:36:13.304680+0000","last_peered":"2023-06-10T21:36:13.304680+0000","last_clean":"2023-06-10T21:36:13.304680+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:36:13.304680+0000","last_undegraded":"2023-06-10T21:36:13.304680+0000","last_fullsized":"2023-06-10T21:36:13.304680+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":284,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032380+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:36:19.032380+0000","last_peered":"2023-06-10T21:36:19.032380+0000","last_clean":"2023-06-10T21:36:19.032380+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:36:19.032380+0000","last_undegraded":"2023-06-10T21:36:19.032380+0000","last_fullsized":"2023-06-10T21:36:19.032380+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032467+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:36:19.032467+0000","last_peered":"2023-06-10T21:36:19.032467+0000","last_clean":"2023-06-10T21:36:19.032467+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:36:19.032467+0000","last_undegraded":"2023-06-10T21:36:19.032467+0000","last_fullsized":"2023-06-10T21:36:19.032467+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":265,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304293+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:36:13.304293+0000","last_peered":"2023-06-10T21:36:13.304293+0000","last_clean":"2023-06-10T21:36:13.304293+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:36:13.304293+0000","last_undegraded":"2023-06-10T21:36:13.304293+0000","last_fullsized":"2023-06-10T21:36:13.304293+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":38,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035754+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:36:19.035754+0000","last_peered":"2023-06-10T21:36:19.035754+0000","last_clean":"2023-06-10T21:36:19.035754+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:36:19.035754+0000","last_undegraded":"2023-06-10T21:36:19.035754+0000","last_fullsized":"2023-06-10T21:36:19.035754+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304506+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:36:13.304506+0000","last_peered":"2023-06-10T21:36:13.304506+0000","last_clean":"2023-06-10T21:36:13.304506+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:36:13.304506+0000","last_undegraded":"2023-06-10T21:36:13.304506+0000","last_fullsized":"2023-06-10T21:36:13.304506+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":44,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035572+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:36:19.035572+0000","last_peered":"2023-06-10T21:36:19.035572+0000","last_clean":"2023-06-10T21:36:19.035572+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:36:19.035572+0000","last_undegraded":"2023-06-10T21:36:19.035572+0000","last_fullsized":"2023-06-10T21:36:19.035572+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":281,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.042846+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:36:33.042846+0000","last_peered":"2023-06-10T21:36:33.042846+0000","last_clean":"2023-06-10T21:36:33.042846+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:36:33.042846+0000","last_undegraded":"2023-06-10T21:36:33.042846+0000","last_fullsized":"2023-06-10T21:36:33.042846+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":187,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035534+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:36:19.035534+0000","last_peered":"2023-06-10T21:36:19.035534+0000","last_clean":"2023-06-10T21:36:19.035534+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:36:19.035534+0000","last_undegraded":"2023-06-10T21:36:19.035534+0000","last_fullsized":"2023-06-10T21:36:19.035534+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":48,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035822+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:36:19.035822+0000","last_peered":"2023-06-10T21:36:19.035822+0000","last_clean":"2023-06-10T21:36:19.035822+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:36:19.035822+0000","last_undegraded":"2023-06-10T21:36:19.035822+0000","last_fullsized":"2023-06-10T21:36:19.035822+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":316,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035808+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:36:19.035808+0000","last_peered":"2023-06-10T21:36:19.035808+0000","last_clean":"2023-06-10T21:36:19.035808+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:36:19.035808+0000","last_undegraded":"2023-06-10T21:36:19.035808+0000","last_fullsized":"2023-06-10T21:36:19.035808+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305454+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:36:13.305454+0000","last_peered":"2023-06-10T21:36:13.305454+0000","last_clean":"2023-06-10T21:36:13.305454+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:36:13.305454+0000","last_undegraded":"2023-06-10T21:36:13.305454+0000","last_fullsized":"2023-06-10T21:36:13.305454+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":174,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032273+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:36:19.032273+0000","last_peered":"2023-06-10T21:36:19.032273+0000","last_clean":"2023-06-10T21:36:19.032273+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:36:19.032273+0000","last_undegraded":"2023-06-10T21:36:19.032273+0000","last_fullsized":"2023-06-10T21:36:19.032273+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":245,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035913+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:36:19.035913+0000","last_peered":"2023-06-10T21:36:19.035913+0000","last_clean":"2023-06-10T21:36:19.035913+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:36:19.035913+0000","last_undegraded":"2023-06-10T21:36:19.035913+0000","last_fullsized":"2023-06-10T21:36:19.035913+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":293,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035386+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:36:19.035386+0000","last_peered":"2023-06-10T21:36:19.035386+0000","last_clean":"2023-06-10T21:36:19.035386+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:36:19.035386+0000","last_undegraded":"2023-06-10T21:36:19.035386+0000","last_fullsized":"2023-06-10T21:36:19.035386+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032591+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:36:19.032591+0000","last_peered":"2023-06-10T21:36:19.032591+0000","last_clean":"2023-06-10T21:36:19.032591+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:36:19.032591+0000","last_undegraded":"2023-06-10T21:36:19.032591+0000","last_fullsized":"2023-06-10T21:36:19.032591+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":293,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.031823+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:36:19.031823+0000","last_peered":"2023-06-10T21:36:19.031823+0000","last_clean":"2023-06-10T21:36:19.031823+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:36:19.031823+0000","last_undegraded":"2023-06-10T21:36:19.031823+0000","last_fullsized":"2023-06-10T21:36:19.031823+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2,"reported_epoch":318,"state":"peering","last_fresh":"2023-06-10T21:32:55.506183+0000","last_change":"2023-06-10T21:32:54.532735+0000","last_active":"2023-06-10T21:32:54.532606+0000","last_peered":"2023-06-10T21:32:45.420323+0000","last_clean":"2023-06-10T21:32:45.420323+0000","last_became_active":"2023-06-10T21:31:18.125912+0000","last_became_peered":"2023-06-10T21:31:18.125912+0000","last_unstale":"2023-06-10T21:32:55.506183+0000","last_undegraded":"2023-06-10T21:32:55.506183+0000","last_fullsized":"2023-06-10T21:32:55.506183+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:21:02.524705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.0","version":"318'2035","reported_seq":2051,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-06-10T21:36:34.247808+0000","last_change":"2023-06-10T21:36:04.325572+0000","last_active":"2023-06-10T21:36:34.247808+0000","last_peered":"2023-06-10T21:36:34.247808+0000","last_clean":"2023-06-10T21:36:34.247808+0000","last_became_active":"2023-06-10T21:36:04.325154+0000","last_became_peered":"2023-06-10T21:36:04.325154+0000","last_unstale":"2023-06-10T21:36:34.247808+0000","last_undegraded":"2023-06-10T21:36:34.247808+0000","last_fullsized":"2023-06-10T21:36:34.247808+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2035,"log_dups_size":0,"ondisk_log_size":2035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:53.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133365760,"num_objects":2035,"num_object_clones":0,"num_object_copies":6105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2035,"num_write_kb":130240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":42,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035787+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:36:19.035787+0000","last_peered":"2023-06-10T21:36:19.035787+0000","last_clean":"2023-06-10T21:36:19.035787+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:36:19.035787+0000","last_undegraded":"2023-06-10T21:36:19.035787+0000","last_fullsized":"2023-06-10T21:36:19.035787+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":38,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035243+0000","last_change":"2023-06-10T21:32:55.517863+0000","last_active":"2023-06-10T21:36:19.035243+0000","last_peered":"2023-06-10T21:36:19.035243+0000","last_clean":"2023-06-10T21:36:19.035243+0000","last_became_active":"2023-06-10T21:32:55.517515+0000","last_became_peered":"2023-06-10T21:32:55.517515+0000","last_unstale":"2023-06-10T21:36:19.035243+0000","last_undegraded":"2023-06-10T21:36:19.035243+0000","last_fullsized":"2023-06-10T21:36:19.035243+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"316'1843","reported_seq":1856,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:31.720392+0000","last_change":"2023-06-10T21:36:04.292881+0000","last_active":"2023-06-10T21:36:31.720392+0000","last_peered":"2023-06-10T21:36:31.720392+0000","last_clean":"2023-06-10T21:36:31.720392+0000","last_became_active":"2023-06-10T21:36:04.292606+0000","last_became_peered":"2023-06-10T21:36:04.292606+0000","last_unstale":"2023-06-10T21:36:31.720392+0000","last_undegraded":"2023-06-10T21:36:31.720392+0000","last_fullsized":"2023-06-10T21:36:31.720392+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1843,"log_dups_size":0,"ondisk_log_size":1843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:21.334865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120782848,"num_objects":1843,"num_object_clones":0,"num_object_copies":5529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1843,"num_write_kb":117952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":42,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035811+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:36:19.035811+0000","last_peered":"2023-06-10T21:36:19.035811+0000","last_clean":"2023-06-10T21:36:19.035811+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:36:19.035811+0000","last_undegraded":"2023-06-10T21:36:19.035811+0000","last_fullsized":"2023-06-10T21:36:19.035811+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":190,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304965+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:36:13.304965+0000","last_peered":"2023-06-10T21:36:13.304965+0000","last_clean":"2023-06-10T21:36:13.304965+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:36:13.304965+0000","last_undegraded":"2023-06-10T21:36:13.304965+0000","last_fullsized":"2023-06-10T21:36:13.304965+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"316'1954","reported_seq":1968,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:31.662889+0000","last_change":"2023-06-10T21:36:04.325459+0000","last_active":"2023-06-10T21:36:31.662889+0000","last_peered":"2023-06-10T21:36:31.662889+0000","last_clean":"2023-06-10T21:36:31.662889+0000","last_became_active":"2023-06-10T21:36:04.325111+0000","last_became_peered":"2023-06-10T21:36:04.325111+0000","last_unstale":"2023-06-10T21:36:31.662889+0000","last_undegraded":"2023-06-10T21:36:31.662889+0000","last_fullsized":"2023-06-10T21:36:31.662889+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1954,"log_dups_size":0,"ondisk_log_size":1954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:49:06.786091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128057344,"num_objects":1954,"num_object_clones":0,"num_object_copies":5862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1954,"num_write_kb":125056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":324,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035624+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:36:19.035624+0000","last_peered":"2023-06-10T21:36:19.035624+0000","last_clean":"2023-06-10T21:36:19.035624+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:36:19.035624+0000","last_undegraded":"2023-06-10T21:36:19.035624+0000","last_fullsized":"2023-06-10T21:36:19.035624+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":38,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035368+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:36:19.035368+0000","last_peered":"2023-06-10T21:36:19.035368+0000","last_clean":"2023-06-10T21:36:19.035368+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:36:19.035368+0000","last_undegraded":"2023-06-10T21:36:19.035368+0000","last_fullsized":"2023-06-10T21:36:19.035368+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"317'2076","reported_seq":2091,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.235613+0000","last_change":"2023-06-10T21:36:04.289584+0000","last_active":"2023-06-10T21:36:33.235613+0000","last_peered":"2023-06-10T21:36:33.235613+0000","last_clean":"2023-06-10T21:36:33.235613+0000","last_became_active":"2023-06-10T21:36:04.289267+0000","last_became_peered":"2023-06-10T21:36:04.289267+0000","last_unstale":"2023-06-10T21:36:33.235613+0000","last_undegraded":"2023-06-10T21:36:33.235613+0000","last_fullsized":"2023-06-10T21:36:33.235613+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2076,"log_dups_size":0,"ondisk_log_size":2076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:34.840732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136052736,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2076,"num_write_kb":132864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":198,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305690+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:36:13.305690+0000","last_peered":"2023-06-10T21:36:13.305690+0000","last_clean":"2023-06-10T21:36:13.305690+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:36:13.305690+0000","last_undegraded":"2023-06-10T21:36:13.305690+0000","last_fullsized":"2023-06-10T21:36:13.305690+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032919+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:36:19.032919+0000","last_peered":"2023-06-10T21:36:19.032919+0000","last_clean":"2023-06-10T21:36:19.032919+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:36:19.032919+0000","last_undegraded":"2023-06-10T21:36:19.032919+0000","last_fullsized":"2023-06-10T21:36:19.032919+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"316'1933","reported_seq":1947,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:31.708400+0000","last_change":"2023-06-10T21:36:04.288539+0000","last_active":"2023-06-10T21:36:31.708400+0000","last_peered":"2023-06-10T21:36:31.708400+0000","last_clean":"2023-06-10T21:36:31.708400+0000","last_became_active":"2023-06-10T21:36:04.288222+0000","last_became_peered":"2023-06-10T21:36:04.288222+0000","last_unstale":"2023-06-10T21:36:31.708400+0000","last_undegraded":"2023-06-10T21:36:31.708400+0000","last_fullsized":"2023-06-10T21:36:31.708400+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1933,"log_dups_size":0,"ondisk_log_size":1933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:53.556650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1933,"num_write_kb":123712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":268,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.306181+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:36:13.306181+0000","last_peered":"2023-06-10T21:36:13.306181+0000","last_clean":"2023-06-10T21:36:13.306181+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:36:13.306181+0000","last_undegraded":"2023-06-10T21:36:13.306181+0000","last_fullsized":"2023-06-10T21:36:13.306181+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305767+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:36:13.305767+0000","last_peered":"2023-06-10T21:36:13.305767+0000","last_clean":"2023-06-10T21:36:13.305767+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:36:13.305767+0000","last_undegraded":"2023-06-10T21:36:13.305767+0000","last_fullsized":"2023-06-10T21:36:13.305767+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"318'2144","reported_seq":2160,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-06-10T21:36:34.487991+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:36:34.487991+0000","last_peered":"2023-06-10T21:36:34.487991+0000","last_clean":"2023-06-10T21:36:34.487991+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:36:34.487991+0000","last_undegraded":"2023-06-10T21:36:34.487991+0000","last_fullsized":"2023-06-10T21:36:34.487991+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2144,"log_dups_size":0,"ondisk_log_size":2144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140509184,"num_objects":2144,"num_object_clones":0,"num_object_copies":6432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2144,"num_write_kb":137216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304661+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:36:13.304661+0000","last_peered":"2023-06-10T21:36:13.304661+0000","last_clean":"2023-06-10T21:36:13.304661+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:36:13.304661+0000","last_undegraded":"2023-06-10T21:36:13.304661+0000","last_fullsized":"2023-06-10T21:36:13.304661+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":310,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.046653+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:36:33.046653+0000","last_peered":"2023-06-10T21:36:33.046653+0000","last_clean":"2023-06-10T21:36:33.046653+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:36:33.046653+0000","last_undegraded":"2023-06-10T21:36:33.046653+0000","last_fullsized":"2023-06-10T21:36:33.046653+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"317'1964","reported_seq":1979,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.233998+0000","last_change":"2023-06-10T21:36:04.281931+0000","last_active":"2023-06-10T21:36:33.233998+0000","last_peered":"2023-06-10T21:36:33.233998+0000","last_clean":"2023-06-10T21:36:33.233998+0000","last_became_active":"2023-06-10T21:36:04.281632+0000","last_became_peered":"2023-06-10T21:36:04.281632+0000","last_unstale":"2023-06-10T21:36:33.233998+0000","last_undegraded":"2023-06-10T21:36:33.233998+0000","last_fullsized":"2023-06-10T21:36:33.233998+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1964,"log_dups_size":0,"ondisk_log_size":1964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:57.076427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128712704,"num_objects":1964,"num_object_clones":0,"num_object_copies":5892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1964,"num_write_kb":125696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":296,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.044005+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:36:33.044005+0000","last_peered":"2023-06-10T21:36:33.044005+0000","last_clean":"2023-06-10T21:36:33.044005+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:36:33.044005+0000","last_undegraded":"2023-06-10T21:36:33.044005+0000","last_fullsized":"2023-06-10T21:36:33.044005+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305277+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:36:13.305277+0000","last_peered":"2023-06-10T21:36:13.305277+0000","last_clean":"2023-06-10T21:36:13.305277+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:36:13.305277+0000","last_undegraded":"2023-06-10T21:36:13.305277+0000","last_fullsized":"2023-06-10T21:36:13.305277+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"317'2012","reported_seq":2026,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.899090+0000","last_change":"2023-06-10T21:36:04.289902+0000","last_active":"2023-06-10T21:36:33.899090+0000","last_peered":"2023-06-10T21:36:33.899090+0000","last_clean":"2023-06-10T21:36:33.899090+0000","last_became_active":"2023-06-10T21:36:04.289647+0000","last_became_peered":"2023-06-10T21:36:04.289647+0000","last_unstale":"2023-06-10T21:36:33.899090+0000","last_undegraded":"2023-06-10T21:36:33.899090+0000","last_fullsized":"2023-06-10T21:36:33.899090+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2012,"log_dups_size":0,"ondisk_log_size":2012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131858432,"num_objects":2012,"num_object_clones":0,"num_object_copies":6036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2012,"num_write_kb":128768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":271,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.306324+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:36:13.306324+0000","last_peered":"2023-06-10T21:36:13.306324+0000","last_clean":"2023-06-10T21:36:13.306324+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:36:13.306324+0000","last_undegraded":"2023-06-10T21:36:13.306324+0000","last_fullsized":"2023-06-10T21:36:13.306324+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":227,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305209+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:36:13.305209+0000","last_peered":"2023-06-10T21:36:13.305209+0000","last_clean":"2023-06-10T21:36:13.305209+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:36:13.305209+0000","last_undegraded":"2023-06-10T21:36:13.305209+0000","last_fullsized":"2023-06-10T21:36:13.305209+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"318'2149","reported_seq":2165,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-06-10T21:36:34.380941+0000","last_change":"2023-06-10T21:36:04.291075+0000","last_active":"2023-06-10T21:36:34.380941+0000","last_peered":"2023-06-10T21:36:34.380941+0000","last_clean":"2023-06-10T21:36:34.380941+0000","last_became_active":"2023-06-10T21:36:04.290808+0000","last_became_peered":"2023-06-10T21:36:04.290808+0000","last_unstale":"2023-06-10T21:36:34.380941+0000","last_undegraded":"2023-06-10T21:36:34.380941+0000","last_fullsized":"2023-06-10T21:36:34.380941+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2149,"log_dups_size":0,"ondisk_log_size":2149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:13:41.537032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140836864,"num_objects":2149,"num_object_clones":0,"num_object_copies":6447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2149,"num_write_kb":137536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032846+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:36:19.032846+0000","last_peered":"2023-06-10T21:36:19.032846+0000","last_clean":"2023-06-10T21:36:19.032846+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:36:19.032846+0000","last_undegraded":"2023-06-10T21:36:19.032846+0000","last_fullsized":"2023-06-10T21:36:19.032846+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305285+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:36:13.305285+0000","last_peered":"2023-06-10T21:36:13.305285+0000","last_clean":"2023-06-10T21:36:13.305285+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:36:13.305285+0000","last_undegraded":"2023-06-10T21:36:13.305285+0000","last_fullsized":"2023-06-10T21:36:13.305285+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":161,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032190+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:36:19.032190+0000","last_peered":"2023-06-10T21:36:19.032190+0000","last_clean":"2023-06-10T21:36:19.032190+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:36:19.032190+0000","last_undegraded":"2023-06-10T21:36:19.032190+0000","last_fullsized":"2023-06-10T21:36:19.032190+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"317'2146","reported_seq":2161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.212179+0000","last_change":"2023-06-10T21:36:04.286731+0000","last_active":"2023-06-10T21:36:33.212179+0000","last_peered":"2023-06-10T21:36:33.212179+0000","last_clean":"2023-06-10T21:36:33.212179+0000","last_became_active":"2023-06-10T21:36:04.286427+0000","last_became_peered":"2023-06-10T21:36:04.286427+0000","last_unstale":"2023-06-10T21:36:33.212179+0000","last_undegraded":"2023-06-10T21:36:33.212179+0000","last_fullsized":"2023-06-10T21:36:33.212179+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2146,"log_dups_size":0,"ondisk_log_size":2146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:26:20.203365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140640256,"num_objects":2146,"num_object_clones":0,"num_object_copies":6438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2146,"num_write_kb":137344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":187,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032217+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:36:19.032217+0000","last_peered":"2023-06-10T21:36:19.032217+0000","last_clean":"2023-06-10T21:36:19.032217+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:36:19.032217+0000","last_undegraded":"2023-06-10T21:36:19.032217+0000","last_fullsized":"2023-06-10T21:36:19.032217+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":261,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.033066+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:36:19.033066+0000","last_peered":"2023-06-10T21:36:19.033066+0000","last_clean":"2023-06-10T21:36:19.033066+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:36:19.033066+0000","last_undegraded":"2023-06-10T21:36:19.033066+0000","last_fullsized":"2023-06-10T21:36:19.033066+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.044462+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:36:33.044462+0000","last_peered":"2023-06-10T21:36:33.044462+0000","last_clean":"2023-06-10T21:36:33.044462+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:36:33.044462+0000","last_undegraded":"2023-06-10T21:36:33.044462+0000","last_fullsized":"2023-06-10T21:36:33.044462+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"318'2054","reported_seq":2070,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-06-10T21:36:34.390182+0000","last_change":"2023-06-10T21:36:04.325488+0000","last_active":"2023-06-10T21:36:34.390182+0000","last_peered":"2023-06-10T21:36:34.390182+0000","last_clean":"2023-06-10T21:36:34.390182+0000","last_became_active":"2023-06-10T21:36:04.325162+0000","last_became_peered":"2023-06-10T21:36:04.325162+0000","last_unstale":"2023-06-10T21:36:34.390182+0000","last_undegraded":"2023-06-10T21:36:34.390182+0000","last_fullsized":"2023-06-10T21:36:34.390182+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2054,"log_dups_size":0,"ondisk_log_size":2054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:41:33.200826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134610944,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2054,"num_write_kb":131456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":163,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.044080+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:36:33.044080+0000","last_peered":"2023-06-10T21:36:33.044080+0000","last_clean":"2023-06-10T21:36:33.044080+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:36:33.044080+0000","last_undegraded":"2023-06-10T21:36:33.044080+0000","last_fullsized":"2023-06-10T21:36:33.044080+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.042936+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:36:33.042936+0000","last_peered":"2023-06-10T21:36:33.042936+0000","last_clean":"2023-06-10T21:36:33.042936+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:36:33.042936+0000","last_undegraded":"2023-06-10T21:36:33.042936+0000","last_fullsized":"2023-06-10T21:36:33.042936+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":210,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032534+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:36:19.032534+0000","last_peered":"2023-06-10T21:36:19.032534+0000","last_clean":"2023-06-10T21:36:19.032534+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:36:19.032534+0000","last_undegraded":"2023-06-10T21:36:19.032534+0000","last_fullsized":"2023-06-10T21:36:19.032534+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"317'2035","reported_seq":2049,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.900677+0000","last_change":"2023-06-10T21:36:04.289917+0000","last_active":"2023-06-10T21:36:33.900677+0000","last_peered":"2023-06-10T21:36:33.900677+0000","last_clean":"2023-06-10T21:36:33.900677+0000","last_became_active":"2023-06-10T21:36:04.289698+0000","last_became_peered":"2023-06-10T21:36:04.289698+0000","last_unstale":"2023-06-10T21:36:33.900677+0000","last_undegraded":"2023-06-10T21:36:33.900677+0000","last_fullsized":"2023-06-10T21:36:33.900677+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2035,"log_dups_size":0,"ondisk_log_size":2035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133365760,"num_objects":2035,"num_object_clones":0,"num_object_copies":6105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2035,"num_write_kb":130240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":343,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.043300+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:36:33.043300+0000","last_peered":"2023-06-10T21:36:33.043300+0000","last_clean":"2023-06-10T21:36:33.043300+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:36:33.043300+0000","last_undegraded":"2023-06-10T21:36:33.043300+0000","last_fullsized":"2023-06-10T21:36:33.043300+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":480,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035265+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:36:19.035265+0000","last_peered":"2023-06-10T21:36:19.035265+0000","last_clean":"2023-06-10T21:36:19.035265+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:36:19.035265+0000","last_undegraded":"2023-06-10T21:36:19.035265+0000","last_fullsized":"2023-06-10T21:36:19.035265+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304794+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:36:13.304794+0000","last_peered":"2023-06-10T21:36:13.304794+0000","last_clean":"2023-06-10T21:36:13.304794+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:36:13.304794+0000","last_undegraded":"2023-06-10T21:36:13.304794+0000","last_fullsized":"2023-06-10T21:36:13.304794+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":131,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035840+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:36:19.035840+0000","last_peered":"2023-06-10T21:36:19.035840+0000","last_clean":"2023-06-10T21:36:19.035840+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:36:19.035840+0000","last_undegraded":"2023-06-10T21:36:19.035840+0000","last_fullsized":"2023-06-10T21:36:19.035840+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"317'2056","reported_seq":2071,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.229603+0000","last_change":"2023-06-10T21:36:04.325644+0000","last_active":"2023-06-10T21:36:33.229603+0000","last_peered":"2023-06-10T21:36:33.229603+0000","last_clean":"2023-06-10T21:36:33.229603+0000","last_became_active":"2023-06-10T21:36:04.325301+0000","last_became_peered":"2023-06-10T21:36:04.325301+0000","last_unstale":"2023-06-10T21:36:33.229603+0000","last_undegraded":"2023-06-10T21:36:33.229603+0000","last_fullsized":"2023-06-10T21:36:33.229603+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2056,"log_dups_size":0,"ondisk_log_size":2056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:28:03.018219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134742016,"num_objects":2056,"num_object_clones":0,"num_object_copies":6168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2056,"num_write_kb":131584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":336,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035349+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:36:19.035349+0000","last_peered":"2023-06-10T21:36:19.035349+0000","last_clean":"2023-06-10T21:36:19.035349+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:36:19.035349+0000","last_undegraded":"2023-06-10T21:36:19.035349+0000","last_fullsized":"2023-06-10T21:36:19.035349+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":156,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032016+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:36:19.032016+0000","last_peered":"2023-06-10T21:36:19.032016+0000","last_clean":"2023-06-10T21:36:19.032016+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:36:19.032016+0000","last_undegraded":"2023-06-10T21:36:19.032016+0000","last_fullsized":"2023-06-10T21:36:19.032016+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":141,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032672+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:36:19.032672+0000","last_peered":"2023-06-10T21:36:19.032672+0000","last_clean":"2023-06-10T21:36:19.032672+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:36:19.032672+0000","last_undegraded":"2023-06-10T21:36:19.032672+0000","last_fullsized":"2023-06-10T21:36:19.032672+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"316'1823","reported_seq":1837,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:31.717487+0000","last_change":"2023-06-10T21:36:04.325466+0000","last_active":"2023-06-10T21:36:31.717487+0000","last_peered":"2023-06-10T21:36:31.717487+0000","last_clean":"2023-06-10T21:36:31.717487+0000","last_became_active":"2023-06-10T21:36:04.325178+0000","last_became_peered":"2023-06-10T21:36:04.325178+0000","last_unstale":"2023-06-10T21:36:31.717487+0000","last_undegraded":"2023-06-10T21:36:31.717487+0000","last_fullsized":"2023-06-10T21:36:31.717487+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1823,"log_dups_size":0,"ondisk_log_size":1823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:03.551659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119472128,"num_objects":1823,"num_object_clones":0,"num_object_copies":5469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1823,"num_write_kb":116672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":321,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035667+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:36:19.035667+0000","last_peered":"2023-06-10T21:36:19.035667+0000","last_clean":"2023-06-10T21:36:19.035667+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:36:19.035667+0000","last_undegraded":"2023-06-10T21:36:19.035667+0000","last_fullsized":"2023-06-10T21:36:19.035667+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":161,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.044553+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:36:33.044553+0000","last_peered":"2023-06-10T21:36:33.044553+0000","last_clean":"2023-06-10T21:36:33.044553+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:36:33.044553+0000","last_undegraded":"2023-06-10T21:36:33.044553+0000","last_fullsized":"2023-06-10T21:36:33.044553+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.033165+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:36:19.033165+0000","last_peered":"2023-06-10T21:36:19.033165+0000","last_clean":"2023-06-10T21:36:19.033165+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:36:19.033165+0000","last_undegraded":"2023-06-10T21:36:19.033165+0000","last_fullsized":"2023-06-10T21:36:19.033165+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"316'1961","reported_seq":1975,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:31.686926+0000","last_change":"2023-06-10T21:36:04.292180+0000","last_active":"2023-06-10T21:36:31.686926+0000","last_peered":"2023-06-10T21:36:31.686926+0000","last_clean":"2023-06-10T21:36:31.686926+0000","last_became_active":"2023-06-10T21:36:04.291901+0000","last_became_peered":"2023-06-10T21:36:04.291901+0000","last_unstale":"2023-06-10T21:36:31.686926+0000","last_undegraded":"2023-06-10T21:36:31.686926+0000","last_fullsized":"2023-06-10T21:36:31.686926+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":1961,"log_dups_size":0,"ondisk_log_size":1961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:45.649615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1961,"num_write_kb":125504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":294,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305096+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:36:13.305096+0000","last_peered":"2023-06-10T21:36:13.305096+0000","last_clean":"2023-06-10T21:36:13.305096+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:36:13.305096+0000","last_undegraded":"2023-06-10T21:36:13.305096+0000","last_fullsized":"2023-06-10T21:36:13.305096+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.305132+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:36:13.305132+0000","last_peered":"2023-06-10T21:36:13.305132+0000","last_clean":"2023-06-10T21:36:13.305132+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:36:13.305132+0000","last_undegraded":"2023-06-10T21:36:13.305132+0000","last_fullsized":"2023-06-10T21:36:13.305132+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":275,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304514+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:36:13.304514+0000","last_peered":"2023-06-10T21:36:13.304514+0000","last_clean":"2023-06-10T21:36:13.304514+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:36:13.304514+0000","last_undegraded":"2023-06-10T21:36:13.304514+0000","last_fullsized":"2023-06-10T21:36:13.304514+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"318'2068","reported_seq":2082,"reported_epoch":318,"state":"active+clean","last_fresh":"2023-06-10T21:36:34.391614+0000","last_change":"2023-06-10T21:36:04.296839+0000","last_active":"2023-06-10T21:36:34.391614+0000","last_peered":"2023-06-10T21:36:34.391614+0000","last_clean":"2023-06-10T21:36:34.391614+0000","last_became_active":"2023-06-10T21:36:04.294647+0000","last_became_peered":"2023-06-10T21:36:04.294647+0000","last_unstale":"2023-06-10T21:36:34.391614+0000","last_undegraded":"2023-06-10T21:36:34.391614+0000","last_fullsized":"2023-06-10T21:36:34.391614+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":2068,"log_dups_size":0,"ondisk_log_size":2068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:16.069639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135528448,"num_objects":2068,"num_object_clones":0,"num_object_copies":6204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2068,"num_write_kb":132352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":38,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035871+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:36:19.035871+0000","last_peered":"2023-06-10T21:36:19.035871+0000","last_clean":"2023-06-10T21:36:19.035871+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:36:19.035871+0000","last_undegraded":"2023-06-10T21:36:19.035871+0000","last_fullsized":"2023-06-10T21:36:19.035871+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":145,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032508+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:36:19.032508+0000","last_peered":"2023-06-10T21:36:19.032508+0000","last_clean":"2023-06-10T21:36:19.032508+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:36:19.032508+0000","last_undegraded":"2023-06-10T21:36:19.032508+0000","last_fullsized":"2023-06-10T21:36:19.032508+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":273,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.031893+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:36:19.031893+0000","last_peered":"2023-06-10T21:36:19.031893+0000","last_clean":"2023-06-10T21:36:19.031893+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:36:19.031893+0000","last_undegraded":"2023-06-10T21:36:19.031893+0000","last_fullsized":"2023-06-10T21:36:19.031893+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":141,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.033195+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:36:19.033195+0000","last_peered":"2023-06-10T21:36:19.033195+0000","last_clean":"2023-06-10T21:36:19.033195+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:36:19.033195+0000","last_undegraded":"2023-06-10T21:36:19.033195+0000","last_fullsized":"2023-06-10T21:36:19.033195+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":160,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.044542+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:36:33.044542+0000","last_peered":"2023-06-10T21:36:33.044542+0000","last_clean":"2023-06-10T21:36:33.044542+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:36:33.044542+0000","last_undegraded":"2023-06-10T21:36:33.044542+0000","last_fullsized":"2023-06-10T21:36:33.044542+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":42,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.035596+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:36:19.035596+0000","last_peered":"2023-06-10T21:36:19.035596+0000","last_clean":"2023-06-10T21:36:19.035596+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:36:19.035596+0000","last_undegraded":"2023-06-10T21:36:19.035596+0000","last_fullsized":"2023-06-10T21:36:19.035596+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":199,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.025078+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:36:19.025078+0000","last_peered":"2023-06-10T21:36:19.025078+0000","last_clean":"2023-06-10T21:36:19.025078+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:36:19.025078+0000","last_undegraded":"2023-06-10T21:36:19.025078+0000","last_fullsized":"2023-06-10T21:36:19.025078+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":95,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.024350+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:36:19.024350+0000","last_peered":"2023-06-10T21:36:19.024350+0000","last_clean":"2023-06-10T21:36:19.024350+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:36:19.024350+0000","last_undegraded":"2023-06-10T21:36:19.024350+0000","last_fullsized":"2023-06-10T21:36:19.024350+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":255,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.024460+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:36:19.024460+0000","last_peered":"2023-06-10T21:36:19.024460+0000","last_clean":"2023-06-10T21:36:19.024460+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:36:19.024460+0000","last_undegraded":"2023-06-10T21:36:19.024460+0000","last_fullsized":"2023-06-10T21:36:19.024460+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":300,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304943+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:36:13.304943+0000","last_peered":"2023-06-10T21:36:13.304943+0000","last_clean":"2023-06-10T21:36:13.304943+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:36:13.304943+0000","last_undegraded":"2023-06-10T21:36:13.304943+0000","last_fullsized":"2023-06-10T21:36:13.304943+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":304,"reported_epoch":315,"state":"active+clean","last_fresh":"2023-06-10T21:36:13.304643+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:36:13.304643+0000","last_peered":"2023-06-10T21:36:13.304643+0000","last_clean":"2023-06-10T21:36:13.304643+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:36:13.304643+0000","last_undegraded":"2023-06-10T21:36:13.304643+0000","last_fullsized":"2023-06-10T21:36:13.304643+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":206,"reported_epoch":316,"state":"active+clean","last_fresh":"2023-06-10T21:36:19.032610+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:36:19.032610+0000","last_peered":"2023-06-10T21:36:19.032610+0000","last_clean":"2023-06-10T21:36:19.032610+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:36:19.032610+0000","last_undegraded":"2023-06-10T21:36:19.032610+0000","last_fullsized":"2023-06-10T21:36:19.032610+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":52,"reported_epoch":317,"state":"active+clean","last_fresh":"2023-06-10T21:36:33.042697+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:36:33.042697+0000","last_peered":"2023-06-10T21:36:33.042697+0000","last_clean":"2023-06-10T21:36:33.042697+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:36:33.042697+0000","last_undegraded":"2023-06-10T21:36:33.042697+0000","last_fullsized":"2023-06-10T21:36:33.042697+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2113732608,"num_objects":32253,"num_object_clones":0,"num_object_copies":96759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":32253,"num_write_kb":2064192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6330908672,"data_stored":6330908672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32253,"ondisk_log_size":32253,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430674,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2053916,"kb_used_data":1424724,"kb_used_omap":0,"kb_used_meta":629184,"kb_avail":91691236,"statfs":{"total":95995035648,"available":93891825664,"internally_reserved":0,"allocated":1458917376,"data_stored":1458343241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":644284416},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":4.5199999999999996,"15min":7.4340000000000002},"min":{"1min":0.23200000000000001,"5min":0.224,"15min":0.224},"max":{"1min":28.875,"5min":184.83699999999999,"15min":184.83699999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":3.734,"5min":4.7830000000000004,"15min":7.7130000000000001},"min":{"1min":0.29899999999999999,"5min":0.26400000000000001,"15min":0.223},"max":{"1min":28.943000000000001,"5min":185.58000000000001,"15min":185.58000000000001},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:36:15 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":3.681,"15min":3.4830000000000001},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":54.43,"5min":87.417000000000002,"15min":130.19399999999999},"last":0.54600000000000004},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.1029999999999998,"15min":4.0469999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":54.564,"5min":87.438000000000002,"15min":212.20599999999999},"last":0.70799999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":5.5179999999999998,"15min":4.6100000000000003},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.3170000000000002,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.7720000000000002,"15min":5.3330000000000002},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":3.6949999999999998,"5min":172.185,"15min":212.13399999999999},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:36:14 2023","interfaces":[{"interface":"back","average":{"1min":11.154,"5min":6.0650000000000004,"15min":3.649},"min":{"1min":0.38100000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.00700000000001,"5min":147.00700000000001,"15min":147.00700000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":11.568,"5min":5.8860000000000001,"15min":3.165},"min":{"1min":0.41399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":146.93899999999999,"5min":146.93899999999999,"15min":146.93899999999999},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39000000000000001,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.59499999999999997,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.438,"15min":1.0209999999999999},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.66600000000000004,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":1.8120000000000001},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.68700000000000006,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.45400000000000001,"15min":1.607},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.29299999999999998},"max":{"1min":0.68600000000000005,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.50900000000000001,"15min":1.069},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":3.2559999999999998,"15min":13.073},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.48499999999999999,"15min":0.92900000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":1.1879999999999999,"15min":10.874000000000001},"last":0.47599999999999998}]}]},{"osd":6,"up_from":0,"seq":180388626902,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566344,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1800548,"kb_used_data":1099356,"kb_used_omap":4,"kb_used_meta":701179,"kb_avail":91944604,"statfs":{"total":95995035648,"available":94151274496,"internally_reserved":0,"allocated":1125740544,"data_stored":1124742560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":718008206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:48 2023","interfaces":[{"interface":"back","average":{"1min":3.7559999999999998,"5min":4.9550000000000001,"15min":10.315},"min":{"1min":0.20999999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":28.507999999999999,"5min":154.31399999999999,"15min":154.31399999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":3.714,"5min":4.5700000000000003,"15min":10.029},"min":{"1min":0.27300000000000002,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":28.045999999999999,"5min":154.27199999999999,"15min":154.27199999999999},"last":0.311}]},{"osd":2,"last update":"Sat Jun 10 21:35:41 2023","interfaces":[{"interface":"back","average":{"1min":11.297000000000001,"5min":9.1150000000000002,"15min":5.7110000000000003},"min":{"1min":0.221,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":134.79900000000001,"5min":191.101,"15min":191.101},"last":2.0369999999999999},{"interface":"front","average":{"1min":4.8479999999999999,"5min":7.702,"15min":5.0469999999999997},"min":{"1min":0.27100000000000002,"5min":0.251,"15min":0.218},"max":{"1min":15.923,"5min":190.87899999999999,"15min":190.87899999999999},"last":2.4039999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":7.6280000000000001,"5min":5.734,"15min":3.2719999999999998},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":83.810000000000002,"5min":83.810000000000002,"15min":83.810000000000002},"last":1.889},{"interface":"front","average":{"1min":7.7050000000000001,"5min":5.8680000000000003,"15min":4.2050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":92.757000000000005,"5min":92.757000000000005,"15min":92.757000000000005},"last":2.7530000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.24,"15min":1.5920000000000001},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":7.609,"5min":9.3659999999999997,"15min":16.059999999999999},"last":1.9399999999999999},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.5289999999999999,"15min":1.48},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":5.5389999999999997,"5min":8.8629999999999995,"15min":14.460000000000001},"last":1.8460000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.1099999999999999,"15min":4.3200000000000003},"min":{"1min":0.434,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.959,"5min":11.959,"15min":383.83100000000002},"last":2.1269999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.2509999999999999,"15min":3.847},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":7.6429999999999998,"5min":9.3160000000000007,"15min":383.84800000000001},"last":2.0219999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.1469999999999998,"15min":2.226},"min":{"1min":0.40699999999999997,"5min":0.316,"15min":0.316},"max":{"1min":11.923,"5min":11.923,"15min":14.858000000000001},"last":2.0840000000000001},{"interface":"front","average":{"1min":1.411,"5min":1.4019999999999999,"15min":1.772},"min":{"1min":0.45500000000000002,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":7.5700000000000003,"5min":9.3849999999999998,"15min":13.48},"last":2.052}]},{"osd":7,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.9470000000000001,"15min":2.7090000000000001},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":7.4749999999999996,"5min":47.844000000000001,"15min":162.399},"last":1.98},{"interface":"front","average":{"1min":1.115,"5min":2.3490000000000002,"15min":2.5059999999999998},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":5.5650000000000004,"5min":47.753999999999998,"15min":162.27199999999999},"last":1.8029999999999999}]}]},{"osd":0,"up_from":244,"seq":1047972020321,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069232,"kb_used_data":773224,"kb_used_omap":10,"kb_used_meta":295925,"kb_avail":92675920,"statfs":{"total":95995035648,"available":94900142080,"internally_reserved":0,"allocated":791781376,"data_stored":790710958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":303027252},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":4.0869999999999997,"15min":4.391},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":4.6500000000000004,"5min":139.68000000000001,"15min":139.68000000000001},"last":3.234},{"interface":"front","average":{"1min":4.5839999999999996,"5min":4.5140000000000002,"15min":5.4619999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":15.403,"5min":141.79499999999999,"15min":141.79499999999999},"last":2.7010000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2569999999999997,"5min":7.1600000000000001,"15min":6.7930000000000001},"min":{"1min":0.31,"5min":0.216,"15min":0.216},"max":{"1min":24.166,"5min":132.245,"15min":205.10900000000001},"last":2.5070000000000001},{"interface":"front","average":{"1min":4.0499999999999998,"5min":6.7140000000000004,"15min":6.2770000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":13.06,"5min":132.26599999999999,"15min":208.34299999999999},"last":3.1850000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:36:12 2023","interfaces":[{"interface":"back","average":{"1min":15.250999999999999,"5min":6.3860000000000001,"15min":6.1669999999999998},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":211.864,"5min":211.864,"15min":211.864},"last":2.1659999999999999},{"interface":"front","average":{"1min":14.010999999999999,"5min":6.7969999999999997,"15min":6.9569999999999999},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":207.517,"5min":207.517,"15min":207.517},"last":3.2669999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.9710000000000001,"15min":3.9460000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":8.9550000000000001,"5min":10.744999999999999,"15min":29.709},"last":2.7360000000000002},{"interface":"front","average":{"1min":3.4049999999999998,"5min":2.2440000000000002,"15min":3.5110000000000001},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":24.222999999999999,"5min":24.222999999999999,"15min":24.222999999999999},"last":2.2109999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":2.2639999999999998,"15min":3.5859999999999999},"min":{"1min":0.438,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":24.288,"5min":24.288,"15min":24.288},"last":2.2450000000000001},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.8169999999999999,"15min":3.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":6.9870000000000001,"5min":8.0079999999999991,"15min":19.846},"last":1.3140000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.056,"15min":4.1100000000000003},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":9,"5min":10.771000000000001,"15min":29.806000000000001},"last":2.8039999999999998},{"interface":"front","average":{"1min":3.4430000000000001,"5min":2.3839999999999999,"15min":3.6509999999999998},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":24.260999999999999,"5min":24.260999999999999,"15min":24.260999999999999},"last":2.2919999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.0449999999999999,"15min":3.3839999999999999},"min":{"1min":0.48199999999999998,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.0149999999999997,"5min":17.523,"15min":19.866},"last":1.268},{"interface":"front","average":{"1min":2.843,"5min":2.2120000000000002,"15min":4.1609999999999996},"min":{"1min":0.44800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0259999999999998,"5min":17.792999999999999,"15min":29.77},"last":2.7730000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075733,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1862820,"kb_used_data":1325852,"kb_used_omap":7,"kb_used_meta":536952,"kb_avail":91882332,"statfs":{"total":95995035648,"available":94087507968,"internally_reserved":0,"allocated":1357672448,"data_stored":1356681334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":549838929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:44 2023","interfaces":[{"interface":"back","average":{"1min":4.7000000000000002,"5min":4.6079999999999997,"15min":4.694},"min":{"1min":0.217,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":10.9,"5min":70.510999999999996,"15min":70.510999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":5.0209999999999999,"5min":4.4329999999999998,"15min":5.1479999999999997},"min":{"1min":0.27700000000000002,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":11.733000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":1.1830000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":5.0979999999999999,"15min":3.2799999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.19},"max":{"1min":7.7910000000000004,"5min":147.20099999999999,"15min":147.20099999999999},"last":5.0970000000000004},{"interface":"front","average":{"1min":4.3780000000000001,"5min":4.968,"15min":3.0059999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.214},"max":{"1min":9.4619999999999997,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:36:11 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":6.0970000000000004,"15min":5.3760000000000003},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.15,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":3.7360000000000002,"5min":6.3019999999999996,"15min":6.0659999999999998},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":17.553999999999998,"5min":138.095,"15min":138.095},"last":1.226}]},{"osd":4,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":1.6879999999999999,"15min":2.976},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":224.00999999999999},"last":1.5349999999999999},{"interface":"front","average":{"1min":1.831,"5min":2.3029999999999999,"15min":3.3260000000000001},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":7.7030000000000003,"5min":29.486999999999998,"15min":223.28399999999999},"last":3.1850000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.681,"15min":1.671},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":5.4470000000000001,"5min":10.554,"15min":14.180999999999999},"last":0.98499999999999999},{"interface":"front","average":{"1min":2.7879999999999998,"5min":1.73,"15min":1.8939999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":7.3200000000000003,"5min":7.3200000000000003,"15min":16.125},"last":1.4810000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8140000000000001,"5min":1.802,"15min":1.966},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":7.4180000000000001,"5min":7.4180000000000001,"15min":16.183},"last":1.7370000000000001},{"interface":"front","average":{"1min":2.1339999999999999,"5min":2.4239999999999999,"15min":2.3500000000000001},"min":{"1min":0.39700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.7510000000000003,"5min":29.538,"15min":29.538},"last":3.1389999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:35:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":3.4630000000000001,"15min":2.6429999999999998},"min":{"1min":0.36699999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.8399999999999999,"5min":73.445999999999998,"15min":113.913},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.8010000000000002,"5min":3.4340000000000002,"15min":2.9390000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":7.3620000000000001,"5min":73.475999999999999,"15min":113.937},"last":1.2589999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824092,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1547700,"kb_used_data":1269548,"kb_used_omap":7,"kb_used_meta":278136,"kb_avail":92197452,"statfs":{"total":95995035648,"available":94410190848,"internally_reserved":0,"allocated":1300017152,"data_stored":1298943760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":284811283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":4.4569999999999999,"15min":4.9820000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.1890000000000001,"5min":105.09,"15min":105.09},"last":4.2359999999999998},{"interface":"front","average":{"1min":1.472,"5min":4.4160000000000004,"15min":4.7969999999999997},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":99.852999999999994,"15min":99.852999999999994},"last":3.359}]},{"osd":1,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":4.766,"15min":4.0380000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":10.419,"5min":94.766000000000005,"15min":94.766000000000005},"last":3.7949999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":4.6639999999999997,"15min":3.7320000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.9629999999999992,"5min":94.691000000000003,"15min":94.691000000000003},"last":1.0700000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":3.7690000000000001,"15min":9.1449999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7480000000000002,"5min":100.676,"15min":100.676},"last":3.2759999999999998},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.2400000000000002,"15min":10.669},"min":{"1min":0.30399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.835000000000001,"5min":103.032,"15min":103.032},"last":3.7519999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.446,"15min":1.405},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":3.7770000000000001,"5min":13.417,"15min":13.417},"last":1.3660000000000001},{"interface":"front","average":{"1min":1.244,"5min":2.3780000000000001,"15min":2.6680000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.2030000000000003,"5min":14.384,"15min":19.728000000000002},"last":1.575}]},{"osd":5,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.5169999999999999,"15min":1.397},"min":{"1min":0.377,"5min":0.35699999999999998,"15min":0.32400000000000001},"max":{"1min":1.4079999999999999,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":1.21},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.6060000000000001,"15min":1.4870000000000001},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.7999999999999998,"5min":13.470000000000001,"15min":13.470000000000001},"last":1.222}]},{"osd":6,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.3919999999999999,"15min":2.6899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.1740000000000004,"5min":14.269,"15min":19.805},"last":1.399},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.552,"15min":1.4470000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.39100000000000001},"max":{"1min":1.4379999999999999,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":1.3169999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0760000000000001,"15min":2.9260000000000002},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.1449999999999996,"5min":79.984999999999999,"15min":79.984999999999999},"last":1.4830000000000001},{"interface":"front","average":{"1min":0.72999999999999998,"5min":2.2360000000000002,"15min":1.6579999999999999},"min":{"1min":0.44,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":1.6899999999999999,"5min":80.272999999999996,"15min":80.272999999999996},"last":1.1639999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019358,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266388,"kb_used_data":658380,"kb_used_omap":0,"kb_used_meta":608000,"kb_avail":92478764,"statfs":{"total":95995035648,"available":94698254336,"internally_reserved":0,"allocated":674181120,"data_stored":673613625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":622592000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6819999999999999,"5min":4.1260000000000003,"15min":7.899},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":37.503,"5min":234.203,"15min":234.203},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.2770000000000001,"5min":4.3730000000000002,"15min":7.8220000000000001},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":37.399999999999999,"5min":234.19,"15min":234.19},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":3.0179999999999998,"15min":3.48},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.125,"5min":88.700999999999993,"15min":182.035},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":3.165,"15min":3.3849999999999998},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":3.641,"5min":87.853999999999999,"15min":181.73099999999999},"last":0.627}]},{"osd":2,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":7.4420000000000002,"5min":7.6150000000000002,"15min":3.7549999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":85.308999999999997,"5min":255.386,"15min":255.386},"last":0.59299999999999997},{"interface":"front","average":{"1min":6.4720000000000004,"5min":7.5499999999999998,"15min":3.6850000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":85.322000000000003,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3959999999999999,"5min":8.0939999999999994,"15min":4.2489999999999997},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.28399999999999997},"max":{"1min":36.366999999999997,"5min":234.91900000000001,"15min":234.91900000000001},"last":0.84299999999999997},{"interface":"front","average":{"1min":3.6619999999999999,"5min":8.4410000000000007,"15min":4.1900000000000004},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":38.659999999999997,"5min":235.05099999999999,"15min":235.05099999999999},"last":1.1910000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.379,"15min":0.99299999999999999},"min":{"1min":0.29699999999999999,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.54300000000000004,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.45700000000000002,"15min":1.004},"min":{"1min":0.33200000000000002,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":0.45500000000000002,"5min":5.25,"15min":11.904999999999999},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42899999999999999,"15min":1.0149999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.67700000000000005,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46600000000000003,"15min":1.079},"min":{"1min":0.35499999999999998,"5min":0.34699999999999998,"15min":0.309},"max":{"1min":0.63400000000000001,"5min":1.71,"15min":13.845000000000001},"last":0.501}]},{"osd":7,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.77900000000000003,"15min":1.2669999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":0.65300000000000002,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.441},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.79200000000000004,"15min":1.21},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.68500000000000005,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.56599999999999995}]}]},{"osd":5,"up_from":0,"seq":154618823130,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":811204608,"data_stored":811204608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":861732864,"data_stored":861732864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1363738624,"data_stored":1363738624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1135411200,"data_stored":1135411200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":671678464,"data_stored":671678464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1487142912,"data_stored":1487142912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:36:39.028 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:36:39.104 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:39.212 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:39.223 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:39.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:39.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:39.609 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:39.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:39.691 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:39.743 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 39051 39035 93.797 66.875 0.0189533 0.0106538 2023-06-10T21:36:39.744 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:39.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:39.819 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:36:39.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:39.864 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:36:39.884 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:39.910 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:36:39.933 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:39.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:39.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3701593710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:39.974 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3701593710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:39.974 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3701593710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:39.978 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:36:40.017 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:36:40.040 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:40.057 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:36:40.095 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:36:40.142 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:36:40.150 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.150 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.150 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.150 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.151 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.151 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.151 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.151 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:36:40.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:40.744 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 40314 40298 93.2464 78.9375 0.010387 0.0107191 2023-06-10T21:36:41.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:40 smithi064 ceph-mon[180297]: pgmap v1178: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:41.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:40 smithi064 ceph-mon[186706]: pgmap v1178: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:41.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:40 smithi152 ceph-mon[156819]: pgmap v1178: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-10T21:36:41.744 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 41622 41606 92.8357 81.75 0.00738646 0.0107661 2023-06-10T21:36:42.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:41 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:36:42.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:41 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:36:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:41 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:36:42.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:41 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:36:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:41 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-10T21:36:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:41 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:36:42.746 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 42804 42788 92.1765 73.875 0.0104943 0.0108434 2023-06-10T21:36:42.881 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:42 smithi064 ceph-mon[186706]: pgmap v1179: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:36:42.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:42 smithi064 ceph-mon[180297]: pgmap v1179: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:36:42.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:42.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:43.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:43.095 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:43.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:42] "GET /metrics HTTP/1.1" 200 224103 "" "Prometheus/2.33.4" 2023-06-10T21:36:43.161 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:43.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:42 smithi152 ceph-mon[156819]: pgmap v1179: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-06-10T21:36:43.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:43.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:43.270 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:43.342 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:43.551 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:43.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:43.581 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:43.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:43.726 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:43.745 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 43852 43836 91.2874 65.5 0.0217705 0.0109465 2023-06-10T21:36:43.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:43.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:44.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:44 smithi152 ceph-mon[156819]: pgmap v1180: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-10T21:36:44.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:44.155 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:44.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:44 smithi064 ceph-mon[186706]: pgmap v1180: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-10T21:36:44.327 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:44 smithi064 ceph-mon[180297]: pgmap v1180: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-10T21:36:44.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:44.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:44.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:44.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:44.741 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:44.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 45023 45007 90.6969 73.1875 0.00667943 0.0110199 2023-06-10T21:36:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:44.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:44.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:44.870 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:44.870 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:44.887 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:44.939 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:44.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:45.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:45.145 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:45.299 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:36:45.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:36:45.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/836755333' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.632 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/836755333' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/836755333' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:36:45.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 46197 46181 90.1556 73.375 0.0275696 0.0110854 2023-06-10T21:36:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: osdmap e320: 8 total, 8 up, 6 in 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[180297]: pgmap v1182: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:46.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: osdmap e320: 8 total, 8 up, 6 in 2023-06-10T21:36:46.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:46 smithi064 ceph-mon[186706]: pgmap v1182: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:36:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:36:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:36:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: osdmap e320: 8 total, 8 up, 6 in 2023-06-10T21:36:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: failed to encode map e320 with expected crc 2023-06-10T21:36:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:46 smithi152 ceph-mon[156819]: pgmap v1182: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:36:46.749 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 47176 47160 89.2779 61.1875 0.0096038 0.0111963 2023-06-10T21:36:47.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:47.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:47.547 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:47.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:47.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 48099 48083 88.3489 57.6875 0.00302052 0.0112729 2023-06-10T21:36:47.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:36:48.182 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:48.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:48.293 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:48 smithi064 ceph-mon[186706]: pgmap v1183: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-06-10T21:36:48.293 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:48 smithi064 ceph-mon[180297]: pgmap v1183: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-06-10T21:36:48.303 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:48.404 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:48 smithi152 ceph-mon[156819]: pgmap v1183: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-06-10T21:36:48.560 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:48.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:48.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:48.701 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:48.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 49003 48987 87.4391 56.5 0.0193505 0.011429 2023-06-10T21:36:48.764 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:48.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:48.846 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:48.871 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:48.905 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:48.946 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:49.121 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:49.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:49.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:49.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.233 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:49.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.235 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:49.235 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.255 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:49.306 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:49.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:49.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:49.381 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:49.381 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1682,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":320,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":6,"osd_in_since":1686432375,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":45018,"data_bytes":2949322638,"bytes_used":12332027904,"bytes_avail":563638185984,"bytes_total":575970213888,"write_bytes_sec":72997210,"read_op_per_sec":0,"write_op_per_sec":1113},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":26,"modified":"2023-06-10T21:36:35.932491+0000","services":{}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:36:49.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:36:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:36:49.482 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:49.715 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:49.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:49.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 49982 49966 86.71 61.1875 0.00856869 0.0115278 2023-06-10T21:36:49.825 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:49.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:50.342 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:36:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2205272428' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:50 smithi152 ceph-mon[156819]: pgmap v1184: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:36:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2205272428' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:50 smithi064 ceph-mon[180297]: pgmap v1184: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:36:50.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2205272428' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:36:50.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:50 smithi064 ceph-mon[186706]: pgmap v1184: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-06-10T21:36:50.748 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 50863 50847 85.8547 55.0625 0.00481393 0.0116239 2023-06-10T21:36:51.393 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:51.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:51.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:51.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:51.531 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:51.580 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:51.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:51.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:51.686 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:51.749 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 51895 51879 85.2927 64.5 0.0215359 0.011717 2023-06-10T21:36:51.763 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-06-10T21:36:51.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:36:51.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:52.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:51 smithi064 ceph-mon[186706]: pgmap v1185: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:36:52.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:51 smithi064 ceph-mon[180297]: pgmap v1185: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:36:52.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:51 smithi152 ceph-mon[156819]: pgmap v1185: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:36:52.602 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:52.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:52.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:52.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:52.743 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:52.749 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 52657 52641 84.3271 47.625 0.011348 0.0118532 2023-06-10T21:36:52.794 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:52.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:52.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:52.928 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:52.982 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:52.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:52.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:53.029 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:36:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:36:52] "GET /metrics HTTP/1.1" 200 224120 "" "Prometheus/2.33.4" 2023-06-10T21:36:53.089 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:53.120 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:53.173 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:53.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:53.281 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:53.372 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:53.466 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:53.496 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:53.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:53.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:53.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:53.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:53.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:36:53.754224+0000 min lat: 0.00116179 max lat: 0.261793 avg lat: 0.0119271 2023-06-10T21:36:53.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:36:53.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 53679 53663 83.8012 63.875 0.015928 0.0119271 2023-06-10T21:36:53.982 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:54.047 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:54 smithi064 ceph-mon[180297]: pgmap v1186: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:36:54.087 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:54.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:54.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:54.138 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:36:54.138 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:36:54.163 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:36:54.197 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:36:54.200 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:36:54.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:36:54.275 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:54.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:36:54.336 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:36:54.376 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:36:54.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:54 smithi064 ceph-mon[186706]: pgmap v1186: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:36:54.411 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:36:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:54 smithi152 ceph-mon[156819]: pgmap v1186: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:36:54.448 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:36:54.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.451 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:36:54.664 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:36:54.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:54.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:36:54.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 54661 54645 83.2543 61.375 0.00410229 0.0120058 2023-06-10T21:36:54.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:54.845 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:55.051 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:55 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/65714509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:55.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:55 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/65714509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:55 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/65714509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:36:55.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 55615 55599 82.6918 59.625 0.0102289 0.0120868 2023-06-10T21:36:55.822 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:36:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:55.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:36:55.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:36:55.963 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:56.014 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:36:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:56.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:36:56.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:56.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:36:56.155 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:56.212 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:56.215 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:56 smithi152 ceph-mon[156819]: pgmap v1187: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-06-10T21:36:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:56.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:56.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:56.371 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:56 smithi064 ceph-mon[186706]: pgmap v1187: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-06-10T21:36:56.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:56 smithi064 ceph-mon[180297]: pgmap v1187: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 58 MiB/s wr, 934 op/s 2023-06-10T21:36:56.374 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:56.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 56478 56462 82.0232 53.9375 0.0365996 0.012186 2023-06-10T21:36:57.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:57.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:57.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:57.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:36:57.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:57.756 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 57353 57337 81.4016 54.6875 0.0148359 0.012279 2023-06-10T21:36:57.863 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:36:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:57.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:36:57.972 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:58.054 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:58.146 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:36:58 smithi064 ceph-mon[186706]: pgmap v1188: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:36:58.146 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:36:58 smithi064 ceph-mon[180297]: pgmap v1188: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:36:58.165 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:58.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:58.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:58.315 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:36:58 smithi152 ceph-mon[156819]: pgmap v1188: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s 2023-06-10T21:36:58.317 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:58.372 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:58.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:58.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:58.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:36:58.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:58.567 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:36:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:58.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:36:58.676 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:36:58.706 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:58.762 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 58445 58429 81.0992 68.25 0.0093302 0.0123255 2023-06-10T21:36:58.766 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:36:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:58.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:36:58.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:36:58.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:59.114 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:36:59.178 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:36:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:59.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:36:59.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:59.320 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:59.368 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:59.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:59.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:59.477 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:36:59.504 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:36:59.556 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:36:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:36:59.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:36:59.644 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:36:59.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:36:59.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:36:59.676 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:36:59.763 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 59361 59345 80.5797 57.25 0.0165592 0.0124024 2023-06-10T21:36:59.764 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:00.040 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:00 smithi064 ceph-mon[180297]: from='client.56365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:00.041 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:00 smithi064 ceph-mon[180297]: pgmap v1189: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:37:00.343 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:00.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:00.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:00.359 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:00 smithi064 ceph-mon[186706]: from='client.56365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:00.359 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:00 smithi064 ceph-mon[186706]: pgmap v1189: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:37:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:00 smithi152 ceph-mon[156819]: from='client.56365 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:00 smithi152 ceph-mon[156819]: pgmap v1189: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-10T21:37:00.459 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:00.485 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:00.527 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:00.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:00.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:00.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:00.660 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:00.764 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 60024 60008 79.7461 41.4375 0.0211026 0.0125331 2023-06-10T21:37:01.048 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:37:01.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2028756002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2028756002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2028756002' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:01.764 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 60506 60490 78.7128 30.125 0.00220215 0.0126587 2023-06-10T21:37:02.144 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:02.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:02.253 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:02.282 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:02.329 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: osdmap e321: 8 total, 8 up, 6 in 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:02 smithi152 ceph-mon[156819]: pgmap v1191: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-10T21:37:02.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:02.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:02.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:02.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:02.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: osdmap e321: 8 total, 8 up, 6 in 2023-06-10T21:37:02.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[186706]: pgmap v1191: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-10T21:37:02.346 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1188,"stamp":"2023-06-10T21:36:57.943017+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3583186830,"num_objects":54690,"num_object_clones":0,"num_object_copies":164070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54690,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":54927,"num_write_kb":3501070,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":1777362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54861,"ondisk_log_size":54861,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13901404,"kb_used_data":10773612,"kb_used_omap":30,"kb_used_meta":3127713,"kb_avail":548569508,"statfs":{"total":575970213888,"available":561735176192,"internally_reserved":0,"allocated":11032178688,"data_stored":11026875956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":3202778150},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":746979328,"num_objects":11398,"num_object_clones":0,"num_object_copies":34194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11398,"num_write_kb":729472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005655"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":314,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.205045+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:36:35.205045+0000","last_peered":"2023-06-10T21:36:35.205045+0000","last_clean":"2023-06-10T21:36:35.205045+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:36:35.205045+0000","last_undegraded":"2023-06-10T21:36:35.205045+0000","last_fullsized":"2023-06-10T21:36:35.205045+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213927+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:36:35.213927+0000","last_peered":"2023-06-10T21:36:35.213927+0000","last_clean":"2023-06-10T21:36:35.213927+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:36:35.213927+0000","last_undegraded":"2023-06-10T21:36:35.213927+0000","last_fullsized":"2023-06-10T21:36:35.213927+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":230,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213549+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:36:35.213549+0000","last_peered":"2023-06-10T21:36:35.213549+0000","last_clean":"2023-06-10T21:36:35.213549+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:36:35.213549+0000","last_undegraded":"2023-06-10T21:36:35.213549+0000","last_fullsized":"2023-06-10T21:36:35.213549+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":178,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207853+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:36:35.207853+0000","last_peered":"2023-06-10T21:36:35.207853+0000","last_clean":"2023-06-10T21:36:35.207853+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:36:35.207853+0000","last_undegraded":"2023-06-10T21:36:35.207853+0000","last_fullsized":"2023-06-10T21:36:35.207853+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":295,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207689+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:36:35.207689+0000","last_peered":"2023-06-10T21:36:35.207689+0000","last_clean":"2023-06-10T21:36:35.207689+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:36:35.207689+0000","last_undegraded":"2023-06-10T21:36:35.207689+0000","last_fullsized":"2023-06-10T21:36:35.207689+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":288,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040497+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:36:46.040497+0000","last_peered":"2023-06-10T21:36:46.040497+0000","last_clean":"2023-06-10T21:36:46.040497+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:36:46.040497+0000","last_undegraded":"2023-06-10T21:36:46.040497+0000","last_fullsized":"2023-06-10T21:36:46.040497+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":177,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207871+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:36:35.207871+0000","last_peered":"2023-06-10T21:36:35.207871+0000","last_clean":"2023-06-10T21:36:35.207871+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:36:35.207871+0000","last_undegraded":"2023-06-10T21:36:35.207871+0000","last_fullsized":"2023-06-10T21:36:35.207871+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":269,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207063+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:36:35.207063+0000","last_peered":"2023-06-10T21:36:35.207063+0000","last_clean":"2023-06-10T21:36:35.207063+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:36:35.207063+0000","last_undegraded":"2023-06-10T21:36:35.207063+0000","last_fullsized":"2023-06-10T21:36:35.207063+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":41,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202574+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:36:35.202574+0000","last_peered":"2023-06-10T21:36:35.202574+0000","last_clean":"2023-06-10T21:36:35.202574+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:36:35.202574+0000","last_undegraded":"2023-06-10T21:36:35.202574+0000","last_fullsized":"2023-06-10T21:36:35.202574+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":43,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.212584+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:36:35.212584+0000","last_peered":"2023-06-10T21:36:35.212584+0000","last_clean":"2023-06-10T21:36:35.212584+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:36:35.212584+0000","last_undegraded":"2023-06-10T21:36:35.212584+0000","last_fullsized":"2023-06-10T21:36:35.212584+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":47,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202372+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:36:35.202372+0000","last_peered":"2023-06-10T21:36:35.202372+0000","last_clean":"2023-06-10T21:36:35.202372+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:36:35.202372+0000","last_undegraded":"2023-06-10T21:36:35.202372+0000","last_fullsized":"2023-06-10T21:36:35.202372+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":283,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.204949+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:36:35.204949+0000","last_peered":"2023-06-10T21:36:35.204949+0000","last_clean":"2023-06-10T21:36:35.204949+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:36:35.204949+0000","last_undegraded":"2023-06-10T21:36:35.204949+0000","last_fullsized":"2023-06-10T21:36:35.204949+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":190,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202269+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:36:35.202269+0000","last_peered":"2023-06-10T21:36:35.202269+0000","last_clean":"2023-06-10T21:36:35.202269+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:36:35.202269+0000","last_undegraded":"2023-06-10T21:36:35.202269+0000","last_fullsized":"2023-06-10T21:36:35.202269+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202095+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:36:35.202095+0000","last_peered":"2023-06-10T21:36:35.202095+0000","last_clean":"2023-06-10T21:36:35.202095+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:36:35.202095+0000","last_undegraded":"2023-06-10T21:36:35.202095+0000","last_fullsized":"2023-06-10T21:36:35.202095+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202112+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:36:35.202112+0000","last_peered":"2023-06-10T21:36:35.202112+0000","last_clean":"2023-06-10T21:36:35.202112+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:36:35.202112+0000","last_undegraded":"2023-06-10T21:36:35.202112+0000","last_fullsized":"2023-06-10T21:36:35.202112+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":255,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.212864+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:36:35.212864+0000","last_peered":"2023-06-10T21:36:35.212864+0000","last_clean":"2023-06-10T21:36:35.212864+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:36:35.212864+0000","last_undegraded":"2023-06-10T21:36:35.212864+0000","last_fullsized":"2023-06-10T21:36:35.212864+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":177,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207715+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:36:35.207715+0000","last_peered":"2023-06-10T21:36:35.207715+0000","last_clean":"2023-06-10T21:36:35.207715+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:36:35.207715+0000","last_undegraded":"2023-06-10T21:36:35.207715+0000","last_fullsized":"2023-06-10T21:36:35.207715+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":248,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202598+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:36:35.202598+0000","last_peered":"2023-06-10T21:36:35.202598+0000","last_clean":"2023-06-10T21:36:35.202598+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:36:35.202598+0000","last_undegraded":"2023-06-10T21:36:35.202598+0000","last_fullsized":"2023-06-10T21:36:35.202598+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":296,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202084+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:36:35.202084+0000","last_peered":"2023-06-10T21:36:35.202084+0000","last_clean":"2023-06-10T21:36:35.202084+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:36:35.202084+0000","last_undegraded":"2023-06-10T21:36:35.202084+0000","last_fullsized":"2023-06-10T21:36:35.202084+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":177,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.208040+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:36:35.208040+0000","last_peered":"2023-06-10T21:36:35.208040+0000","last_clean":"2023-06-10T21:36:35.208040+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:36:35.208040+0000","last_undegraded":"2023-06-10T21:36:35.208040+0000","last_fullsized":"2023-06-10T21:36:35.208040+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":296,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207779+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:36:35.207779+0000","last_peered":"2023-06-10T21:36:35.207779+0000","last_clean":"2023-06-10T21:36:35.207779+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:36:35.207779+0000","last_undegraded":"2023-06-10T21:36:35.207779+0000","last_fullsized":"2023-06-10T21:36:35.207779+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.214719+0000","last_change":"2023-06-10T21:36:35.210652+0000","last_active":"2023-06-10T21:36:35.214719+0000","last_peered":"2023-06-10T21:36:35.214719+0000","last_clean":"2023-06-10T21:36:35.214719+0000","last_became_active":"2023-06-10T21:36:35.210329+0000","last_became_peered":"2023-06-10T21:36:35.210329+0000","last_unstale":"2023-06-10T21:36:35.214719+0000","last_undegraded":"2023-06-10T21:36:35.214719+0000","last_fullsized":"2023-06-10T21:36:35.214719+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:21:02.524705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.0","version":"320'3375","reported_seq":3393,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.500142+0000","last_change":"2023-06-10T21:36:04.325572+0000","last_active":"2023-06-10T21:36:54.500142+0000","last_peered":"2023-06-10T21:36:54.500142+0000","last_clean":"2023-06-10T21:36:54.500142+0000","last_became_active":"2023-06-10T21:36:04.325154+0000","last_became_peered":"2023-06-10T21:36:04.325154+0000","last_unstale":"2023-06-10T21:36:54.500142+0000","last_undegraded":"2023-06-10T21:36:54.500142+0000","last_fullsized":"2023-06-10T21:36:54.500142+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3375,"log_dups_size":0,"ondisk_log_size":3375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:53.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221184000,"num_objects":3375,"num_object_clones":0,"num_object_copies":10125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3375,"num_write_kb":216000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":45,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202729+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:36:35.202729+0000","last_peered":"2023-06-10T21:36:35.202729+0000","last_clean":"2023-06-10T21:36:35.202729+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:36:35.202729+0000","last_undegraded":"2023-06-10T21:36:35.202729+0000","last_fullsized":"2023-06-10T21:36:35.202729+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.214455+0000","last_change":"2023-06-10T21:36:35.214211+0000","last_active":"2023-06-10T21:36:35.214455+0000","last_peered":"2023-06-10T21:36:35.214455+0000","last_clean":"2023-06-10T21:36:35.214455+0000","last_became_active":"2023-06-10T21:36:35.213859+0000","last_became_peered":"2023-06-10T21:36:35.213859+0000","last_unstale":"2023-06-10T21:36:35.214455+0000","last_undegraded":"2023-06-10T21:36:35.214455+0000","last_fullsized":"2023-06-10T21:36:35.214455+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"320'3428","reported_seq":3446,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.716480+0000","last_change":"2023-06-10T21:36:04.292881+0000","last_active":"2023-06-10T21:36:56.716480+0000","last_peered":"2023-06-10T21:36:56.716480+0000","last_clean":"2023-06-10T21:36:56.716480+0000","last_became_active":"2023-06-10T21:36:04.292606+0000","last_became_peered":"2023-06-10T21:36:04.292606+0000","last_unstale":"2023-06-10T21:36:56.716480+0000","last_undegraded":"2023-06-10T21:36:56.716480+0000","last_fullsized":"2023-06-10T21:36:56.716480+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3428,"log_dups_size":0,"ondisk_log_size":3428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:21.334865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224657408,"num_objects":3428,"num_object_clones":0,"num_object_copies":10284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3428,"num_write_kb":219392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":45,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202635+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:36:35.202635+0000","last_peered":"2023-06-10T21:36:35.202635+0000","last_clean":"2023-06-10T21:36:35.202635+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:36:35.202635+0000","last_undegraded":"2023-06-10T21:36:35.202635+0000","last_fullsized":"2023-06-10T21:36:35.202635+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213718+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:36:35.213718+0000","last_peered":"2023-06-10T21:36:35.213718+0000","last_clean":"2023-06-10T21:36:35.213718+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:36:35.213718+0000","last_undegraded":"2023-06-10T21:36:35.213718+0000","last_fullsized":"2023-06-10T21:36:35.213718+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"320'3505","reported_seq":3523,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.642120+0000","last_change":"2023-06-10T21:36:04.325459+0000","last_active":"2023-06-10T21:36:56.642120+0000","last_peered":"2023-06-10T21:36:56.642120+0000","last_clean":"2023-06-10T21:36:56.642120+0000","last_became_active":"2023-06-10T21:36:04.325111+0000","last_became_peered":"2023-06-10T21:36:04.325111+0000","last_unstale":"2023-06-10T21:36:56.642120+0000","last_undegraded":"2023-06-10T21:36:56.642120+0000","last_fullsized":"2023-06-10T21:36:56.642120+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3505,"log_dups_size":0,"ondisk_log_size":3505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:49:06.786091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229703680,"num_objects":3505,"num_object_clones":0,"num_object_copies":10515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3505,"num_write_kb":224320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":327,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202438+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:36:35.202438+0000","last_peered":"2023-06-10T21:36:35.202438+0000","last_clean":"2023-06-10T21:36:35.202438+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:36:35.202438+0000","last_undegraded":"2023-06-10T21:36:35.202438+0000","last_fullsized":"2023-06-10T21:36:35.202438+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":41,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202030+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:36:35.202030+0000","last_peered":"2023-06-10T21:36:35.202030+0000","last_clean":"2023-06-10T21:36:35.202030+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:36:35.202030+0000","last_undegraded":"2023-06-10T21:36:35.202030+0000","last_fullsized":"2023-06-10T21:36:35.202030+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"320'3384","reported_seq":3402,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.249724+0000","last_change":"2023-06-10T21:36:04.289584+0000","last_active":"2023-06-10T21:36:53.249724+0000","last_peered":"2023-06-10T21:36:53.249724+0000","last_clean":"2023-06-10T21:36:53.249724+0000","last_became_active":"2023-06-10T21:36:04.289267+0000","last_became_peered":"2023-06-10T21:36:04.289267+0000","last_unstale":"2023-06-10T21:36:53.249724+0000","last_undegraded":"2023-06-10T21:36:53.249724+0000","last_fullsized":"2023-06-10T21:36:53.249724+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3384,"log_dups_size":0,"ondisk_log_size":3384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:34.840732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221773824,"num_objects":3384,"num_object_clones":0,"num_object_copies":10152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3384,"num_write_kb":216576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":202,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213110+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:36:35.213110+0000","last_peered":"2023-06-10T21:36:35.213110+0000","last_clean":"2023-06-10T21:36:35.213110+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:36:35.213110+0000","last_undegraded":"2023-06-10T21:36:35.213110+0000","last_fullsized":"2023-06-10T21:36:35.213110+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207305+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:36:35.207305+0000","last_peered":"2023-06-10T21:36:35.207305+0000","last_clean":"2023-06-10T21:36:35.207305+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:36:35.207305+0000","last_undegraded":"2023-06-10T21:36:35.207305+0000","last_fullsized":"2023-06-10T21:36:35.207305+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"320'3537","reported_seq":3555,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.716780+0000","last_change":"2023-06-10T21:36:04.288539+0000","last_active":"2023-06-10T21:36:56.716780+0000","last_peered":"2023-06-10T21:36:56.716780+0000","last_clean":"2023-06-10T21:36:56.716780+0000","last_became_active":"2023-06-10T21:36:04.288222+0000","last_became_peered":"2023-06-10T21:36:04.288222+0000","last_unstale":"2023-06-10T21:36:56.716780+0000","last_undegraded":"2023-06-10T21:36:56.716780+0000","last_fullsized":"2023-06-10T21:36:56.716780+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3537,"log_dups_size":0,"ondisk_log_size":3537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:53.556650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231800832,"num_objects":3537,"num_object_clones":0,"num_object_copies":10611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3537,"num_write_kb":226368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":272,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213213+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:36:35.213213+0000","last_peered":"2023-06-10T21:36:35.213213+0000","last_clean":"2023-06-10T21:36:35.213213+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:36:35.213213+0000","last_undegraded":"2023-06-10T21:36:35.213213+0000","last_fullsized":"2023-06-10T21:36:35.213213+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213190+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:36:35.213190+0000","last_peered":"2023-06-10T21:36:35.213190+0000","last_clean":"2023-06-10T21:36:35.213190+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:36:35.213190+0000","last_undegraded":"2023-06-10T21:36:35.213190+0000","last_fullsized":"2023-06-10T21:36:35.213190+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"320'3417","reported_seq":3435,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.452840+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:36:54.452840+0000","last_peered":"2023-06-10T21:36:54.452840+0000","last_clean":"2023-06-10T21:36:54.452840+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:36:54.452840+0000","last_undegraded":"2023-06-10T21:36:54.452840+0000","last_fullsized":"2023-06-10T21:36:54.452840+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3417,"log_dups_size":0,"ondisk_log_size":3417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223936512,"num_objects":3417,"num_object_clones":0,"num_object_copies":10251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3417,"num_write_kb":218688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":95,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.214107+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:36:35.214107+0000","last_peered":"2023-06-10T21:36:35.214107+0000","last_clean":"2023-06-10T21:36:35.214107+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:36:35.214107+0000","last_undegraded":"2023-06-10T21:36:35.214107+0000","last_fullsized":"2023-06-10T21:36:35.214107+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":312,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.205569+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:36:35.205569+0000","last_peered":"2023-06-10T21:36:35.205569+0000","last_clean":"2023-06-10T21:36:35.205569+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:36:35.205569+0000","last_undegraded":"2023-06-10T21:36:35.205569+0000","last_fullsized":"2023-06-10T21:36:35.205569+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"320'3285","reported_seq":3303,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.243935+0000","last_change":"2023-06-10T21:36:04.281931+0000","last_active":"2023-06-10T21:36:53.243935+0000","last_peered":"2023-06-10T21:36:53.243935+0000","last_clean":"2023-06-10T21:36:53.243935+0000","last_became_active":"2023-06-10T21:36:04.281632+0000","last_became_peered":"2023-06-10T21:36:04.281632+0000","last_unstale":"2023-06-10T21:36:53.243935+0000","last_undegraded":"2023-06-10T21:36:53.243935+0000","last_fullsized":"2023-06-10T21:36:53.243935+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3285,"log_dups_size":0,"ondisk_log_size":3285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:57.076427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215285760,"num_objects":3285,"num_object_clones":0,"num_object_copies":9855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3285,"num_write_kb":210240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":299,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.052905+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:36:46.052905+0000","last_peered":"2023-06-10T21:36:46.052905+0000","last_clean":"2023-06-10T21:36:46.052905+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:36:46.052905+0000","last_undegraded":"2023-06-10T21:36:46.052905+0000","last_fullsized":"2023-06-10T21:36:46.052905+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":95,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213283+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:36:35.213283+0000","last_peered":"2023-06-10T21:36:35.213283+0000","last_clean":"2023-06-10T21:36:35.213283+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:36:35.213283+0000","last_undegraded":"2023-06-10T21:36:35.213283+0000","last_fullsized":"2023-06-10T21:36:35.213283+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"320'3348","reported_seq":3364,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.934857+0000","last_change":"2023-06-10T21:36:04.289902+0000","last_active":"2023-06-10T21:36:53.934857+0000","last_peered":"2023-06-10T21:36:53.934857+0000","last_clean":"2023-06-10T21:36:53.934857+0000","last_became_active":"2023-06-10T21:36:04.289647+0000","last_became_peered":"2023-06-10T21:36:04.289647+0000","last_unstale":"2023-06-10T21:36:53.934857+0000","last_undegraded":"2023-06-10T21:36:53.934857+0000","last_fullsized":"2023-06-10T21:36:53.934857+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3348,"log_dups_size":0,"ondisk_log_size":3348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219414528,"num_objects":3348,"num_object_clones":0,"num_object_copies":10044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3348,"num_write_kb":214272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":275,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207557+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:36:35.207557+0000","last_peered":"2023-06-10T21:36:35.207557+0000","last_clean":"2023-06-10T21:36:35.207557+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:36:35.207557+0000","last_undegraded":"2023-06-10T21:36:35.207557+0000","last_fullsized":"2023-06-10T21:36:35.207557+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":231,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213325+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:36:35.213325+0000","last_peered":"2023-06-10T21:36:35.213325+0000","last_clean":"2023-06-10T21:36:35.213325+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:36:35.213325+0000","last_undegraded":"2023-06-10T21:36:35.213325+0000","last_fullsized":"2023-06-10T21:36:35.213325+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"320'3430","reported_seq":3448,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.498687+0000","last_change":"2023-06-10T21:36:04.291075+0000","last_active":"2023-06-10T21:36:54.498687+0000","last_peered":"2023-06-10T21:36:54.498687+0000","last_clean":"2023-06-10T21:36:54.498687+0000","last_became_active":"2023-06-10T21:36:04.290808+0000","last_became_peered":"2023-06-10T21:36:04.290808+0000","last_unstale":"2023-06-10T21:36:54.498687+0000","last_undegraded":"2023-06-10T21:36:54.498687+0000","last_fullsized":"2023-06-10T21:36:54.498687+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3430,"log_dups_size":0,"ondisk_log_size":3430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:13:41.537032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224788480,"num_objects":3430,"num_object_clones":0,"num_object_copies":10290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3430,"num_write_kb":219520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040607+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:36:46.040607+0000","last_peered":"2023-06-10T21:36:46.040607+0000","last_clean":"2023-06-10T21:36:46.040607+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:36:46.040607+0000","last_undegraded":"2023-06-10T21:36:46.040607+0000","last_fullsized":"2023-06-10T21:36:46.040607+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":324,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213810+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:36:35.213810+0000","last_peered":"2023-06-10T21:36:35.213810+0000","last_clean":"2023-06-10T21:36:35.213810+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:36:35.213810+0000","last_undegraded":"2023-06-10T21:36:35.213810+0000","last_fullsized":"2023-06-10T21:36:35.213810+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":165,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.039267+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:36:46.039267+0000","last_peered":"2023-06-10T21:36:46.039267+0000","last_clean":"2023-06-10T21:36:46.039267+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:36:46.039267+0000","last_undegraded":"2023-06-10T21:36:46.039267+0000","last_fullsized":"2023-06-10T21:36:46.039267+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"320'3439","reported_seq":3457,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.214895+0000","last_change":"2023-06-10T21:36:04.286731+0000","last_active":"2023-06-10T21:36:53.214895+0000","last_peered":"2023-06-10T21:36:53.214895+0000","last_clean":"2023-06-10T21:36:53.214895+0000","last_became_active":"2023-06-10T21:36:04.286427+0000","last_became_peered":"2023-06-10T21:36:04.286427+0000","last_unstale":"2023-06-10T21:36:53.214895+0000","last_undegraded":"2023-06-10T21:36:53.214895+0000","last_fullsized":"2023-06-10T21:36:53.214895+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3439,"log_dups_size":0,"ondisk_log_size":3439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:26:20.203365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225378304,"num_objects":3439,"num_object_clones":0,"num_object_copies":10317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3439,"num_write_kb":220096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":191,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.039309+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:36:46.039309+0000","last_peered":"2023-06-10T21:36:46.039309+0000","last_clean":"2023-06-10T21:36:46.039309+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:36:46.039309+0000","last_undegraded":"2023-06-10T21:36:46.039309+0000","last_fullsized":"2023-06-10T21:36:46.039309+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":265,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042063+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:36:46.042063+0000","last_peered":"2023-06-10T21:36:46.042063+0000","last_clean":"2023-06-10T21:36:46.042063+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:36:46.042063+0000","last_undegraded":"2023-06-10T21:36:46.042063+0000","last_fullsized":"2023-06-10T21:36:46.042063+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":154,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.206769+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:36:35.206769+0000","last_peered":"2023-06-10T21:36:35.206769+0000","last_clean":"2023-06-10T21:36:35.206769+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:36:35.206769+0000","last_undegraded":"2023-06-10T21:36:35.206769+0000","last_fullsized":"2023-06-10T21:36:35.206769+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"320'3380","reported_seq":3398,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.426721+0000","last_change":"2023-06-10T21:36:04.325488+0000","last_active":"2023-06-10T21:36:54.426721+0000","last_peered":"2023-06-10T21:36:54.426721+0000","last_clean":"2023-06-10T21:36:54.426721+0000","last_became_active":"2023-06-10T21:36:04.325162+0000","last_became_peered":"2023-06-10T21:36:04.325162+0000","last_unstale":"2023-06-10T21:36:54.426721+0000","last_undegraded":"2023-06-10T21:36:54.426721+0000","last_fullsized":"2023-06-10T21:36:54.426721+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3380,"log_dups_size":0,"ondisk_log_size":3380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:41:33.200826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221511680,"num_objects":3380,"num_object_clones":0,"num_object_copies":10140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3380,"num_write_kb":216320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":165,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.204791+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:36:35.204791+0000","last_peered":"2023-06-10T21:36:35.204791+0000","last_clean":"2023-06-10T21:36:35.204791+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:36:35.204791+0000","last_undegraded":"2023-06-10T21:36:35.204791+0000","last_fullsized":"2023-06-10T21:36:35.204791+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":163,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.205952+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:36:35.205952+0000","last_peered":"2023-06-10T21:36:35.205952+0000","last_clean":"2023-06-10T21:36:35.205952+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:36:35.205952+0000","last_undegraded":"2023-06-10T21:36:35.205952+0000","last_fullsized":"2023-06-10T21:36:35.205952+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207925+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:36:35.207925+0000","last_peered":"2023-06-10T21:36:35.207925+0000","last_clean":"2023-06-10T21:36:35.207925+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:36:35.207925+0000","last_undegraded":"2023-06-10T21:36:35.207925+0000","last_fullsized":"2023-06-10T21:36:35.207925+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"320'3316","reported_seq":3333,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.921661+0000","last_change":"2023-06-10T21:36:04.289917+0000","last_active":"2023-06-10T21:36:53.921661+0000","last_peered":"2023-06-10T21:36:53.921661+0000","last_clean":"2023-06-10T21:36:53.921661+0000","last_became_active":"2023-06-10T21:36:04.289698+0000","last_became_peered":"2023-06-10T21:36:04.289698+0000","last_unstale":"2023-06-10T21:36:53.921661+0000","last_undegraded":"2023-06-10T21:36:53.921661+0000","last_fullsized":"2023-06-10T21:36:53.921661+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3316,"log_dups_size":0,"ondisk_log_size":3316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217317376,"num_objects":3316,"num_object_clones":0,"num_object_copies":9948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3316,"num_write_kb":212224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":345,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.205346+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:36:35.205346+0000","last_peered":"2023-06-10T21:36:35.205346+0000","last_clean":"2023-06-10T21:36:35.205346+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:36:35.205346+0000","last_undegraded":"2023-06-10T21:36:35.205346+0000","last_fullsized":"2023-06-10T21:36:35.205346+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":483,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.201892+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:36:35.201892+0000","last_peered":"2023-06-10T21:36:35.201892+0000","last_clean":"2023-06-10T21:36:35.201892+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:36:35.201892+0000","last_undegraded":"2023-06-10T21:36:35.201892+0000","last_fullsized":"2023-06-10T21:36:35.201892+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.214206+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:36:35.214206+0000","last_peered":"2023-06-10T21:36:35.214206+0000","last_clean":"2023-06-10T21:36:35.214206+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:36:35.214206+0000","last_undegraded":"2023-06-10T21:36:35.214206+0000","last_fullsized":"2023-06-10T21:36:35.214206+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202536+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:36:35.202536+0000","last_peered":"2023-06-10T21:36:35.202536+0000","last_clean":"2023-06-10T21:36:35.202536+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:36:35.202536+0000","last_undegraded":"2023-06-10T21:36:35.202536+0000","last_fullsized":"2023-06-10T21:36:35.202536+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"320'3320","reported_seq":3338,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.222657+0000","last_change":"2023-06-10T21:36:04.325644+0000","last_active":"2023-06-10T21:36:53.222657+0000","last_peered":"2023-06-10T21:36:53.222657+0000","last_clean":"2023-06-10T21:36:53.222657+0000","last_became_active":"2023-06-10T21:36:04.325301+0000","last_became_peered":"2023-06-10T21:36:04.325301+0000","last_unstale":"2023-06-10T21:36:53.222657+0000","last_undegraded":"2023-06-10T21:36:53.222657+0000","last_fullsized":"2023-06-10T21:36:53.222657+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3320,"log_dups_size":0,"ondisk_log_size":3320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:28:03.018219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217579520,"num_objects":3320,"num_object_clones":0,"num_object_copies":9960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3320,"num_write_kb":212480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":339,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.201992+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:36:35.201992+0000","last_peered":"2023-06-10T21:36:35.201992+0000","last_clean":"2023-06-10T21:36:35.201992+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:36:35.201992+0000","last_undegraded":"2023-06-10T21:36:35.201992+0000","last_fullsized":"2023-06-10T21:36:35.201992+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":160,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.039085+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:36:46.039085+0000","last_peered":"2023-06-10T21:36:46.039085+0000","last_clean":"2023-06-10T21:36:46.039085+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:36:46.039085+0000","last_undegraded":"2023-06-10T21:36:46.039085+0000","last_fullsized":"2023-06-10T21:36:46.039085+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":144,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.208188+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:36:35.208188+0000","last_peered":"2023-06-10T21:36:35.208188+0000","last_clean":"2023-06-10T21:36:35.208188+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:36:35.208188+0000","last_undegraded":"2023-06-10T21:36:35.208188+0000","last_fullsized":"2023-06-10T21:36:35.208188+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"320'3520","reported_seq":3537,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.697699+0000","last_change":"2023-06-10T21:36:04.325466+0000","last_active":"2023-06-10T21:36:56.697699+0000","last_peered":"2023-06-10T21:36:56.697699+0000","last_clean":"2023-06-10T21:36:56.697699+0000","last_became_active":"2023-06-10T21:36:04.325178+0000","last_became_peered":"2023-06-10T21:36:04.325178+0000","last_unstale":"2023-06-10T21:36:56.697699+0000","last_undegraded":"2023-06-10T21:36:56.697699+0000","last_fullsized":"2023-06-10T21:36:56.697699+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3520,"log_dups_size":0,"ondisk_log_size":3520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:03.551659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230686720,"num_objects":3520,"num_object_clones":0,"num_object_copies":10560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3520,"num_write_kb":225280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":324,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202445+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:36:35.202445+0000","last_peered":"2023-06-10T21:36:35.202445+0000","last_clean":"2023-06-10T21:36:35.202445+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:36:35.202445+0000","last_undegraded":"2023-06-10T21:36:35.202445+0000","last_fullsized":"2023-06-10T21:36:35.202445+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":163,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.206836+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:36:35.206836+0000","last_peered":"2023-06-10T21:36:35.206836+0000","last_clean":"2023-06-10T21:36:35.206836+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:36:35.206836+0000","last_undegraded":"2023-06-10T21:36:35.206836+0000","last_fullsized":"2023-06-10T21:36:35.206836+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":141,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042135+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:36:46.042135+0000","last_peered":"2023-06-10T21:36:46.042135+0000","last_clean":"2023-06-10T21:36:46.042135+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:36:46.042135+0000","last_undegraded":"2023-06-10T21:36:46.042135+0000","last_fullsized":"2023-06-10T21:36:46.042135+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"320'3534","reported_seq":3552,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.702563+0000","last_change":"2023-06-10T21:36:04.292180+0000","last_active":"2023-06-10T21:36:56.702563+0000","last_peered":"2023-06-10T21:36:56.702563+0000","last_clean":"2023-06-10T21:36:56.702563+0000","last_became_active":"2023-06-10T21:36:04.291901+0000","last_became_peered":"2023-06-10T21:36:04.291901+0000","last_unstale":"2023-06-10T21:36:56.702563+0000","last_undegraded":"2023-06-10T21:36:56.702563+0000","last_fullsized":"2023-06-10T21:36:56.702563+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3534,"log_dups_size":0,"ondisk_log_size":3534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:45.649615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3534,"num_write_kb":226176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":298,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213118+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:36:35.213118+0000","last_peered":"2023-06-10T21:36:35.213118+0000","last_clean":"2023-06-10T21:36:35.213118+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:36:35.213118+0000","last_undegraded":"2023-06-10T21:36:35.213118+0000","last_fullsized":"2023-06-10T21:36:35.213118+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":95,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213152+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:36:35.213152+0000","last_peered":"2023-06-10T21:36:35.213152+0000","last_clean":"2023-06-10T21:36:35.213152+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:36:35.213152+0000","last_undegraded":"2023-06-10T21:36:35.213152+0000","last_fullsized":"2023-06-10T21:36:35.213152+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":280,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042193+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:36:46.042193+0000","last_peered":"2023-06-10T21:36:46.042193+0000","last_clean":"2023-06-10T21:36:46.042193+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:36:46.042193+0000","last_undegraded":"2023-06-10T21:36:46.042193+0000","last_fullsized":"2023-06-10T21:36:46.042193+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"320'3448","reported_seq":3466,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.452934+0000","last_change":"2023-06-10T21:36:04.296839+0000","last_active":"2023-06-10T21:36:54.452934+0000","last_peered":"2023-06-10T21:36:54.452934+0000","last_clean":"2023-06-10T21:36:54.452934+0000","last_became_active":"2023-06-10T21:36:04.294647+0000","last_became_peered":"2023-06-10T21:36:04.294647+0000","last_unstale":"2023-06-10T21:36:54.452934+0000","last_undegraded":"2023-06-10T21:36:54.452934+0000","last_fullsized":"2023-06-10T21:36:54.452934+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":3448,"log_dups_size":0,"ondisk_log_size":3448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:16.069639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225968128,"num_objects":3448,"num_object_clones":0,"num_object_copies":10344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3448,"num_write_kb":220672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":41,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202726+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:36:35.202726+0000","last_peered":"2023-06-10T21:36:35.202726+0000","last_clean":"2023-06-10T21:36:35.202726+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:36:35.202726+0000","last_undegraded":"2023-06-10T21:36:35.202726+0000","last_fullsized":"2023-06-10T21:36:35.202726+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":148,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207951+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:36:35.207951+0000","last_peered":"2023-06-10T21:36:35.207951+0000","last_clean":"2023-06-10T21:36:35.207951+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:36:35.207951+0000","last_undegraded":"2023-06-10T21:36:35.207951+0000","last_fullsized":"2023-06-10T21:36:35.207951+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":277,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.038892+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:36:46.038892+0000","last_peered":"2023-06-10T21:36:46.038892+0000","last_clean":"2023-06-10T21:36:46.038892+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:36:46.038892+0000","last_undegraded":"2023-06-10T21:36:46.038892+0000","last_fullsized":"2023-06-10T21:36:46.038892+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":144,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.207646+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:36:35.207646+0000","last_peered":"2023-06-10T21:36:35.207646+0000","last_clean":"2023-06-10T21:36:35.207646+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:36:35.207646+0000","last_undegraded":"2023-06-10T21:36:35.207646+0000","last_fullsized":"2023-06-10T21:36:35.207646+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.206889+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:36:35.206889+0000","last_peered":"2023-06-10T21:36:35.206889+0000","last_clean":"2023-06-10T21:36:35.206889+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:36:35.206889+0000","last_undegraded":"2023-06-10T21:36:35.206889+0000","last_fullsized":"2023-06-10T21:36:35.206889+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":45,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.202369+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:36:35.202369+0000","last_peered":"2023-06-10T21:36:35.202369+0000","last_clean":"2023-06-10T21:36:35.202369+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:36:35.202369+0000","last_undegraded":"2023-06-10T21:36:35.202369+0000","last_fullsized":"2023-06-10T21:36:35.202369+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.212999+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:36:35.212999+0000","last_peered":"2023-06-10T21:36:35.212999+0000","last_clean":"2023-06-10T21:36:35.212999+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:36:35.212999+0000","last_undegraded":"2023-06-10T21:36:35.212999+0000","last_fullsized":"2023-06-10T21:36:35.212999+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":99,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213576+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:36:35.213576+0000","last_peered":"2023-06-10T21:36:35.213576+0000","last_clean":"2023-06-10T21:36:35.213576+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:36:35.213576+0000","last_undegraded":"2023-06-10T21:36:35.213576+0000","last_fullsized":"2023-06-10T21:36:35.213576+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":259,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.214184+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:36:35.214184+0000","last_peered":"2023-06-10T21:36:35.214184+0000","last_clean":"2023-06-10T21:36:35.214184+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:36:35.214184+0000","last_undegraded":"2023-06-10T21:36:35.214184+0000","last_fullsized":"2023-06-10T21:36:35.214184+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":305,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041574+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:36:46.041574+0000","last_peered":"2023-06-10T21:36:46.041574+0000","last_clean":"2023-06-10T21:36:46.041574+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:36:46.041574+0000","last_undegraded":"2023-06-10T21:36:46.041574+0000","last_fullsized":"2023-06-10T21:36:46.041574+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":308,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.213400+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:36:35.213400+0000","last_peered":"2023-06-10T21:36:35.213400+0000","last_clean":"2023-06-10T21:36:35.213400+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:36:35.213400+0000","last_undegraded":"2023-06-10T21:36:35.213400+0000","last_fullsized":"2023-06-10T21:36:35.213400+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":210,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040747+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:36:46.040747+0000","last_peered":"2023-06-10T21:36:46.040747+0000","last_clean":"2023-06-10T21:36:46.040747+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:36:46.040747+0000","last_undegraded":"2023-06-10T21:36:46.040747+0000","last_fullsized":"2023-06-10T21:36:46.040747+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-06-10T21:36:35.205741+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:36:35.205741+0000","last_peered":"2023-06-10T21:36:35.205741+0000","last_clean":"2023-06-10T21:36:35.205741+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:36:35.205741+0000","last_undegraded":"2023-06-10T21:36:35.205741+0000","last_fullsized":"2023-06-10T21:36:35.205741+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3582590976,"num_objects":54666,"num_object_clones":0,"num_object_copies":163998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":54666,"num_write_kb":3498624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10762780672,"data_stored":10762780672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":54666,"ondisk_log_size":54666,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430678,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2973364,"kb_used_data":2337196,"kb_used_omap":0,"kb_used_meta":636160,"kb_avail":90771788,"statfs":{"total":95995035648,"available":92950310912,"internally_reserved":0,"allocated":2393288704,"data_stored":2392713687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":651427840},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":2.2050000000000001,"15min":6.8360000000000003},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2629999999999999,"5min":52.100000000000001,"15min":184.83699999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.601,"15min":7.1120000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.223},"max":{"1min":6.7999999999999998,"5min":51.146999999999998,"15min":185.58000000000001},"last":0.254}]},{"osd":1,"last update":"Sat Jun 10 21:36:15 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":3.681,"15min":3.4830000000000001},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":54.43,"5min":87.417000000000002,"15min":130.19399999999999},"last":1.857},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.1029999999999998,"15min":4.0469999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":54.564,"5min":87.438000000000002,"15min":212.20599999999999},"last":1.8069999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":5.5179999999999998,"15min":4.6100000000000003},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.3170000000000002,"5min":172.33600000000001,"15min":172.33600000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.7720000000000002,"15min":5.3330000000000002},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":3.6949999999999998,"5min":172.185,"15min":212.13399999999999},"last":1.9119999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:14 2023","interfaces":[{"interface":"back","average":{"1min":11.154,"5min":6.0650000000000004,"15min":3.649},"min":{"1min":0.38100000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.00700000000001,"5min":147.00700000000001,"15min":147.00700000000001},"last":2.1800000000000002},{"interface":"front","average":{"1min":11.568,"5min":5.8860000000000001,"15min":3.165},"min":{"1min":0.41399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":146.93899999999999,"5min":146.93899999999999,"15min":146.93899999999999},"last":2.246}]},{"osd":4,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39000000000000001,"15min":0.70799999999999996},"min":{"1min":0.27700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.59499999999999997,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.311},{"interface":"front","average":{"1min":0.436,"5min":0.438,"15min":1.0209999999999999},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.66600000000000004,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46600000000000003,"15min":1.8120000000000001},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.68700000000000006,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.45400000000000001,"15min":1.607},"min":{"1min":0.34100000000000003,"5min":0.32500000000000001,"15min":0.29299999999999998},"max":{"1min":0.68600000000000005,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.50900000000000001,"15min":1.069},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.27800000000000002},"max":{"1min":0.70899999999999996,"5min":3.2559999999999998,"15min":13.073},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.48499999999999999,"15min":0.92900000000000005},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":1.1879999999999999,"15min":10.874000000000001},"last":0.623}]}]},{"osd":6,"up_from":0,"seq":180388626906,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566349,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2492108,"kb_used_data":1786820,"kb_used_omap":4,"kb_used_meta":705275,"kb_avail":91253044,"statfs":{"total":95995035648,"available":93443117056,"internally_reserved":0,"allocated":1829703680,"data_stored":1828697180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":722202510},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":3.3279999999999998,"15min":9.4480000000000004},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.8980000000000001,"5min":70.254000000000005,"15min":154.31399999999999},"last":1.458},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.028,"15min":9.1950000000000003},"min":{"1min":0.28499999999999998,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":2.9319999999999999,"5min":70.100999999999999,"15min":154.27199999999999},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:36:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":7.9550000000000001,"15min":5.7329999999999997},"min":{"1min":0.25900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":5.4550000000000001,"5min":191.101,"15min":191.101},"last":1.381},{"interface":"front","average":{"1min":0.81200000000000006,"5min":6.532,"15min":5.0650000000000004},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.222},"max":{"1min":2.5590000000000002,"5min":190.87899999999999,"15min":190.87899999999999},"last":2.9489999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":7.6280000000000001,"5min":5.734,"15min":3.2719999999999998},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":83.810000000000002,"5min":83.810000000000002,"15min":83.810000000000002},"last":2.9209999999999998},{"interface":"front","average":{"1min":7.7050000000000001,"5min":5.8680000000000003,"15min":4.2050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":92.757000000000005,"5min":92.757000000000005,"15min":92.757000000000005},"last":3.1699999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.24,"15min":1.5920000000000001},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":7.609,"5min":9.3659999999999997,"15min":16.059999999999999},"last":2.8010000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.5289999999999999,"15min":1.48},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":5.5389999999999997,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.1099999999999999,"15min":4.3200000000000003},"min":{"1min":0.434,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.959,"5min":11.959,"15min":383.83100000000002},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.2509999999999999,"15min":3.847},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":7.6429999999999998,"5min":9.3160000000000007,"15min":383.84800000000001},"last":2.8500000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.1469999999999998,"15min":2.226},"min":{"1min":0.40699999999999997,"5min":0.316,"15min":0.316},"max":{"1min":11.923,"5min":11.923,"15min":14.858000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.411,"5min":1.4019999999999999,"15min":1.772},"min":{"1min":0.45500000000000002,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":7.5700000000000003,"5min":9.3849999999999998,"15min":13.48},"last":2.8849999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.9470000000000001,"15min":2.7090000000000001},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":7.4749999999999996,"5min":47.844000000000001,"15min":162.399},"last":2.9780000000000002},{"interface":"front","average":{"1min":1.115,"5min":2.3490000000000002,"15min":2.5059999999999998},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":5.5650000000000004,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.77700000000000002}]}]},{"osd":0,"up_from":244,"seq":1047972020325,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580176,"kb_used_data":1276040,"kb_used_omap":10,"kb_used_meta":304117,"kb_avail":92164976,"statfs":{"total":95995035648,"available":94376935424,"internally_reserved":0,"allocated":1306664960,"data_stored":1305593308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":311415860},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":4.0869999999999997,"15min":4.391},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.191},"max":{"1min":4.6500000000000004,"5min":139.68000000000001,"15min":139.68000000000001},"last":7.6299999999999999},{"interface":"front","average":{"1min":4.5839999999999996,"5min":4.5140000000000002,"15min":5.4619999999999997},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":15.403,"5min":141.79499999999999,"15min":141.79499999999999},"last":5.3949999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":4.2569999999999997,"5min":7.1600000000000001,"15min":6.7930000000000001},"min":{"1min":0.31,"5min":0.216,"15min":0.216},"max":{"1min":24.166,"5min":132.245,"15min":205.10900000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":4.0499999999999998,"5min":6.7140000000000004,"15min":6.2770000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":13.06,"5min":132.26599999999999,"15min":208.34299999999999},"last":9.3170000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:36:12 2023","interfaces":[{"interface":"back","average":{"1min":15.250999999999999,"5min":6.3860000000000001,"15min":6.1669999999999998},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":211.864,"5min":211.864,"15min":211.864},"last":0.70499999999999996},{"interface":"front","average":{"1min":14.010999999999999,"5min":6.7969999999999997,"15min":6.9569999999999999},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":207.517,"5min":207.517,"15min":207.517},"last":7.7030000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8039999999999998,"5min":1.9710000000000001,"15min":3.9460000000000002},"min":{"1min":0.34699999999999998,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":8.9550000000000001,"5min":10.744999999999999,"15min":29.709},"last":4.8710000000000004},{"interface":"front","average":{"1min":3.4049999999999998,"5min":2.2440000000000002,"15min":3.5110000000000001},"min":{"1min":0.42999999999999999,"5min":0.308,"15min":0.308},"max":{"1min":24.222999999999999,"5min":24.222999999999999,"15min":24.222999999999999},"last":0.78900000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4169999999999998,"5min":2.2639999999999998,"15min":3.5859999999999999},"min":{"1min":0.438,"5min":0.34300000000000003,"15min":0.34100000000000003},"max":{"1min":24.288,"5min":24.288,"15min":24.288},"last":0.76400000000000001},{"interface":"front","average":{"1min":2.3029999999999999,"5min":1.8169999999999999,"15min":3.0230000000000001},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":6.9870000000000001,"5min":8.0079999999999991,"15min":19.846},"last":7.7560000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8279999999999998,"5min":2.056,"15min":4.1100000000000003},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":9,"5min":10.771000000000001,"15min":29.806000000000001},"last":4.9199999999999999},{"interface":"front","average":{"1min":3.4430000000000001,"5min":2.3839999999999999,"15min":3.6509999999999998},"min":{"1min":0.40899999999999997,"5min":0.373,"15min":0.373},"max":{"1min":24.260999999999999,"5min":24.260999999999999,"15min":24.260999999999999},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:35:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.0449999999999999,"15min":3.3839999999999999},"min":{"1min":0.48199999999999998,"5min":0.38100000000000001,"15min":0.32400000000000001},"max":{"1min":7.0149999999999997,"5min":17.523,"15min":19.866},"last":7.8369999999999997},{"interface":"front","average":{"1min":2.843,"5min":2.2120000000000002,"15min":4.1609999999999996},"min":{"1min":0.44800000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0259999999999998,"5min":17.792999999999999,"15min":29.77},"last":4.9550000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075737,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2729092,"kb_used_data":2169980,"kb_used_omap":7,"kb_used_meta":559096,"kb_avail":91016060,"statfs":{"total":95995035648,"available":93200445440,"internally_reserved":0,"allocated":2222059520,"data_stored":2221060242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":572514385},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:47 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":3.819,"15min":4.4050000000000002},"min":{"1min":0.24199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.85599999999999998,"5min":70.510999999999996,"15min":70.510999999999996},"last":0.19},{"interface":"front","average":{"1min":0.67700000000000005,"5min":3.6520000000000001,"15min":4.8330000000000002},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":3.2490000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":8.2680000000000007}]},{"osd":1,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":5.1100000000000003,"15min":3.3239999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19},"max":{"1min":7.4050000000000002,"5min":147.20099999999999,"15min":147.20099999999999},"last":2.7690000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":4.7240000000000002,"15min":2.988},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.214},"max":{"1min":2.3279999999999998,"5min":147.81299999999999,"15min":147.81299999999999},"last":0.245}]},{"osd":3,"last update":"Sat Jun 10 21:36:11 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":6.0970000000000004,"15min":5.3760000000000003},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.15,"5min":138.03299999999999,"15min":138.03299999999999},"last":0.38},{"interface":"front","average":{"1min":3.7360000000000002,"5min":6.3019999999999996,"15min":6.0659999999999998},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":17.553999999999998,"5min":138.095,"15min":138.095},"last":5.3799999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.7210000000000001,"15min":2.996},"min":{"1min":0.35899999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":5.9470000000000001,"5min":7.3929999999999998,"15min":224.00999999999999},"last":5.4969999999999999},{"interface":"front","average":{"1min":1.115,"5min":2.3839999999999999,"15min":3.3599999999999999},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":5.7869999999999999,"5min":29.486999999999998,"15min":223.28399999999999},"last":2.7029999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.6830000000000001,"15min":1.651},"min":{"1min":0.39300000000000002,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":0.98499999999999999,"5min":10.554,"15min":14.180999999999999},"last":0.437},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.7,"15min":1.8979999999999999},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":1.8580000000000001,"5min":7.3200000000000003,"15min":16.125},"last":5.4800000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.7769999999999999,"15min":1.966},"min":{"1min":0.43099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":1.8859999999999999,"5min":7.4180000000000001,"15min":16.183},"last":5.4560000000000004},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.4580000000000002,"15min":2.3620000000000001},"min":{"1min":0.40500000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.1819999999999999,"5min":29.538,"15min":29.538},"last":2.7410000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":16.013000000000002,"5min":6.5259999999999998,"15min":3.6400000000000001},"min":{"1min":0.48999999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":219.131,"5min":219.131,"15min":219.131},"last":0.41199999999999998},{"interface":"front","average":{"1min":16.175999999999998,"5min":6.4640000000000004,"15min":3.9630000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":219.19,"5min":219.19,"15min":219.19},"last":5.4299999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824097,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449500,"kb_used_data":2149204,"kb_used_omap":7,"kb_used_meta":300280,"kb_avail":91295652,"statfs":{"total":95995035648,"available":93486747648,"internally_reserved":0,"allocated":2200784896,"data_stored":2199702988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":307486739},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":4.4569999999999999,"15min":4.9820000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.1890000000000001,"5min":105.09,"15min":105.09},"last":2.0019999999999998},{"interface":"front","average":{"1min":1.472,"5min":4.4160000000000004,"15min":4.7969999999999997},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":99.852999999999994,"15min":99.852999999999994},"last":2.915}]},{"osd":1,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":4.766,"15min":4.0380000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":10.419,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.20499999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":4.6639999999999997,"15min":3.7320000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.9629999999999992,"5min":94.691000000000003,"15min":94.691000000000003},"last":2.081}]},{"osd":2,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":3.7690000000000001,"15min":9.1449999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7480000000000002,"5min":100.676,"15min":100.676},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.2400000000000002,"15min":10.669},"min":{"1min":0.30399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.835000000000001,"5min":103.032,"15min":103.032},"last":0.63500000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.446,"15min":1.405},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":3.7770000000000001,"5min":13.417,"15min":13.417},"last":2.8799999999999999},{"interface":"front","average":{"1min":1.244,"5min":2.3780000000000001,"15min":2.6680000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.2030000000000003,"5min":14.384,"15min":19.728000000000002},"last":0.27400000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.5169999999999999,"15min":1.397},"min":{"1min":0.377,"5min":0.35699999999999998,"15min":0.32400000000000001},"max":{"1min":1.4079999999999999,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":1.8979999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.6060000000000001,"15min":1.4870000000000001},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.7999999999999998,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.3919999999999999,"15min":2.6899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.1740000000000004,"5min":14.269,"15min":19.805},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.552,"15min":1.4470000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.39100000000000001},"max":{"1min":1.4379999999999999,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":1.9399999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0760000000000001,"15min":2.9260000000000002},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.1449999999999996,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":2.2360000000000002,"15min":1.6579999999999999},"min":{"1min":0.44,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":1.6899999999999999,"5min":80.272999999999996,"15min":80.272999999999996},"last":1.9730000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019362,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1677164,"kb_used_data":1054372,"kb_used_omap":0,"kb_used_meta":622784,"kb_avail":92067988,"statfs":{"total":95995035648,"available":94277619712,"internally_reserved":0,"allocated":1079676928,"data_stored":1079108551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":637730816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:46 2023","interfaces":[{"interface":"back","average":{"1min":5.8570000000000002,"5min":5.1840000000000002,"15min":7.5780000000000003},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":115.831,"5min":234.203,"15min":234.203},"last":0.33900000000000002},{"interface":"front","average":{"1min":6.157,"5min":5.4509999999999996,"15min":7.5700000000000003},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":117.111,"5min":234.19,"15min":234.19},"last":0.5}]},{"osd":1,"last update":"Sat Jun 10 21:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":3.0179999999999998,"15min":3.48},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.125,"5min":88.700999999999993,"15min":182.035},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.67200000000000004,"5min":3.165,"15min":3.3849999999999998},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":3.641,"5min":87.853999999999999,"15min":181.73099999999999},"last":1.5529999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:36:34 2023","interfaces":[{"interface":"back","average":{"1min":5.7279999999999998,"5min":8.4760000000000009,"15min":4.1020000000000003},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":79.097999999999999,"5min":255.386,"15min":255.386},"last":0.61699999999999999},{"interface":"front","average":{"1min":5.593,"5min":8.3559999999999999,"15min":4.016},"min":{"1min":0.34100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":79.808999999999997,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3959999999999999,"5min":8.0939999999999994,"15min":4.2489999999999997},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.28399999999999997},"max":{"1min":36.366999999999997,"5min":234.91900000000001,"15min":234.91900000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.6619999999999999,"5min":8.4410000000000007,"15min":4.1900000000000004},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":38.659999999999997,"5min":235.05099999999999,"15min":235.05099999999999},"last":4.9420000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:36:34 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.38500000000000001,"15min":0.98399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.57799999999999996,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.46999999999999997,"15min":0.997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.247},"max":{"1min":0.95799999999999996,"5min":5.25,"15min":11.904999999999999},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42899999999999999,"15min":1.0149999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.67700000000000005,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46600000000000003,"15min":1.079},"min":{"1min":0.35499999999999998,"5min":0.34699999999999998,"15min":0.309},"max":{"1min":0.63400000000000001,"5min":1.71,"15min":13.845000000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.77900000000000003,"15min":1.2669999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":0.65300000000000002,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.79200000000000004,"15min":1.21},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.68500000000000005,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.60799999999999998}]}]},{"osd":5,"up_from":0,"seq":154618823134,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1318780928,"data_stored":1318780928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1605238784,"data_stored":1605238784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2235564032,"data_stored":2235564032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2084962304,"data_stored":2084962304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1098317824,"data_stored":1098317824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2419916800,"data_stored":2419916800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:37:02.352 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:02.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:02.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: osdmap e321: 8 total, 8 up, 6 in 2023-06-10T21:37:02.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: failed to encode map e321 with expected crc 2023-06-10T21:37:02.353 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:02 smithi064 ceph-mon[180297]: pgmap v1191: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-10T21:37:02.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:02.466 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:02.520 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:02.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:02.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:02.658 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:02.715 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:02.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:02.764 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 61475 61459 78.3425 60.5625 0.0190623 0.0127587 2023-06-10T21:37:02.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:02.832 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:02.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:02] "GET /metrics HTTP/1.1" 200 224120 "" "Prometheus/2.33.4" 2023-06-10T21:37:02.903 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:03.016 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:03.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:03.104 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:03.104 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:37:03.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:37:03.105 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:37:03.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.105 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.25280.0:49 2.2 2:4e99cc3e:::rbd_mirror_snapshot_schedule:head [omap-get-vals] snapc 0=[] RETRY=2 ondisk+retry+read+known_if_redirected+supports_pool_eio e244)", 2023-06-10T21:37:03.106 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:37:03.106 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.613244549, 2023-06-10T21:37:03.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0624303289999999, 2023-06-10T21:37:03.106 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.107 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:37:03.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.25280", 2023-06-10T21:37:03.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2947718283", 2023-06-10T21:37:03.108 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 49 2023-06-10T21:37:03.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.108 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:37:03.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479550+0000", 2023-06-10T21:37:03.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479548+0000", 2023-06-10T21:37:03.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479556+0000", 2023-06-10T21:37:03.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3620000000000006e-06 2023-06-10T21:37:03.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479566+0000", 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5880000000000004e-06 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.479573+0000", 2023-06-10T21:37:03.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.046e-06 2023-06-10T21:37:03.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487169+0000", 2023-06-10T21:37:03.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075960059999999998 2023-06-10T21:37:03.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.487178+0000", 2023-06-10T21:37:03.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7660000000000004e-06 2023-06-10T21:37:03.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541881+0000", 2023-06-10T21:37:03.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.054703094 2023-06-10T21:37:03.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541980+0000", 2023-06-10T21:37:03.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8992000000000004e-05 2023-06-10T21:37:03.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.121 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.121 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.123 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111488 10.4 10:2ebf3782:::benchmark_data_smithi064_312713_object20732:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e244)", 2023-06-10T21:37:03.123 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:37:03.123 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.61065642699998, 2023-06-10T21:37:03.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058806785, 2023-06-10T21:37:03.124 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.125 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.125 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.126 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.126 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.126 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111488 2023-06-10T21:37:03.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.127 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:37:03.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482138+0000", 2023-06-10T21:37:03.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482137+0000", 2023-06-10T21:37:03.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:03.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482142+0000", 2023-06-10T21:37:03.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4210000000000003e-06 2023-06-10T21:37:03.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482152+0000", 2023-06-10T21:37:03.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:37:03.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482162+0000", 2023-06-10T21:37:03.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0344e-05 2023-06-10T21:37:03.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482209+0000", 2023-06-10T21:37:03.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7237000000000002e-05 2023-06-10T21:37:03.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482218+0000", 2023-06-10T21:37:03.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2040000000000003e-06 2023-06-10T21:37:03.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525337+0000", 2023-06-10T21:37:03.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.043118188 2023-06-10T21:37:03.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525473+0000", 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136673 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:37:03.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.525610+0000", 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000136231 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532648+0000", 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070388239999999999 2023-06-10T21:37:03.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540822+0000", 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008173441 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540860+0000", 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7815999999999998e-05 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.540944+0000", 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4839999999999994e-05 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111524 10.4 10:203e818a:::benchmark_data_smithi064_312713_object20768:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.610619058, 2023-06-10T21:37:03.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0589736080000001, 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111524 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.144 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482175+0000", 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482178+0000", 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2090000000000001e-06 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482184+0000", 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6080000000000001e-06 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482192+0000", 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:37:03.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5090000000000003e-05 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6470000000000004e-06 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532673+0000", 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0504169059999999 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532795+0000", 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122521 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:37:03.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.532918+0000", 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012313 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538256+0000", 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053373800000000001 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541023+0000", 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002766967 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541067+0000", 2023-06-10T21:37:03.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4270000000000001e-05 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541149+0000", 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1736000000000005e-05 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.154 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111526 10.4 10:21f5efce:::benchmark_data_smithi064_312713_object20770:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.61059579699997, 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0590798290000001, 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.155 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111526 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:37:03.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482198+0000", 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482201+0000", 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6919999999999998e-06 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482207+0000", 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1460000000000001e-06 2023-06-10T21:37:03.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482221+0000", 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3876999999999999e-05 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7775999999999997e-05 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482285+0000", 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1879999999999999e-06 2023-06-10T21:37:03.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535227+0000", 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0529423769999999 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535306+0000", 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8954000000000003e-05 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.535404+0000", 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8250000000000003e-05 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538295+0000", 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028906679999999999 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541164+0000", 2023-06-10T21:37:03.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028689599999999998 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541202+0000", 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8213999999999997e-05 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541278+0000", 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6106999999999995e-05 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.166 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111533 10.4 10:20aecf0c:::benchmark_data_smithi064_312713_object20777:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.61054962999998, 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0641654469999999, 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.167 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111533 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:37:03.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482244+0000", 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482247+0000", 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.315e-06 2023-06-10T21:37:03.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482256+0000", 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9400000000000008e-06 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482263+0000", 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.063e-06 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482314+0000", 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1063999999999998e-05 2023-06-10T21:37:03.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482321+0000", 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8589999999999999e-06 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538345+0000", 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.056023894 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538495+0000", 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149498 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.538638+0000", 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014375100000000001 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541316+0000", 2023-06-10T21:37:03.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026775509999999998 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546267+0000", 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004951324 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546317+0000", 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9876999999999997e-05 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.546410+0000", 2023-06-10T21:37:03.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2786000000000002e-05 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111555 10.4 10:24e69e32:::benchmark_data_smithi064_312713_object20799:head [delete] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:37:03.178 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.61051641699999, 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0654074570000001, 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111555 2023-06-10T21:37:03.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482278+0000", 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482277+0000", 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482280+0000", 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5069999999999999e-06 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482286+0000", 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4180000000000002e-06 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.482292+0000", 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3399999999999997e-06 2023-06-10T21:37:03.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483443+0000", 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151383 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:33.483452+0000", 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1309999999999995e-06 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541485+0000", 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058032919 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541643+0000", 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158212 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 4", 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.541783+0000", 2023-06-10T21:37:03.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140111 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.544945+0000", 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031613840000000002 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547575+0000", 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026302230000000001 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547622+0000", 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7429999999999998e-05 2023-06-10T21:37:03.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.547685+0000", 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2838999999999999e-05 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111621 10.a 10:56721995:::benchmark_data_smithi064_312713_object20865:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.535517648, 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96906832399999998, 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.190 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111621 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557276+0000", 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557279+0000", 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.503e-06 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557289+0000", 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0631999999999999e-05 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557294+0000", 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7770000000000002e-06 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557348+0000", 2023-06-10T21:37:03.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3779999999999997e-05 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.557364+0000", 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6325e-05 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.516904+0000", 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95953967500000004 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517083+0000", 2023-06-10T21:37:03.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017900899999999999 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.517231+0000", 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014771599999999999 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526113+0000", 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0088829010000000003 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526152+0000", 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8665999999999999e-05 2023-06-10T21:37:03.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526194+0000", 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2048000000000001e-05 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526241+0000", 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7145999999999999e-05 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526345+0000", 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103491 2023-06-10T21:37:03.201 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111622 10.a 10:5c194bad:::benchmark_data_smithi064_312713_object20866:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.532654713, 2023-06-10T21:37:03.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97161865400000003, 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111622 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.203 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560139+0000", 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560138+0000", 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560142+0000", 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4659999999999999e-06 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560149+0000", 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0509999999999999e-06 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560155+0000", 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8510000000000001e-06 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560208+0000", 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2911000000000002e-05 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.560219+0000", 2023-06-10T21:37:03.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0859999999999999e-05 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520194+0000", 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95997470299999998 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520350+0000", 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015635700000000001 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.520521+0000", 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017028799999999999 2023-06-10T21:37:03.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526160+0000", 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056392309999999998 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531566+0000", 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054064539999999998 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531604+0000", 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7854e-05 2023-06-10T21:37:03.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531651+0000", 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7215000000000001e-05 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531758+0000", 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010678100000000001 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111629 10.a 10:56a08505:::benchmark_data_smithi064_312713_object20873:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.214 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.52527539499999, 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97554071600000003, 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.215 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111629 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567519+0000", 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567517+0000", 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567525+0000", 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.994e-06 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567537+0000", 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1922999999999999e-05 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567545+0000", 2023-06-10T21:37:03.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5900000000000002e-06 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567602+0000", 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7065999999999999e-05 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.567613+0000", 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523367+0000", 2023-06-10T21:37:03.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95575399000000005 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523517+0000", 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014977799999999999 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.523672+0000", 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015482899999999999 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.538459+0000", 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0147876 2023-06-10T21:37:03.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540585+0000", 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021253919999999998 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542925+0000", 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023401250000000002 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.542957+0000", 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2283999999999998e-05 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.543059+0000", 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102527 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111634 10.a 10:5fd06ea2:::benchmark_data_smithi064_312713_object20878:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.51607629300003, 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.969888847, 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.227 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111634 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576718+0000", 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576717+0000", 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576720+0000", 2023-06-10T21:37:03.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6340000000000001e-06 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576727+0000", 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.216e-06 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576733+0000", 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4319999999999998e-06 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576789+0000", 2023-06-10T21:37:03.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6227999999999998e-05 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.576800+0000", 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0572e-05 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526379+0000", 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94957964100000003 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526493+0000", 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113999 2023-06-10T21:37:03.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.526618+0000", 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012495299999999999 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540594+0000", 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013976018999999999 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546331+0000", 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005736568 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546401+0000", 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0585000000000003e-05 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546466+0000", 2023-06-10T21:37:03.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4929000000000001e-05 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546607+0000", 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014047500000000001 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111642 10.a 10:5fdb9826:::benchmark_data_smithi064_312713_object20886:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.50332566700001, 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96564841499999998, 2023-06-10T21:37:03.239 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111642 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589468+0000", 2023-06-10T21:37:03.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589467+0000", 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589472+0000", 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0490000000000003e-06 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589483+0000", 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0729e-05 2023-06-10T21:37:03.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589490+0000", 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6410000000000002e-06 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589573+0000", 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3007e-05 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.589585+0000", 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1836e-05 2023-06-10T21:37:03.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528718+0000", 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93913300899999996 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.528872+0000", 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154108 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.529036+0000", 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164017 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540601+0000", 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011564677000000001 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546679+0000", 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060784560000000003 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555011+0000", 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083312960000000002 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555038+0000", 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7032000000000001e-05 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555117+0000", 2023-06-10T21:37:03.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9177000000000004e-05 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111646 10.a 10:58b447f4:::benchmark_data_smithi064_312713_object20890:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:37:03.251 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.50004325600003, 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96249102799999997, 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111646 2023-06-10T21:37:03.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592751+0000", 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592750+0000", 2023-06-10T21:37:03.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592755+0000", 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6600000000000003e-06 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592764+0000", 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4059999999999994e-06 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592769+0000", 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2590000000000002e-06 2023-06-10T21:37:03.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592816+0000", 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7280000000000001e-05 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.592823+0000", 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1199999999999996e-06 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531787+0000", 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93896378800000002 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.531899+0000", 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111659 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.532018+0000", 2023-06-10T21:37:03.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119275 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540608+0000", 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085904399999999995 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546763+0000", 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061551050000000001 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555159+0000", 2023-06-10T21:37:03.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.008395563 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555184+0000", 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5528e-05 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555242+0000", 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7454000000000003e-05 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.263 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111648 10.a 10:574e8e24:::benchmark_data_smithi064_312713_object20892:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.49549763700003, 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.958220356, 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.264 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111648 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597296+0000", 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597295+0000", 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597300+0000", 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0309999999999998e-06 2023-06-10T21:37:03.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597309+0000", 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8089999999999999e-06 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597316+0000", 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6630000000000004e-06 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597417+0000", 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100995 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.597432+0000", 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5517000000000001e-05 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534105+0000", 2023-06-10T21:37:03.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93667285600000005 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534213+0000", 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010740599999999999 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.534345+0000", 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132714 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.540617+0000", 2023-06-10T21:37:03.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062720099999999997 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.552679+0000", 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012062154 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555339+0000", 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026591420000000002 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555401+0000", 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2757999999999998e-05 2023-06-10T21:37:03.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555517+0000", 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115498 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.35769.0:111681 10.a 10:560529bb:::benchmark_data_smithi064_312713_object20925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e245)", 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 508.44939667, 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91238480600000005, 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.276 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.35769", 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/854778858", 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111681 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643397+0000", 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643395+0000", 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643408+0000", 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2588e-05 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643431+0000", 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3011999999999999e-05 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643444+0000", 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3047000000000001e-05 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643726+0000", 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028253799999999998 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:34.643738+0000", 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1908e-05 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536216+0000", 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89247793399999997 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536328+0000", 2023-06-10T21:37:03.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112285 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.536430+0000", 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101981 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.546177+0000", 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097468780000000005 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555141+0000", 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089643450000000003 2023-06-10T21:37:03.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555638+0000", 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049625199999999998 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555676+0000", 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8074000000000002e-05 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:28:35.555782+0000", 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106461 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.287 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85152 11.5 11:a61fdc6c:::benchmark_data_smithi064_385725_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.58356237300001, 2023-06-10T21:37:03.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.711236145, 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85152 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.289 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509230+0000", 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509235+0000", 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4939999999999997e-06 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509247+0000", 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2203e-05 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509256+0000", 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0589999999999992e-06 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509326+0000", 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9915999999999999e-05 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509339+0000", 2023-06-10T21:37:03.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527999999999999e-05 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.195228+0000", 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6858897430000002 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.196887+0000", 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016585160000000001 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.197074+0000", 2023-06-10T21:37:03.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018745799999999999 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205769+0000", 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086945730000000006 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205810+0000", 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1607000000000003e-05 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217677+0000", 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011866642 2023-06-10T21:37:03.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217739+0000", 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2154000000000002e-05 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220468+0000", 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027286659999999998 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.300 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85191 11.5 11:ad36ddde:::benchmark_data_smithi064_385725_object918:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.58350780500001, 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7144825799999999, 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85191 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509285+0000", 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509290+0000", 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.203e-06 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509304+0000", 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3966e-05 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509314+0000", 2023-06-10T21:37:03.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5650000000000007e-06 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509379+0000", 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5468000000000004e-05 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509401+0000", 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1651e-05 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202573+0000", 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.693172154 2023-06-10T21:37:03.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202760+0000", 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018716300000000001 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202937+0000", 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017642100000000001 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217663+0000", 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014726461999999999 2023-06-10T21:37:03.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220485+0000", 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028217699999999999 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220695+0000", 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000209547 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220756+0000", 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0946999999999998e-05 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223769+0000", 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030133080000000001 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85250 11.5 11:ad3b70c3:::benchmark_data_smithi064_385725_object977:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.583473785, 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.718711994, 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.313 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85250 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509323+0000", 2023-06-10T21:37:03.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.83e-06 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509330+0000", 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.951e-06 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509336+0000", 2023-06-10T21:37:03.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1190000000000002e-06 2023-06-10T21:37:03.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.318 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:03.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509513+0000", 2023-06-10T21:37:03.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017721800000000001 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509522+0000", 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.969e-06 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205836+0000", 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6963141390000001 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206003+0000", 2023-06-10T21:37:03.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016704 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206123+0000", 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011994600000000001 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220498+0000", 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014375188000000001 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223827+0000", 2023-06-10T21:37:03.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033290490000000002 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223968+0000", 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014019100000000001 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.224024+0000", 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6718000000000001e-05 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228032+0000", 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040080999999999997 2023-06-10T21:37:03.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85263 11.5 11:a183d882:::benchmark_data_smithi064_385725_object990:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:37:03.326 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.58332678100001, 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7214953290000001, 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85263 2023-06-10T21:37:03.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:03.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509471+0000", 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6689999999999997e-06 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509482+0000", 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0746e-05 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509490+0000", 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4659999999999999e-06 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509846+0000", 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035595000000000001 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509855+0000", 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209100+0000", 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6992444710000001 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209286+0000", 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018623000000000001 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209449+0000", 2023-06-10T21:37:03.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162953 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220508+0000", 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011058678000000001 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223930+0000", 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003422671 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228127+0000", 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041968999999999999 2023-06-10T21:37:03.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228171+0000", 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3764999999999999e-05 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.230963+0000", 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002791689 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.338 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85264 11.5 11:a2a1b725:::benchmark_data_smithi064_385725_object991:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.58303129000001, 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.724233897, 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.339 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85264 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509762+0000", 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509767+0000", 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.135e-06 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509778+0000", 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058e-05 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509787+0000", 2023-06-10T21:37:03.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8769999999999999e-06 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509887+0000", 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010014900000000001 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509893+0000", 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214185+0000", 2023-06-10T21:37:03.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.704292031 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214314+0000", 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012892599999999999 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214446+0000", 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013181 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220634+0000", 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061883290000000002 2023-06-10T21:37:03.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228085+0000", 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074512379999999998 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231052+0000", 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029661269999999998 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231110+0000", 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8733000000000002e-05 2023-06-10T21:37:03.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.233997+0000", 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028864709999999998 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85269 11.5 11:a8f773e1:::benchmark_data_smithi064_385725_object996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 247.583001776, 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7386240339999999, 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:03.351 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85269 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:03.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:03.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509795+0000", 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0850000000000001e-06 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509802+0000", 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509810+0000", 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3529999999999995e-06 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510067+0000", 2023-06-10T21:37:03.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000256896 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510076+0000", 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0270000000000001e-06 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234066+0000", 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7239896319999999 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234235+0000", 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016923200000000001 2023-06-10T21:37:03.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234406+0000", 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017128799999999999 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:03.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240469+0000", 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006062217 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245113+0000", 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046439960000000001 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245151+0000", 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8617e-05 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:03.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245210+0000", 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9166999999999998e-05 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.248416+0000", 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032060209999999999 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.362 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:03.363 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:03.363 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:03.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:03.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:03.437 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:03.764 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 62436 62420 77.9768 60.0625 0.0105015 0.0128183 2023-06-10T21:37:04.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:37:04.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:04 smithi064 ceph-mon[180297]: pgmap v1192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:37:04.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:04 smithi064 ceph-mon[186706]: pgmap v1192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:37:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:04 smithi152 ceph-mon[156819]: pgmap v1192: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:37:04.765 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 63266 63250 77.4636 51.875 0.0180476 0.0129041 2023-06-10T21:37:04.786 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:04.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:04.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:04.921 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:05.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:37:05.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:37:05.358 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:05.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:05.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:05.589 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:05.759 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:05.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:05.765 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 63881 63865 76.7134 38.4375 0.0114069 0.0130274 2023-06-10T21:37:05.833 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:05.865 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:05.967 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:06.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:06.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:06 smithi064 ceph-mon[180297]: pgmap v1193: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s 2023-06-10T21:37:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:06.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:06.316 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:06 smithi064 ceph-mon[186706]: pgmap v1193: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s 2023-06-10T21:37:06.417 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:06 smithi152 ceph-mon[156819]: pgmap v1193: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 54 MiB/s wr, 867 op/s 2023-06-10T21:37:06.449 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:06.498 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:06.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:06.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:37:06.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:06.613 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:37:06.631 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:06.655 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:37:06.679 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:06.702 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:06.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:37:06.747 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:37:06.765 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 64704 64688 76.2367 51.4375 0.0120669 0.01311 2023-06-10T21:37:06.796 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:37:06.839 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:37:06.876 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:37:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:06.888 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.889 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:37:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:06.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:06.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:07.028 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:07.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:07.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:07.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:07.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/749318992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:07.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/749318992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:07.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:07.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/749318992' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:07.765 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 65493 65477 75.7383 49.3125 0.0164697 0.0131983 2023-06-10T21:37:08.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:08.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:08.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:08.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:08 smithi064 ceph-mon[186706]: pgmap v1194: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:37:08.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:08 smithi064 ceph-mon[180297]: pgmap v1194: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:37:08.413 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:08.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:08 smithi152 ceph-mon[156819]: pgmap v1194: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-10T21:37:08.765 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 66335 66319 75.3181 52.625 0.0184246 0.0132707 2023-06-10T21:37:09.221 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:09.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:09.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:09.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:09.365 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:09.421 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:09.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:09.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:09.555 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:09.604 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:09.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:09.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:09.707 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:09.767 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 67284 67268 75.03 59.3125 0.0235434 0.0133225 2023-06-10T21:37:09.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:09.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:09.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:10.075 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:10.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:10.148 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:10.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:10.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:10.288 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:10.338 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:10.345 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:10 smithi152 ceph-mon[156819]: pgmap v1195: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:37:10.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:10.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:10.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:10 smithi064 ceph-mon[186706]: pgmap v1195: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:37:10.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:10 smithi064 ceph-mon[180297]: pgmap v1195: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 810 op/s 2023-06-10T21:37:10.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:10.475 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:10.526 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:10.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:10.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:10.666 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:10.723 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:10.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:10.768 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 68054 68038 74.5578 48.125 0.00860499 0.013407 2023-06-10T21:37:10.830 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:10.858 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:10.906 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:10.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:11.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:11.038 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:11.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:11.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:11.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:11.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:37:11.260 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:11.527 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:11.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:11.636 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:11.723 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:11.770 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 68770 68754 74.0422 44.75 0.0269073 0.0135002 2023-06-10T21:37:12.163 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:12.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:12.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:12.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:12.372 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:11 smithi064 ceph-mon[186706]: pgmap v1196: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:37:12.372 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:11 smithi064 ceph-mon[180297]: pgmap v1196: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:37:12.379 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:11 smithi152 ceph-mon[156819]: pgmap v1196: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-10T21:37:12.539 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:12.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:12.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:12.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:12.673 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:12.722 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:12.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:12.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 69615 69599 73.6786 52.8125 0.0168604 0.0135669 2023-06-10T21:37:12.825 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:37:12.825 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:37:12.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:12.934 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:13.030 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:37:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:12] "GET /metrics HTTP/1.1" 200 224123 "" "Prometheus/2.33.4" 2023-06-10T21:37:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/630909068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/630909068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/630909068' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:37:13.621 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:13.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:13.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:13.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:37:13.771105+0000 min lat: 0.00116179 max lat: 0.288432 avg lat: 0.0136294 2023-06-10T21:37:13.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:37:13.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 70465 70449 73.3362 53.125 0.0355542 0.0136294 2023-06-10T21:37:13.794 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:37:13.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:13.805 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:37:14.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: osdmap e322: 8 total, 8 up, 6 in 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[186706]: pgmap v1198: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s 2023-06-10T21:37:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:37:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: osdmap e322: 8 total, 8 up, 6 in 2023-06-10T21:37:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:14 smithi064 ceph-mon[180297]: pgmap v1198: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s 2023-06-10T21:37:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:37:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: osdmap e322: 8 total, 8 up, 6 in 2023-06-10T21:37:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: failed to encode map e322 with expected crc 2023-06-10T21:37:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: from='client.36534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:37:14.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:14 smithi152 ceph-mon[156819]: pgmap v1198: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 800 op/s 2023-06-10T21:37:14.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 71456 71440 73.1493 61.9375 0.00689961 0.013664 2023-06-10T21:37:14.919 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:14.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:14.937 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:14.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:37:15.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:15.051 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:15.098 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:15.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:15.205 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:15.233 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:15 smithi064 ceph-mon[186706]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.234 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:15 smithi064 ceph-mon[186706]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:15 smithi064 ceph-mon[180297]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:15 smithi064 ceph-mon[180297]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.237 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:15.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:15 smithi152 ceph-mon[156819]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:15 smithi152 ceph-mon[156819]: failed to encode map e322 with expected crc 2023-06-10T21:37:15.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 72246 72230 72.766 49.375 0.0136601 0.0137384 2023-06-10T21:37:15.787 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1197,"stamp":"2023-06-10T21:37:13.010866+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4409268110,"num_objects":67295,"num_object_clones":0,"num_object_copies":201885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67295,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":67532,"num_write_kb":4307790,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":1777362,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67466,"ondisk_log_size":67466,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":269,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16369616,"kb_used_data":13167904,"kb_used_omap":30,"kb_used_meta":3201633,"kb_avail":546101296,"statfs":{"total":575970213888,"available":559207727104,"internally_reserved":0,"allocated":13483933696,"data_stored":13478625551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":3278472230},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":529465344,"num_objects":8079,"num_object_clones":0,"num_object_copies":24237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8079,"num_write_kb":517056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.063693"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":315,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.049443+0000","last_change":"2023-06-10T21:26:01.624889+0000","last_active":"2023-06-10T21:36:46.049443+0000","last_peered":"2023-06-10T21:36:46.049443+0000","last_clean":"2023-06-10T21:36:46.049443+0000","last_became_active":"2023-06-10T21:26:01.624561+0000","last_became_peered":"2023-06-10T21:26:01.624561+0000","last_unstale":"2023-06-10T21:36:46.049443+0000","last_undegraded":"2023-06-10T21:36:46.049443+0000","last_fullsized":"2023-06-10T21:36:46.049443+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:26:42.652109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":256,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041828+0000","last_change":"2023-06-10T21:28:57.894009+0000","last_active":"2023-06-10T21:36:46.041828+0000","last_peered":"2023-06-10T21:36:46.041828+0000","last_clean":"2023-06-10T21:36:46.041828+0000","last_became_active":"2023-06-10T21:28:57.892618+0000","last_became_peered":"2023-06-10T21:28:57.892618+0000","last_unstale":"2023-06-10T21:36:46.041828+0000","last_undegraded":"2023-06-10T21:36:46.041828+0000","last_fullsized":"2023-06-10T21:36:46.041828+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:25:35.950586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":231,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042416+0000","last_change":"2023-06-10T21:28:57.894506+0000","last_active":"2023-06-10T21:36:46.042416+0000","last_peered":"2023-06-10T21:36:46.042416+0000","last_clean":"2023-06-10T21:36:46.042416+0000","last_became_active":"2023-06-10T21:28:57.894044+0000","last_became_peered":"2023-06-10T21:28:57.894044+0000","last_unstale":"2023-06-10T21:36:46.042416+0000","last_undegraded":"2023-06-10T21:36:46.042416+0000","last_fullsized":"2023-06-10T21:36:46.042416+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:41:22.834285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040285+0000","last_change":"2023-06-10T21:32:55.522974+0000","last_active":"2023-06-10T21:36:46.040285+0000","last_peered":"2023-06-10T21:36:46.040285+0000","last_clean":"2023-06-10T21:36:46.040285+0000","last_became_active":"2023-06-10T21:32:55.522525+0000","last_became_peered":"2023-06-10T21:32:55.522525+0000","last_unstale":"2023-06-10T21:36:46.040285+0000","last_undegraded":"2023-06-10T21:36:46.040285+0000","last_fullsized":"2023-06-10T21:36:46.040285+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:21:05.809917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":296,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042458+0000","last_change":"2023-06-10T21:28:57.898638+0000","last_active":"2023-06-10T21:36:46.042458+0000","last_peered":"2023-06-10T21:36:46.042458+0000","last_clean":"2023-06-10T21:36:46.042458+0000","last_became_active":"2023-06-10T21:28:57.898279+0000","last_became_peered":"2023-06-10T21:28:57.898279+0000","last_unstale":"2023-06-10T21:36:46.042458+0000","last_undegraded":"2023-06-10T21:36:46.042458+0000","last_fullsized":"2023-06-10T21:36:46.042458+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:01.751627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":289,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.254067+0000","last_change":"2023-06-10T21:28:57.901590+0000","last_active":"2023-06-10T21:36:53.254067+0000","last_peered":"2023-06-10T21:36:53.254067+0000","last_clean":"2023-06-10T21:36:53.254067+0000","last_became_active":"2023-06-10T21:28:57.900682+0000","last_became_peered":"2023-06-10T21:28:57.900682+0000","last_unstale":"2023-06-10T21:36:53.254067+0000","last_undegraded":"2023-06-10T21:36:53.254067+0000","last_fullsized":"2023-06-10T21:36:53.254067+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:33.439855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":178,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040580+0000","last_change":"2023-06-10T21:28:57.901770+0000","last_active":"2023-06-10T21:36:46.040580+0000","last_peered":"2023-06-10T21:36:46.040580+0000","last_clean":"2023-06-10T21:36:46.040580+0000","last_became_active":"2023-06-10T21:28:57.901102+0000","last_became_peered":"2023-06-10T21:28:57.901102+0000","last_unstale":"2023-06-10T21:36:46.040580+0000","last_undegraded":"2023-06-10T21:36:46.040580+0000","last_fullsized":"2023-06-10T21:36:46.040580+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:08.708837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":270,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041316+0000","last_change":"2023-06-10T21:28:57.894627+0000","last_active":"2023-06-10T21:36:46.041316+0000","last_peered":"2023-06-10T21:36:46.041316+0000","last_clean":"2023-06-10T21:36:46.041316+0000","last_became_active":"2023-06-10T21:28:57.894312+0000","last_became_peered":"2023-06-10T21:28:57.894312+0000","last_unstale":"2023-06-10T21:36:46.041316+0000","last_undegraded":"2023-06-10T21:36:46.041316+0000","last_fullsized":"2023-06-10T21:36:46.041316+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:26:21.650467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":42,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044016+0000","last_change":"2023-06-10T21:32:55.526173+0000","last_active":"2023-06-10T21:36:46.044016+0000","last_peered":"2023-06-10T21:36:46.044016+0000","last_clean":"2023-06-10T21:36:46.044016+0000","last_became_active":"2023-06-10T21:32:55.525779+0000","last_became_peered":"2023-06-10T21:32:55.525779+0000","last_unstale":"2023-06-10T21:36:46.044016+0000","last_undegraded":"2023-06-10T21:36:46.044016+0000","last_fullsized":"2023-06-10T21:36:46.044016+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":44,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042131+0000","last_change":"2023-06-10T21:32:55.516932+0000","last_active":"2023-06-10T21:36:46.042131+0000","last_peered":"2023-06-10T21:36:46.042131+0000","last_clean":"2023-06-10T21:36:46.042131+0000","last_became_active":"2023-06-10T21:32:55.516387+0000","last_became_peered":"2023-06-10T21:32:55.516387+0000","last_unstale":"2023-06-10T21:36:46.042131+0000","last_undegraded":"2023-06-10T21:36:46.042131+0000","last_fullsized":"2023-06-10T21:36:46.042131+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:40:43.246128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":48,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043854+0000","last_change":"2023-06-10T21:32:55.786053+0000","last_active":"2023-06-10T21:36:46.043854+0000","last_peered":"2023-06-10T21:36:46.043854+0000","last_clean":"2023-06-10T21:36:46.043854+0000","last_became_active":"2023-06-10T21:32:55.517614+0000","last_became_peered":"2023-06-10T21:32:55.517614+0000","last_unstale":"2023-06-10T21:36:46.043854+0000","last_undegraded":"2023-06-10T21:36:46.043854+0000","last_fullsized":"2023-06-10T21:36:46.043854+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":284,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.049339+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:36:46.049339+0000","last_peered":"2023-06-10T21:36:46.049339+0000","last_clean":"2023-06-10T21:36:46.049339+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:36:46.049339+0000","last_undegraded":"2023-06-10T21:36:46.049339+0000","last_fullsized":"2023-06-10T21:36:46.049339+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":191,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043843+0000","last_change":"2023-06-10T21:26:15.940674+0000","last_active":"2023-06-10T21:36:46.043843+0000","last_peered":"2023-06-10T21:36:46.043843+0000","last_clean":"2023-06-10T21:36:46.043843+0000","last_became_active":"2023-06-10T21:26:15.746969+0000","last_became_peered":"2023-06-10T21:26:15.746969+0000","last_unstale":"2023-06-10T21:36:46.043843+0000","last_undegraded":"2023-06-10T21:36:46.043843+0000","last_fullsized":"2023-06-10T21:36:46.043843+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":52,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043561+0000","last_change":"2023-06-10T21:31:34.294678+0000","last_active":"2023-06-10T21:36:46.043561+0000","last_peered":"2023-06-10T21:36:46.043561+0000","last_clean":"2023-06-10T21:36:46.043561+0000","last_became_active":"2023-06-10T21:31:34.294128+0000","last_became_peered":"2023-06-10T21:31:34.294128+0000","last_unstale":"2023-06-10T21:36:46.043561+0000","last_undegraded":"2023-06-10T21:36:46.043561+0000","last_fullsized":"2023-06-10T21:36:46.043561+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":320,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043437+0000","last_change":"2023-06-10T21:25:30.288329+0000","last_active":"2023-06-10T21:36:46.043437+0000","last_peered":"2023-06-10T21:36:46.043437+0000","last_clean":"2023-06-10T21:36:46.043437+0000","last_became_active":"2023-06-10T21:25:30.194776+0000","last_became_peered":"2023-06-10T21:25:30.194776+0000","last_unstale":"2023-06-10T21:36:46.043437+0000","last_undegraded":"2023-06-10T21:36:46.043437+0000","last_fullsized":"2023-06-10T21:36:46.043437+0000","mapping_epoch":196,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":256,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041561+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:36:46.041561+0000","last_peered":"2023-06-10T21:36:46.041561+0000","last_clean":"2023-06-10T21:36:46.041561+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:36:46.041561+0000","last_undegraded":"2023-06-10T21:36:46.041561+0000","last_fullsized":"2023-06-10T21:36:46.041561+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.039352+0000","last_change":"2023-06-10T21:32:55.522242+0000","last_active":"2023-06-10T21:36:46.039352+0000","last_peered":"2023-06-10T21:36:46.039352+0000","last_clean":"2023-06-10T21:36:46.039352+0000","last_became_active":"2023-06-10T21:32:55.521912+0000","last_became_peered":"2023-06-10T21:32:55.521912+0000","last_unstale":"2023-06-10T21:36:46.039352+0000","last_undegraded":"2023-06-10T21:36:46.039352+0000","last_fullsized":"2023-06-10T21:36:46.039352+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:21.810286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":249,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044141+0000","last_change":"2023-06-10T21:28:57.899835+0000","last_active":"2023-06-10T21:36:46.044141+0000","last_peered":"2023-06-10T21:36:46.044141+0000","last_clean":"2023-06-10T21:36:46.044141+0000","last_became_active":"2023-06-10T21:28:57.899293+0000","last_became_peered":"2023-06-10T21:28:57.899293+0000","last_unstale":"2023-06-10T21:36:46.044141+0000","last_undegraded":"2023-06-10T21:36:46.044141+0000","last_fullsized":"2023-06-10T21:36:46.044141+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":297,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043347+0000","last_change":"2023-06-10T21:28:34.505255+0000","last_active":"2023-06-10T21:36:46.043347+0000","last_peered":"2023-06-10T21:36:46.043347+0000","last_clean":"2023-06-10T21:36:46.043347+0000","last_became_active":"2023-06-10T21:28:34.504873+0000","last_became_peered":"2023-06-10T21:28:34.504873+0000","last_unstale":"2023-06-10T21:36:46.043347+0000","last_undegraded":"2023-06-10T21:36:46.043347+0000","last_fullsized":"2023-06-10T21:36:46.043347+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040465+0000","last_change":"2023-06-10T21:32:55.522804+0000","last_active":"2023-06-10T21:36:46.040465+0000","last_peered":"2023-06-10T21:36:46.040465+0000","last_clean":"2023-06-10T21:36:46.040465+0000","last_became_active":"2023-06-10T21:32:55.522114+0000","last_became_peered":"2023-06-10T21:32:55.522114+0000","last_unstale":"2023-06-10T21:36:46.040465+0000","last_undegraded":"2023-06-10T21:36:46.040465+0000","last_fullsized":"2023-06-10T21:36:46.040465+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:51:16.840494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":297,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.039907+0000","last_change":"2023-06-10T21:28:57.903746+0000","last_active":"2023-06-10T21:36:46.039907+0000","last_peered":"2023-06-10T21:36:46.039907+0000","last_clean":"2023-06-10T21:36:46.039907+0000","last_became_active":"2023-06-10T21:28:57.903055+0000","last_became_peered":"2023-06-10T21:28:57.903055+0000","last_unstale":"2023-06-10T21:36:46.039907+0000","last_undegraded":"2023-06-10T21:36:46.039907+0000","last_fullsized":"2023-06-10T21:36:46.039907+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:31:59.095952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.056406+0000","last_change":"2023-06-10T21:36:35.210652+0000","last_active":"2023-06-10T21:36:46.056406+0000","last_peered":"2023-06-10T21:36:46.056406+0000","last_clean":"2023-06-10T21:36:46.056406+0000","last_became_active":"2023-06-10T21:36:35.210329+0000","last_became_peered":"2023-06-10T21:36:35.210329+0000","last_unstale":"2023-06-10T21:36:46.056406+0000","last_undegraded":"2023-06-10T21:36:46.056406+0000","last_fullsized":"2023-06-10T21:36:46.056406+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:21:02.524705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.0","version":"321'4140","reported_seq":4159,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:09.526500+0000","last_change":"2023-06-10T21:36:04.325572+0000","last_active":"2023-06-10T21:37:09.526500+0000","last_peered":"2023-06-10T21:37:09.526500+0000","last_clean":"2023-06-10T21:37:09.526500+0000","last_became_active":"2023-06-10T21:36:04.325154+0000","last_became_peered":"2023-06-10T21:36:04.325154+0000","last_unstale":"2023-06-10T21:37:09.526500+0000","last_undegraded":"2023-06-10T21:37:09.526500+0000","last_fullsized":"2023-06-10T21:37:09.526500+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4140,"log_dups_size":0,"ondisk_log_size":4140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:53.858610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271319040,"num_objects":4140,"num_object_clones":0,"num_object_copies":12420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4140,"num_write_kb":264960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":46,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044079+0000","last_change":"2023-06-10T21:32:55.524283+0000","last_active":"2023-06-10T21:36:46.044079+0000","last_peered":"2023-06-10T21:36:46.044079+0000","last_clean":"2023-06-10T21:36:46.044079+0000","last_became_active":"2023-06-10T21:32:55.513505+0000","last_became_peered":"2023-06-10T21:32:55.513505+0000","last_unstale":"2023-06-10T21:36:46.044079+0000","last_undegraded":"2023-06-10T21:36:46.044079+0000","last_fullsized":"2023-06-10T21:36:46.044079+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043195+0000","last_change":"2023-06-10T21:36:35.214211+0000","last_active":"2023-06-10T21:36:46.043195+0000","last_peered":"2023-06-10T21:36:46.043195+0000","last_clean":"2023-06-10T21:36:46.043195+0000","last_became_active":"2023-06-10T21:36:35.213859+0000","last_became_peered":"2023-06-10T21:36:35.213859+0000","last_unstale":"2023-06-10T21:36:46.043195+0000","last_undegraded":"2023-06-10T21:36:46.043195+0000","last_fullsized":"2023-06-10T21:36:46.043195+0000","mapping_epoch":318,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"321'4243","reported_seq":4262,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:11.631881+0000","last_change":"2023-06-10T21:36:04.292881+0000","last_active":"2023-06-10T21:37:11.631881+0000","last_peered":"2023-06-10T21:37:11.631881+0000","last_clean":"2023-06-10T21:37:11.631881+0000","last_became_active":"2023-06-10T21:36:04.292606+0000","last_became_peered":"2023-06-10T21:36:04.292606+0000","last_unstale":"2023-06-10T21:37:11.631881+0000","last_undegraded":"2023-06-10T21:37:11.631881+0000","last_fullsized":"2023-06-10T21:37:11.631881+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4243,"log_dups_size":0,"ondisk_log_size":4243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:30:21.334865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278069248,"num_objects":4243,"num_object_clones":0,"num_object_copies":12729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4243,"num_write_kb":271552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":46,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044072+0000","last_change":"2023-06-10T21:32:55.787965+0000","last_active":"2023-06-10T21:36:46.044072+0000","last_peered":"2023-06-10T21:36:46.044072+0000","last_clean":"2023-06-10T21:36:46.044072+0000","last_became_active":"2023-06-10T21:32:55.525804+0000","last_became_peered":"2023-06-10T21:32:55.525804+0000","last_unstale":"2023-06-10T21:36:46.044072+0000","last_undegraded":"2023-06-10T21:36:46.044072+0000","last_fullsized":"2023-06-10T21:36:46.044072+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":195,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042585+0000","last_change":"2023-06-10T21:28:57.893472+0000","last_active":"2023-06-10T21:36:46.042585+0000","last_peered":"2023-06-10T21:36:46.042585+0000","last_clean":"2023-06-10T21:36:46.042585+0000","last_became_active":"2023-06-10T21:28:57.893160+0000","last_became_peered":"2023-06-10T21:28:57.893160+0000","last_unstale":"2023-06-10T21:36:46.042585+0000","last_undegraded":"2023-06-10T21:36:46.042585+0000","last_fullsized":"2023-06-10T21:36:46.042585+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:02:34.697136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"321'4273","reported_seq":4291,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:11.714627+0000","last_change":"2023-06-10T21:36:04.325459+0000","last_active":"2023-06-10T21:37:11.714627+0000","last_peered":"2023-06-10T21:37:11.714627+0000","last_clean":"2023-06-10T21:37:11.714627+0000","last_became_active":"2023-06-10T21:36:04.325111+0000","last_became_peered":"2023-06-10T21:36:04.325111+0000","last_unstale":"2023-06-10T21:37:11.714627+0000","last_undegraded":"2023-06-10T21:37:11.714627+0000","last_fullsized":"2023-06-10T21:37:11.714627+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4273,"log_dups_size":0,"ondisk_log_size":4273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:49:06.786091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280035328,"num_objects":4273,"num_object_clones":0,"num_object_copies":12819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4273,"num_write_kb":273472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":328,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043898+0000","last_change":"2023-06-10T21:31:34.339313+0000","last_active":"2023-06-10T21:36:46.043898+0000","last_peered":"2023-06-10T21:36:46.043898+0000","last_clean":"2023-06-10T21:36:46.043898+0000","last_became_active":"2023-06-10T21:31:34.294249+0000","last_became_peered":"2023-06-10T21:31:34.294249+0000","last_unstale":"2023-06-10T21:36:46.043898+0000","last_undegraded":"2023-06-10T21:36:46.043898+0000","last_fullsized":"2023-06-10T21:36:46.043898+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":42,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043340+0000","last_change":"2023-06-10T21:32:55.513302+0000","last_active":"2023-06-10T21:36:46.043340+0000","last_peered":"2023-06-10T21:36:46.043340+0000","last_clean":"2023-06-10T21:36:46.043340+0000","last_became_active":"2023-06-10T21:32:55.512903+0000","last_became_peered":"2023-06-10T21:32:55.512903+0000","last_unstale":"2023-06-10T21:36:46.043340+0000","last_undegraded":"2023-06-10T21:36:46.043340+0000","last_fullsized":"2023-06-10T21:36:46.043340+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"321'4210","reported_seq":4229,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.262985+0000","last_change":"2023-06-10T21:36:04.289584+0000","last_active":"2023-06-10T21:37:08.262985+0000","last_peered":"2023-06-10T21:37:08.262985+0000","last_clean":"2023-06-10T21:37:08.262985+0000","last_became_active":"2023-06-10T21:36:04.289267+0000","last_became_peered":"2023-06-10T21:36:04.289267+0000","last_unstale":"2023-06-10T21:37:08.262985+0000","last_undegraded":"2023-06-10T21:37:08.262985+0000","last_fullsized":"2023-06-10T21:37:08.262985+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4210,"log_dups_size":0,"ondisk_log_size":4210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:34.840732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275906560,"num_objects":4210,"num_object_clones":0,"num_object_copies":12630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4210,"num_write_kb":269440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041797+0000","last_change":"2023-06-10T21:28:57.900615+0000","last_active":"2023-06-10T21:36:46.041797+0000","last_peered":"2023-06-10T21:36:46.041797+0000","last_clean":"2023-06-10T21:36:46.041797+0000","last_became_active":"2023-06-10T21:28:57.900064+0000","last_became_peered":"2023-06-10T21:28:57.900064+0000","last_unstale":"2023-06-10T21:36:46.041797+0000","last_undegraded":"2023-06-10T21:36:46.041797+0000","last_fullsized":"2023-06-10T21:36:46.041797+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:07:45.971352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041915+0000","last_change":"2023-06-10T21:28:34.541292+0000","last_active":"2023-06-10T21:36:46.041915+0000","last_peered":"2023-06-10T21:36:46.041915+0000","last_clean":"2023-06-10T21:36:46.041915+0000","last_became_active":"2023-06-10T21:28:34.517823+0000","last_became_peered":"2023-06-10T21:28:34.517823+0000","last_unstale":"2023-06-10T21:36:46.041915+0000","last_undegraded":"2023-06-10T21:36:46.041915+0000","last_fullsized":"2023-06-10T21:36:46.041915+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:22.762397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"321'4285","reported_seq":4304,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:11.716639+0000","last_change":"2023-06-10T21:36:04.288539+0000","last_active":"2023-06-10T21:37:11.716639+0000","last_peered":"2023-06-10T21:37:11.716639+0000","last_clean":"2023-06-10T21:37:11.716639+0000","last_became_active":"2023-06-10T21:36:04.288222+0000","last_became_peered":"2023-06-10T21:36:04.288222+0000","last_unstale":"2023-06-10T21:37:11.716639+0000","last_undegraded":"2023-06-10T21:37:11.716639+0000","last_fullsized":"2023-06-10T21:37:11.716639+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4285,"log_dups_size":0,"ondisk_log_size":4285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:53.556650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280821760,"num_objects":4285,"num_object_clones":0,"num_object_copies":12855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4285,"num_write_kb":274240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":273,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042961+0000","last_change":"2023-06-10T21:28:57.899332+0000","last_active":"2023-06-10T21:36:46.042961+0000","last_peered":"2023-06-10T21:36:46.042961+0000","last_clean":"2023-06-10T21:36:46.042961+0000","last_became_active":"2023-06-10T21:28:57.898986+0000","last_became_peered":"2023-06-10T21:28:57.898986+0000","last_unstale":"2023-06-10T21:36:46.042961+0000","last_undegraded":"2023-06-10T21:36:46.042961+0000","last_fullsized":"2023-06-10T21:36:46.042961+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:47:43.935837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":193,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041873+0000","last_change":"2023-06-10T21:28:57.895946+0000","last_active":"2023-06-10T21:36:46.041873+0000","last_peered":"2023-06-10T21:36:46.041873+0000","last_clean":"2023-06-10T21:36:46.041873+0000","last_became_active":"2023-06-10T21:28:57.895433+0000","last_became_peered":"2023-06-10T21:28:57.895433+0000","last_unstale":"2023-06-10T21:36:46.041873+0000","last_undegraded":"2023-06-10T21:36:46.041873+0000","last_fullsized":"2023-06-10T21:36:46.041873+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:15:42.708789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"321'4226","reported_seq":4245,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:09.507817+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:37:09.507817+0000","last_peered":"2023-06-10T21:37:09.507817+0000","last_clean":"2023-06-10T21:37:09.507817+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:37:09.507817+0000","last_undegraded":"2023-06-10T21:37:09.507817+0000","last_fullsized":"2023-06-10T21:37:09.507817+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4226,"log_dups_size":0,"ondisk_log_size":4226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276955136,"num_objects":4226,"num_object_clones":0,"num_object_copies":12678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4226,"num_write_kb":270464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042531+0000","last_change":"2023-06-10T21:28:57.895839+0000","last_active":"2023-06-10T21:36:46.042531+0000","last_peered":"2023-06-10T21:36:46.042531+0000","last_clean":"2023-06-10T21:36:46.042531+0000","last_became_active":"2023-06-10T21:28:57.895502+0000","last_became_peered":"2023-06-10T21:28:57.895502+0000","last_unstale":"2023-06-10T21:36:46.042531+0000","last_undegraded":"2023-06-10T21:36:46.042531+0000","last_fullsized":"2023-06-10T21:36:46.042531+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:08:42.745471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":313,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.056316+0000","last_change":"2023-06-10T21:26:01.624960+0000","last_active":"2023-06-10T21:36:46.056316+0000","last_peered":"2023-06-10T21:36:46.056316+0000","last_clean":"2023-06-10T21:36:46.056316+0000","last_became_active":"2023-06-10T21:26:01.624656+0000","last_became_peered":"2023-06-10T21:26:01.624656+0000","last_unstale":"2023-06-10T21:36:46.056316+0000","last_undegraded":"2023-06-10T21:36:46.056316+0000","last_fullsized":"2023-06-10T21:36:46.056316+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:33:39.010264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"321'4095","reported_seq":4114,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.239559+0000","last_change":"2023-06-10T21:36:04.281931+0000","last_active":"2023-06-10T21:37:08.239559+0000","last_peered":"2023-06-10T21:37:08.239559+0000","last_clean":"2023-06-10T21:37:08.239559+0000","last_became_active":"2023-06-10T21:36:04.281632+0000","last_became_peered":"2023-06-10T21:36:04.281632+0000","last_unstale":"2023-06-10T21:37:08.239559+0000","last_undegraded":"2023-06-10T21:37:08.239559+0000","last_fullsized":"2023-06-10T21:37:08.239559+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4095,"log_dups_size":0,"ondisk_log_size":4095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:57.076427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268369920,"num_objects":4095,"num_object_clones":0,"num_object_copies":12285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4095,"num_write_kb":262080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":300,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:54.513423+0000","last_change":"2023-06-10T21:26:01.626635+0000","last_active":"2023-06-10T21:36:54.513423+0000","last_peered":"2023-06-10T21:36:54.513423+0000","last_clean":"2023-06-10T21:36:54.513423+0000","last_became_active":"2023-06-10T21:26:01.626291+0000","last_became_peered":"2023-06-10T21:26:01.626291+0000","last_unstale":"2023-06-10T21:36:54.513423+0000","last_undegraded":"2023-06-10T21:36:54.513423+0000","last_fullsized":"2023-06-10T21:36:54.513423+0000","mapping_epoch":210,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":211,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:31:28.427087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":96,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041904+0000","last_change":"2023-06-10T21:28:57.893821+0000","last_active":"2023-06-10T21:36:46.041904+0000","last_peered":"2023-06-10T21:36:46.041904+0000","last_clean":"2023-06-10T21:36:46.041904+0000","last_became_active":"2023-06-10T21:28:57.893121+0000","last_became_peered":"2023-06-10T21:28:57.893121+0000","last_unstale":"2023-06-10T21:36:46.041904+0000","last_undegraded":"2023-06-10T21:36:46.041904+0000","last_fullsized":"2023-06-10T21:36:46.041904+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:56:06.527572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"321'4093","reported_seq":4110,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.945266+0000","last_change":"2023-06-10T21:36:04.289902+0000","last_active":"2023-06-10T21:37:08.945266+0000","last_peered":"2023-06-10T21:37:08.945266+0000","last_clean":"2023-06-10T21:37:08.945266+0000","last_became_active":"2023-06-10T21:36:04.289647+0000","last_became_peered":"2023-06-10T21:36:04.289647+0000","last_unstale":"2023-06-10T21:37:08.945266+0000","last_undegraded":"2023-06-10T21:37:08.945266+0000","last_fullsized":"2023-06-10T21:37:08.945266+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4093,"log_dups_size":0,"ondisk_log_size":4093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268238848,"num_objects":4093,"num_object_clones":0,"num_object_copies":12279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4093,"num_write_kb":261952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":276,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042280+0000","last_change":"2023-06-10T21:32:55.519532+0000","last_active":"2023-06-10T21:36:46.042280+0000","last_peered":"2023-06-10T21:36:46.042280+0000","last_clean":"2023-06-10T21:36:46.042280+0000","last_became_active":"2023-06-10T21:32:55.515726+0000","last_became_peered":"2023-06-10T21:32:55.515726+0000","last_unstale":"2023-06-10T21:36:46.042280+0000","last_undegraded":"2023-06-10T21:36:46.042280+0000","last_fullsized":"2023-06-10T21:36:46.042280+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:55:27.300189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":232,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042202+0000","last_change":"2023-06-10T21:28:57.895485+0000","last_active":"2023-06-10T21:36:46.042202+0000","last_peered":"2023-06-10T21:36:46.042202+0000","last_clean":"2023-06-10T21:36:46.042202+0000","last_became_active":"2023-06-10T21:28:57.894553+0000","last_became_peered":"2023-06-10T21:28:57.894553+0000","last_unstale":"2023-06-10T21:36:46.042202+0000","last_undegraded":"2023-06-10T21:36:46.042202+0000","last_fullsized":"2023-06-10T21:36:46.042202+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:15:50.780788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"321'4217","reported_seq":4236,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:09.457422+0000","last_change":"2023-06-10T21:36:04.291075+0000","last_active":"2023-06-10T21:37:09.457422+0000","last_peered":"2023-06-10T21:37:09.457422+0000","last_clean":"2023-06-10T21:37:09.457422+0000","last_became_active":"2023-06-10T21:36:04.290808+0000","last_became_peered":"2023-06-10T21:36:04.290808+0000","last_unstale":"2023-06-10T21:37:09.457422+0000","last_undegraded":"2023-06-10T21:37:09.457422+0000","last_fullsized":"2023-06-10T21:37:09.457422+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4217,"log_dups_size":0,"ondisk_log_size":4217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:13:41.537032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276365312,"num_objects":4217,"num_object_clones":0,"num_object_copies":12651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4217,"num_write_kb":269888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":146,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253989+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:36:53.253989+0000","last_peered":"2023-06-10T21:36:53.253989+0000","last_clean":"2023-06-10T21:36:53.253989+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:36:53.253989+0000","last_undegraded":"2023-06-10T21:36:53.253989+0000","last_fullsized":"2023-06-10T21:36:53.253989+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":325,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041914+0000","last_change":"2023-06-10T21:32:55.527141+0000","last_active":"2023-06-10T21:36:46.041914+0000","last_peered":"2023-06-10T21:36:46.041914+0000","last_clean":"2023-06-10T21:36:46.041914+0000","last_became_active":"2023-06-10T21:32:55.523371+0000","last_became_peered":"2023-06-10T21:32:55.523371+0000","last_unstale":"2023-06-10T21:36:46.041914+0000","last_undegraded":"2023-06-10T21:36:46.041914+0000","last_fullsized":"2023-06-10T21:36:46.041914+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:43.083271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":166,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253886+0000","last_change":"2023-06-10T21:28:34.505530+0000","last_active":"2023-06-10T21:36:53.253886+0000","last_peered":"2023-06-10T21:36:53.253886+0000","last_clean":"2023-06-10T21:36:53.253886+0000","last_became_active":"2023-06-10T21:28:34.504654+0000","last_became_peered":"2023-06-10T21:28:34.504654+0000","last_unstale":"2023-06-10T21:36:53.253886+0000","last_undegraded":"2023-06-10T21:36:53.253886+0000","last_fullsized":"2023-06-10T21:36:53.253886+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:04:09.030219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"321'4231","reported_seq":4250,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.224512+0000","last_change":"2023-06-10T21:36:04.286731+0000","last_active":"2023-06-10T21:37:08.224512+0000","last_peered":"2023-06-10T21:37:08.224512+0000","last_clean":"2023-06-10T21:37:08.224512+0000","last_became_active":"2023-06-10T21:36:04.286427+0000","last_became_peered":"2023-06-10T21:36:04.286427+0000","last_unstale":"2023-06-10T21:37:08.224512+0000","last_undegraded":"2023-06-10T21:37:08.224512+0000","last_fullsized":"2023-06-10T21:37:08.224512+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4231,"log_dups_size":0,"ondisk_log_size":4231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:26:20.203365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277282816,"num_objects":4231,"num_object_clones":0,"num_object_copies":12693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4231,"num_write_kb":270784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":192,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253910+0000","last_change":"2023-06-10T21:28:57.909582+0000","last_active":"2023-06-10T21:36:53.253910+0000","last_peered":"2023-06-10T21:36:53.253910+0000","last_clean":"2023-06-10T21:36:53.253910+0000","last_became_active":"2023-06-10T21:28:57.909217+0000","last_became_peered":"2023-06-10T21:28:57.909217+0000","last_unstale":"2023-06-10T21:36:53.253910+0000","last_undegraded":"2023-06-10T21:36:53.253910+0000","last_fullsized":"2023-06-10T21:36:53.253910+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:30:38.071602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":266,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253950+0000","last_change":"2023-06-10T21:28:34.540915+0000","last_active":"2023-06-10T21:36:53.253950+0000","last_peered":"2023-06-10T21:36:53.253950+0000","last_clean":"2023-06-10T21:36:53.253950+0000","last_became_active":"2023-06-10T21:28:34.517023+0000","last_became_peered":"2023-06-10T21:28:34.517023+0000","last_unstale":"2023-06-10T21:36:53.253950+0000","last_undegraded":"2023-06-10T21:36:53.253950+0000","last_fullsized":"2023-06-10T21:36:53.253950+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:16:27.132599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.056879+0000","last_change":"2023-06-10T21:28:57.958907+0000","last_active":"2023-06-10T21:36:46.056879+0000","last_peered":"2023-06-10T21:36:46.056879+0000","last_clean":"2023-06-10T21:36:46.056879+0000","last_became_active":"2023-06-10T21:28:57.958608+0000","last_became_peered":"2023-06-10T21:28:57.958608+0000","last_unstale":"2023-06-10T21:36:46.056879+0000","last_undegraded":"2023-06-10T21:36:46.056879+0000","last_fullsized":"2023-06-10T21:36:46.056879+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:37:49.088322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"321'4196","reported_seq":4215,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:09.514375+0000","last_change":"2023-06-10T21:36:04.325488+0000","last_active":"2023-06-10T21:37:09.514375+0000","last_peered":"2023-06-10T21:37:09.514375+0000","last_clean":"2023-06-10T21:37:09.514375+0000","last_became_active":"2023-06-10T21:36:04.325162+0000","last_became_peered":"2023-06-10T21:36:04.325162+0000","last_unstale":"2023-06-10T21:37:09.514375+0000","last_undegraded":"2023-06-10T21:37:09.514375+0000","last_fullsized":"2023-06-10T21:37:09.514375+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4196,"log_dups_size":0,"ondisk_log_size":4196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:41:33.200826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274989056,"num_objects":4196,"num_object_clones":0,"num_object_copies":12588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4196,"num_write_kb":268544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":166,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.052982+0000","last_change":"2023-06-10T21:28:57.931846+0000","last_active":"2023-06-10T21:36:46.052982+0000","last_peered":"2023-06-10T21:36:46.052982+0000","last_clean":"2023-06-10T21:36:46.052982+0000","last_became_active":"2023-06-10T21:28:57.931565+0000","last_became_peered":"2023-06-10T21:28:57.931565+0000","last_unstale":"2023-06-10T21:36:46.052982+0000","last_undegraded":"2023-06-10T21:36:46.052982+0000","last_fullsized":"2023-06-10T21:36:46.052982+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:30:11.804307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":164,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.050171+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:36:46.050171+0000","last_peered":"2023-06-10T21:36:46.050171+0000","last_clean":"2023-06-10T21:36:46.050171+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:36:46.050171+0000","last_undegraded":"2023-06-10T21:36:46.050171+0000","last_fullsized":"2023-06-10T21:36:46.050171+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":214,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040658+0000","last_change":"2023-06-10T21:32:55.515042+0000","last_active":"2023-06-10T21:36:46.040658+0000","last_peered":"2023-06-10T21:36:46.040658+0000","last_clean":"2023-06-10T21:36:46.040658+0000","last_became_active":"2023-06-10T21:32:55.514699+0000","last_became_peered":"2023-06-10T21:32:55.514699+0000","last_unstale":"2023-06-10T21:36:46.040658+0000","last_undegraded":"2023-06-10T21:36:46.040658+0000","last_fullsized":"2023-06-10T21:36:46.040658+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:59:25.578821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"321'4090","reported_seq":4108,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.909913+0000","last_change":"2023-06-10T21:36:04.289917+0000","last_active":"2023-06-10T21:37:08.909913+0000","last_peered":"2023-06-10T21:37:08.909913+0000","last_clean":"2023-06-10T21:37:08.909913+0000","last_became_active":"2023-06-10T21:36:04.289698+0000","last_became_peered":"2023-06-10T21:36:04.289698+0000","last_unstale":"2023-06-10T21:37:08.909913+0000","last_undegraded":"2023-06-10T21:37:08.909913+0000","last_fullsized":"2023-06-10T21:37:08.909913+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4090,"log_dups_size":0,"ondisk_log_size":4090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4090,"num_write_kb":261760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":346,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.056216+0000","last_change":"2023-06-10T21:32:55.522409+0000","last_active":"2023-06-10T21:36:46.056216+0000","last_peered":"2023-06-10T21:36:46.056216+0000","last_clean":"2023-06-10T21:36:46.056216+0000","last_became_active":"2023-06-10T21:32:55.522064+0000","last_became_peered":"2023-06-10T21:32:55.522064+0000","last_unstale":"2023-06-10T21:36:46.056216+0000","last_undegraded":"2023-06-10T21:36:46.056216+0000","last_fullsized":"2023-06-10T21:36:46.056216+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:24.584376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":484,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043207+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:36:46.043207+0000","last_peered":"2023-06-10T21:36:46.043207+0000","last_clean":"2023-06-10T21:36:46.043207+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:36:46.043207+0000","last_undegraded":"2023-06-10T21:36:46.043207+0000","last_fullsized":"2023-06-10T21:36:46.043207+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042694+0000","last_change":"2023-06-10T21:28:57.896248+0000","last_active":"2023-06-10T21:36:46.042694+0000","last_peered":"2023-06-10T21:36:46.042694+0000","last_clean":"2023-06-10T21:36:46.042694+0000","last_became_active":"2023-06-10T21:28:57.895913+0000","last_became_peered":"2023-06-10T21:28:57.895913+0000","last_unstale":"2023-06-10T21:36:46.042694+0000","last_undegraded":"2023-06-10T21:36:46.042694+0000","last_fullsized":"2023-06-10T21:36:46.042694+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:33:12.842409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":135,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044056+0000","last_change":"2023-06-10T21:28:34.502022+0000","last_active":"2023-06-10T21:36:46.044056+0000","last_peered":"2023-06-10T21:36:46.044056+0000","last_clean":"2023-06-10T21:36:46.044056+0000","last_became_active":"2023-06-10T21:28:34.501721+0000","last_became_peered":"2023-06-10T21:28:34.501721+0000","last_unstale":"2023-06-10T21:36:46.044056+0000","last_undegraded":"2023-06-10T21:36:46.044056+0000","last_fullsized":"2023-06-10T21:36:46.044056+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"321'4122","reported_seq":4141,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:08.211349+0000","last_change":"2023-06-10T21:36:04.325644+0000","last_active":"2023-06-10T21:37:08.211349+0000","last_peered":"2023-06-10T21:37:08.211349+0000","last_clean":"2023-06-10T21:37:08.211349+0000","last_became_active":"2023-06-10T21:36:04.325301+0000","last_became_peered":"2023-06-10T21:36:04.325301+0000","last_unstale":"2023-06-10T21:37:08.211349+0000","last_undegraded":"2023-06-10T21:37:08.211349+0000","last_fullsized":"2023-06-10T21:37:08.211349+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4122,"log_dups_size":0,"ondisk_log_size":4122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:28:03.018219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270139392,"num_objects":4122,"num_object_clones":0,"num_object_copies":12366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4122,"num_write_kb":263808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":340,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043283+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:36:46.043283+0000","last_peered":"2023-06-10T21:36:46.043283+0000","last_clean":"2023-06-10T21:36:46.043283+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:36:46.043283+0000","last_undegraded":"2023-06-10T21:36:46.043283+0000","last_fullsized":"2023-06-10T21:36:46.043283+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253871+0000","last_change":"2023-06-10T21:28:57.901546+0000","last_active":"2023-06-10T21:36:53.253871+0000","last_peered":"2023-06-10T21:36:53.253871+0000","last_clean":"2023-06-10T21:36:53.253871+0000","last_became_active":"2023-06-10T21:28:57.901077+0000","last_became_peered":"2023-06-10T21:28:57.901077+0000","last_unstale":"2023-06-10T21:36:53.253871+0000","last_undegraded":"2023-06-10T21:36:53.253871+0000","last_fullsized":"2023-06-10T21:36:53.253871+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:16:59.648423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040616+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:36:46.040616+0000","last_peered":"2023-06-10T21:36:46.040616+0000","last_clean":"2023-06-10T21:36:46.040616+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:36:46.040616+0000","last_undegraded":"2023-06-10T21:36:46.040616+0000","last_fullsized":"2023-06-10T21:36:46.040616+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"321'4296","reported_seq":4315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:11.704809+0000","last_change":"2023-06-10T21:36:04.325466+0000","last_active":"2023-06-10T21:37:11.704809+0000","last_peered":"2023-06-10T21:37:11.704809+0000","last_clean":"2023-06-10T21:37:11.704809+0000","last_became_active":"2023-06-10T21:36:04.325178+0000","last_became_peered":"2023-06-10T21:36:04.325178+0000","last_unstale":"2023-06-10T21:37:11.704809+0000","last_undegraded":"2023-06-10T21:37:11.704809+0000","last_fullsized":"2023-06-10T21:37:11.704809+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4296,"log_dups_size":0,"ondisk_log_size":4296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:03.551659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281542656,"num_objects":4296,"num_object_clones":0,"num_object_copies":12888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4296,"num_write_kb":274944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043943+0000","last_change":"2023-06-10T21:28:34.502265+0000","last_active":"2023-06-10T21:36:46.043943+0000","last_peered":"2023-06-10T21:36:46.043943+0000","last_clean":"2023-06-10T21:36:46.043943+0000","last_became_active":"2023-06-10T21:28:34.501984+0000","last_became_peered":"2023-06-10T21:28:34.501984+0000","last_unstale":"2023-06-10T21:36:46.043943+0000","last_undegraded":"2023-06-10T21:36:46.043943+0000","last_fullsized":"2023-06-10T21:36:46.043943+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":164,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.056954+0000","last_change":"2023-06-10T21:28:34.513914+0000","last_active":"2023-06-10T21:36:46.056954+0000","last_peered":"2023-06-10T21:36:46.056954+0000","last_clean":"2023-06-10T21:36:46.056954+0000","last_became_active":"2023-06-10T21:28:34.513586+0000","last_became_peered":"2023-06-10T21:28:34.513586+0000","last_unstale":"2023-06-10T21:36:46.056954+0000","last_undegraded":"2023-06-10T21:36:46.056954+0000","last_fullsized":"2023-06-10T21:36:46.056954+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:49:21.634570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":142,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253964+0000","last_change":"2023-06-10T21:28:34.541690+0000","last_active":"2023-06-10T21:36:53.253964+0000","last_peered":"2023-06-10T21:36:53.253964+0000","last_clean":"2023-06-10T21:36:53.253964+0000","last_became_active":"2023-06-10T21:28:34.519157+0000","last_became_peered":"2023-06-10T21:28:34.519157+0000","last_unstale":"2023-06-10T21:36:53.253964+0000","last_undegraded":"2023-06-10T21:36:53.253964+0000","last_fullsized":"2023-06-10T21:36:53.253964+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:57:53.288134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"321'4331","reported_seq":4350,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:11.690481+0000","last_change":"2023-06-10T21:36:04.292180+0000","last_active":"2023-06-10T21:37:11.690481+0000","last_peered":"2023-06-10T21:37:11.690481+0000","last_clean":"2023-06-10T21:37:11.690481+0000","last_became_active":"2023-06-10T21:36:04.291901+0000","last_became_peered":"2023-06-10T21:36:04.291901+0000","last_unstale":"2023-06-10T21:37:11.690481+0000","last_undegraded":"2023-06-10T21:37:11.690481+0000","last_fullsized":"2023-06-10T21:37:11.690481+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4331,"log_dups_size":0,"ondisk_log_size":4331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:45.649615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283836416,"num_objects":4331,"num_object_clones":0,"num_object_copies":12993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4331,"num_write_kb":277184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":299,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042002+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:36:46.042002+0000","last_peered":"2023-06-10T21:36:46.042002+0000","last_clean":"2023-06-10T21:36:46.042002+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:36:46.042002+0000","last_undegraded":"2023-06-10T21:36:46.042002+0000","last_fullsized":"2023-06-10T21:36:46.042002+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":96,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041762+0000","last_change":"2023-06-10T21:28:57.893999+0000","last_active":"2023-06-10T21:36:46.041762+0000","last_peered":"2023-06-10T21:36:46.041762+0000","last_clean":"2023-06-10T21:36:46.041762+0000","last_became_active":"2023-06-10T21:28:57.893526+0000","last_became_peered":"2023-06-10T21:28:57.893526+0000","last_unstale":"2023-06-10T21:36:46.041762+0000","last_undegraded":"2023-06-10T21:36:46.041762+0000","last_fullsized":"2023-06-10T21:36:46.041762+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:42.705816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":281,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.745461+0000","last_change":"2023-06-10T21:32:55.516507+0000","last_active":"2023-06-10T21:36:56.745461+0000","last_peered":"2023-06-10T21:36:56.745461+0000","last_clean":"2023-06-10T21:36:56.745461+0000","last_became_active":"2023-06-10T21:32:55.516165+0000","last_became_peered":"2023-06-10T21:32:55.516165+0000","last_unstale":"2023-06-10T21:36:56.745461+0000","last_undegraded":"2023-06-10T21:36:56.745461+0000","last_fullsized":"2023-06-10T21:36:56.745461+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:29:45.314946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"321'4223","reported_seq":4242,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-06-10T21:37:09.516994+0000","last_change":"2023-06-10T21:36:04.296839+0000","last_active":"2023-06-10T21:37:09.516994+0000","last_peered":"2023-06-10T21:37:09.516994+0000","last_clean":"2023-06-10T21:37:09.516994+0000","last_became_active":"2023-06-10T21:36:04.294647+0000","last_became_peered":"2023-06-10T21:36:04.294647+0000","last_unstale":"2023-06-10T21:37:09.516994+0000","last_undegraded":"2023-06-10T21:37:09.516994+0000","last_fullsized":"2023-06-10T21:37:09.516994+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4223,"log_dups_size":0,"ondisk_log_size":4223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:00:16.069639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276758528,"num_objects":4223,"num_object_clones":0,"num_object_copies":12669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4223,"num_write_kb":270272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":42,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.044102+0000","last_change":"2023-06-10T21:32:55.517939+0000","last_active":"2023-06-10T21:36:46.044102+0000","last_peered":"2023-06-10T21:36:46.044102+0000","last_clean":"2023-06-10T21:36:46.044102+0000","last_became_active":"2023-06-10T21:32:55.517572+0000","last_became_peered":"2023-06-10T21:32:55.517572+0000","last_unstale":"2023-06-10T21:36:46.044102+0000","last_undegraded":"2023-06-10T21:36:46.044102+0000","last_fullsized":"2023-06-10T21:36:46.044102+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":149,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.040384+0000","last_change":"2023-06-10T21:28:34.502052+0000","last_active":"2023-06-10T21:36:46.040384+0000","last_peered":"2023-06-10T21:36:46.040384+0000","last_clean":"2023-06-10T21:36:46.040384+0000","last_became_active":"2023-06-10T21:28:34.501224+0000","last_became_peered":"2023-06-10T21:28:34.501224+0000","last_unstale":"2023-06-10T21:36:46.040384+0000","last_undegraded":"2023-06-10T21:36:46.040384+0000","last_fullsized":"2023-06-10T21:36:46.040384+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:50.895189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":278,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.253853+0000","last_change":"2023-06-10T21:32:55.514084+0000","last_active":"2023-06-10T21:36:53.253853+0000","last_peered":"2023-06-10T21:36:53.253853+0000","last_clean":"2023-06-10T21:36:53.253853+0000","last_became_active":"2023-06-10T21:32:55.513770+0000","last_became_peered":"2023-06-10T21:32:55.513770+0000","last_unstale":"2023-06-10T21:36:53.253853+0000","last_undegraded":"2023-06-10T21:36:53.253853+0000","last_fullsized":"2023-06-10T21:36:53.253853+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:42.603034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":145,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042220+0000","last_change":"2023-06-10T21:28:34.541485+0000","last_active":"2023-06-10T21:36:46.042220+0000","last_peered":"2023-06-10T21:36:46.042220+0000","last_clean":"2023-06-10T21:36:46.042220+0000","last_became_active":"2023-06-10T21:28:34.518226+0000","last_became_peered":"2023-06-10T21:28:34.518226+0000","last_unstale":"2023-06-10T21:36:46.042220+0000","last_undegraded":"2023-06-10T21:36:46.042220+0000","last_fullsized":"2023-06-10T21:36:46.042220+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:42:46.387688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":163,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.057020+0000","last_change":"2023-06-10T21:28:57.958830+0000","last_active":"2023-06-10T21:36:46.057020+0000","last_peered":"2023-06-10T21:36:46.057020+0000","last_clean":"2023-06-10T21:36:46.057020+0000","last_became_active":"2023-06-10T21:28:57.958490+0000","last_became_peered":"2023-06-10T21:28:57.958490+0000","last_unstale":"2023-06-10T21:36:46.057020+0000","last_undegraded":"2023-06-10T21:36:46.057020+0000","last_fullsized":"2023-06-10T21:36:46.057020+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:33:05.237928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":46,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.043933+0000","last_change":"2023-06-10T21:32:55.612337+0000","last_active":"2023-06-10T21:36:46.043933+0000","last_peered":"2023-06-10T21:36:46.043933+0000","last_clean":"2023-06-10T21:36:46.043933+0000","last_became_active":"2023-06-10T21:32:55.515715+0000","last_became_peered":"2023-06-10T21:32:55.515715+0000","last_unstale":"2023-06-10T21:36:46.043933+0000","last_undegraded":"2023-06-10T21:36:46.043933+0000","last_fullsized":"2023-06-10T21:36:46.043933+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":204,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042725+0000","last_change":"2023-06-10T21:31:34.297057+0000","last_active":"2023-06-10T21:36:46.042725+0000","last_peered":"2023-06-10T21:36:46.042725+0000","last_clean":"2023-06-10T21:36:46.042725+0000","last_became_active":"2023-06-10T21:31:34.296730+0000","last_became_peered":"2023-06-10T21:31:34.296730+0000","last_unstale":"2023-06-10T21:36:46.042725+0000","last_undegraded":"2023-06-10T21:36:46.042725+0000","last_fullsized":"2023-06-10T21:36:46.042725+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:43:23.161010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":100,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.041674+0000","last_change":"2023-06-10T21:28:57.893806+0000","last_active":"2023-06-10T21:36:46.041674+0000","last_peered":"2023-06-10T21:36:46.041674+0000","last_clean":"2023-06-10T21:36:46.041674+0000","last_became_active":"2023-06-10T21:28:57.893064+0000","last_became_peered":"2023-06-10T21:28:57.893064+0000","last_unstale":"2023-06-10T21:36:46.041674+0000","last_undegraded":"2023-06-10T21:36:46.041674+0000","last_fullsized":"2023-06-10T21:36:46.041674+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:24.009445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":260,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042610+0000","last_change":"2023-06-10T21:28:57.899696+0000","last_active":"2023-06-10T21:36:46.042610+0000","last_peered":"2023-06-10T21:36:46.042610+0000","last_clean":"2023-06-10T21:36:46.042610+0000","last_became_active":"2023-06-10T21:28:57.899337+0000","last_became_peered":"2023-06-10T21:28:57.899337+0000","last_unstale":"2023-06-10T21:36:46.042610+0000","last_undegraded":"2023-06-10T21:36:46.042610+0000","last_fullsized":"2023-06-10T21:36:46.042610+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:16:17.589543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":306,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:56.745680+0000","last_change":"2023-06-10T21:28:57.905576+0000","last_active":"2023-06-10T21:36:56.745680+0000","last_peered":"2023-06-10T21:36:56.745680+0000","last_clean":"2023-06-10T21:36:56.745680+0000","last_became_active":"2023-06-10T21:28:57.903892+0000","last_became_peered":"2023-06-10T21:28:57.903892+0000","last_unstale":"2023-06-10T21:36:56.745680+0000","last_undegraded":"2023-06-10T21:36:56.745680+0000","last_fullsized":"2023-06-10T21:36:56.745680+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:58:33.739585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":309,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.042266+0000","last_change":"2023-06-10T21:28:57.902047+0000","last_active":"2023-06-10T21:36:46.042266+0000","last_peered":"2023-06-10T21:36:46.042266+0000","last_clean":"2023-06-10T21:36:46.042266+0000","last_became_active":"2023-06-10T21:28:57.901661+0000","last_became_peered":"2023-06-10T21:28:57.901661+0000","last_unstale":"2023-06-10T21:36:46.042266+0000","last_undegraded":"2023-06-10T21:36:46.042266+0000","last_fullsized":"2023-06-10T21:36:46.042266+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:56:15.666002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":211,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:53.254104+0000","last_change":"2023-06-10T21:28:34.514540+0000","last_active":"2023-06-10T21:36:53.254104+0000","last_peered":"2023-06-10T21:36:53.254104+0000","last_clean":"2023-06-10T21:36:53.254104+0000","last_became_active":"2023-06-10T21:28:34.511348+0000","last_became_peered":"2023-06-10T21:28:34.511348+0000","last_unstale":"2023-06-10T21:36:53.254104+0000","last_undegraded":"2023-06-10T21:36:53.254104+0000","last_fullsized":"2023-06-10T21:36:53.254104+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:52.823425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":55,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-06-10T21:36:46.049862+0000","last_change":"2023-06-10T21:31:34.297443+0000","last_active":"2023-06-10T21:36:46.049862+0000","last_peered":"2023-06-10T21:36:46.049862+0000","last_clean":"2023-06-10T21:36:46.049862+0000","last_became_active":"2023-06-10T21:31:34.297127+0000","last_became_peered":"2023-06-10T21:31:34.297127+0000","last_unstale":"2023-06-10T21:36:46.049862+0000","last_undegraded":"2023-06-10T21:36:46.049862+0000","last_fullsized":"2023-06-10T21:36:46.049862+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:40:06.330382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4408672256,"num_objects":67271,"num_object_clones":0,"num_object_copies":201813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":67271,"num_write_kb":4305344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13231718400,"data_stored":13231718400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67271,"ondisk_log_size":67271,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":6258,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430681,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3552060,"kb_used_data":2906420,"kb_used_omap":0,"kb_used_meta":645632,"kb_avail":90193092,"statfs":{"total":95995035648,"available":92357726208,"internally_reserved":0,"allocated":2976174080,"data_stored":2975598769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":661127168},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":2.2050000000000001,"15min":6.8360000000000003},"min":{"1min":0.23999999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2629999999999999,"5min":52.100000000000001,"15min":184.83699999999999},"last":1.994},{"interface":"front","average":{"1min":1.0349999999999999,"5min":2.601,"15min":7.1120000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.223},"max":{"1min":6.7999999999999998,"5min":51.146999999999998,"15min":185.58000000000001},"last":0.874}]},{"osd":1,"last update":"Sat Jun 10 21:36:15 2023","interfaces":[{"interface":"back","average":{"1min":3.407,"5min":3.681,"15min":3.4830000000000001},"min":{"1min":0.27100000000000002,"5min":0.26300000000000001,"15min":0.21299999999999999},"max":{"1min":54.43,"5min":87.417000000000002,"15min":130.19399999999999},"last":1.754},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.1029999999999998,"15min":4.0469999999999997},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":54.564,"5min":87.438000000000002,"15min":212.20599999999999},"last":2.456}]},{"osd":2,"last update":"Sat Jun 10 21:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":5.5179999999999998,"15min":4.6100000000000003},"min":{"1min":0.31900000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.3170000000000002,"5min":172.33600000000001,"15min":172.33600000000001},"last":2.0070000000000001},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.7720000000000002,"15min":5.3330000000000002},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.29499999999999998},"max":{"1min":3.6949999999999998,"5min":172.185,"15min":212.13399999999999},"last":1.853}]},{"osd":3,"last update":"Sat Jun 10 21:36:14 2023","interfaces":[{"interface":"back","average":{"1min":11.154,"5min":6.0650000000000004,"15min":3.649},"min":{"1min":0.38100000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":147.00700000000001,"5min":147.00700000000001,"15min":147.00700000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":11.568,"5min":5.8860000000000001,"15min":3.165},"min":{"1min":0.41399999999999998,"5min":0.31,"15min":0.31},"max":{"1min":146.93899999999999,"5min":146.93899999999999,"15min":146.93899999999999},"last":2.0209999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39600000000000002,"15min":0.70399999999999996},"min":{"1min":0.27600000000000002,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":0.73599999999999999,"5min":1.5049999999999999,"15min":8.1419999999999995},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.44500000000000001,"15min":1.0149999999999999},"min":{"1min":0.29099999999999998,"5min":0.28299999999999997,"15min":0.245},"max":{"1min":0.89800000000000002,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.46700000000000003,"15min":1.768},"min":{"1min":0.29999999999999999,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":0.70899999999999996,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.496,"5min":0.46300000000000002,"15min":1.6040000000000001},"min":{"1min":0.33200000000000002,"5min":0.32500000000000001,"15min":0.29299999999999998},"max":{"1min":0.88800000000000001,"5min":0.92100000000000004,"15min":212.23699999999999},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52200000000000002,"15min":1.0469999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.27800000000000002},"max":{"1min":1.272,"5min":3.2559999999999998,"15min":13.073},"last":1.379},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.49099999999999999,"15min":0.92100000000000004},"min":{"1min":0.39600000000000002,"5min":0.33900000000000002,"15min":0.32700000000000001},"max":{"1min":0.80300000000000005,"5min":1.1879999999999999,"15min":10.874000000000001},"last":0.42099999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626909,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566352,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2808724,"kb_used_data":2103436,"kb_used_omap":4,"kb_used_meta":705275,"kb_avail":90936428,"statfs":{"total":95995035648,"available":93118902272,"internally_reserved":0,"allocated":2153918464,"data_stored":2152911670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":722202510},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:51 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":3.3279999999999998,"15min":9.4480000000000004},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.8980000000000001,"5min":70.254000000000005,"15min":154.31399999999999},"last":4.0359999999999996},{"interface":"front","average":{"1min":0.81200000000000006,"5min":3.028,"15min":9.1950000000000003},"min":{"1min":0.28499999999999998,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":2.9319999999999999,"5min":70.100999999999999,"15min":154.27199999999999},"last":0.215}]},{"osd":2,"last update":"Sat Jun 10 21:36:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":7.9550000000000001,"15min":5.7329999999999997},"min":{"1min":0.25900000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":5.4550000000000001,"5min":191.101,"15min":191.101},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":6.532,"15min":5.0650000000000004},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.222},"max":{"1min":2.5590000000000002,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.61499999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":7.6280000000000001,"5min":5.734,"15min":3.2719999999999998},"min":{"1min":0.34699999999999998,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":83.810000000000002,"5min":83.810000000000002,"15min":83.810000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":7.7050000000000001,"5min":5.8680000000000003,"15min":4.2050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29199999999999998,"15min":0.23999999999999999},"max":{"1min":92.757000000000005,"5min":92.757000000000005,"15min":92.757000000000005},"last":0.38}]},{"osd":4,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.24,"15min":1.5920000000000001},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":7.609,"5min":9.3659999999999997,"15min":16.059999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.5289999999999999,"15min":1.48},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.27400000000000002},"max":{"1min":5.5389999999999997,"5min":8.8629999999999995,"15min":14.460000000000001},"last":0.503}]},{"osd":5,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":2.1099999999999999,"15min":4.3200000000000003},"min":{"1min":0.434,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":11.959,"5min":11.959,"15min":383.83100000000002},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.2509999999999999,"15min":3.847},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":7.6429999999999998,"5min":9.3160000000000007,"15min":383.84800000000001},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3090000000000002,"5min":2.1469999999999998,"15min":2.226},"min":{"1min":0.40699999999999997,"5min":0.316,"15min":0.316},"max":{"1min":11.923,"5min":11.923,"15min":14.858000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.411,"5min":1.4019999999999999,"15min":1.772},"min":{"1min":0.45500000000000002,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":7.5700000000000003,"5min":9.3849999999999998,"15min":13.48},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:36:16 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.9470000000000001,"15min":2.7090000000000001},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.32600000000000001},"max":{"1min":7.4749999999999996,"5min":47.844000000000001,"15min":162.399},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.115,"5min":2.3490000000000002,"15min":2.5059999999999998},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34499999999999997},"max":{"1min":5.5650000000000004,"5min":47.753999999999998,"15min":162.27199999999999},"last":0.67900000000000005}]}]},{"osd":0,"up_from":244,"seq":1047972020328,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1887320,"kb_used_data":1587280,"kb_used_omap":10,"kb_used_meta":300021,"kb_avail":91857832,"statfs":{"total":95995035648,"available":94062419968,"internally_reserved":0,"allocated":1625374720,"data_stored":1624302774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":307221556},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":3.9969999999999999,"15min":4.1630000000000003},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":7.6299999999999999,"5min":139.68000000000001,"15min":139.68000000000001},"last":3.4340000000000002},{"interface":"front","average":{"1min":1.0269999999999999,"5min":4.3940000000000001,"15min":5.1260000000000003},"min":{"1min":0.20999999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.3949999999999996,"5min":141.79499999999999,"15min":141.79499999999999},"last":5.6369999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":7.2270000000000003,"15min":6.5590000000000002},"min":{"1min":0.27900000000000003,"5min":0.216,"15min":0.216},"max":{"1min":30.763000000000002,"5min":132.245,"15min":205.10900000000001},"last":3.2200000000000002},{"interface":"front","average":{"1min":2.9670000000000001,"5min":6.9489999999999998,"15min":6.1340000000000003},"min":{"1min":0.249,"5min":0.248,"15min":0.248},"max":{"1min":31.210999999999999,"5min":132.26599999999999,"15min":208.34299999999999},"last":10.273}]},{"osd":3,"last update":"Sat Jun 10 21:36:12 2023","interfaces":[{"interface":"back","average":{"1min":15.250999999999999,"5min":6.3860000000000001,"15min":6.1669999999999998},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":211.864,"5min":211.864,"15min":211.864},"last":5.032},{"interface":"front","average":{"1min":14.010999999999999,"5min":6.7969999999999997,"15min":6.9569999999999999},"min":{"1min":0.28399999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":207.517,"5min":207.517,"15min":207.517},"last":3.927}]},{"osd":4,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":2.206,"5min":2.2589999999999999,"15min":3.754},"min":{"1min":0.36399999999999999,"5min":0.317,"15min":0.27100000000000002},"max":{"1min":19.783999999999999,"5min":19.783999999999999,"15min":29.709},"last":5.157},{"interface":"front","average":{"1min":1.8640000000000001,"5min":2.3370000000000002,"15min":3.351},"min":{"1min":0.35899999999999999,"5min":0.308,"15min":0.308},"max":{"1min":19.762,"5min":24.222999999999999,"15min":24.222999999999999},"last":3.2650000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":2.1629999999999998,"15min":3.3540000000000001},"min":{"1min":0.36199999999999999,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":2.7080000000000002,"5min":24.288,"15min":24.288},"last":3.327},{"interface":"front","average":{"1min":1.085,"5min":1.875,"15min":2.855},"min":{"1min":0.40400000000000003,"5min":0.38500000000000001,"15min":0.35199999999999998},"max":{"1min":7.7560000000000002,"5min":8.0079999999999991,"15min":19.846},"last":3.5150000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1600000000000001,"15min":3.8450000000000002},"min":{"1min":0.43099999999999999,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":4.9199999999999999,"5min":10.771000000000001,"15min":29.806000000000001},"last":5.1900000000000004},{"interface":"front","average":{"1min":0.95499999999999996,"5min":2.286,"15min":3.4199999999999999},"min":{"1min":0.379,"5min":0.373,"15min":0.373},"max":{"1min":2.722,"5min":24.260999999999999,"15min":24.260999999999999},"last":3.294}]},{"osd":7,"last update":"Sat Jun 10 21:36:56 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":2.2370000000000001,"15min":3.23},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.32400000000000001},"max":{"1min":14.743,"5min":17.523,"15min":19.866},"last":3.476},{"interface":"front","average":{"1min":2.0710000000000002,"5min":2.4590000000000001,"15min":3.9449999999999998},"min":{"1min":0.41699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":14.994999999999999,"5min":17.792999999999999,"15min":29.77},"last":5.1180000000000003}]}]},{"osd":2,"up_from":120,"seq":515396075740,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3236876,"kb_used_data":2673668,"kb_used_omap":7,"kb_used_meta":563192,"kb_avail":90508276,"statfs":{"total":95995035648,"available":92680474624,"internally_reserved":0,"allocated":2737836032,"data_stored":2736836460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":576708689},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:47 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":3.819,"15min":4.4050000000000002},"min":{"1min":0.24199999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.85599999999999998,"5min":70.510999999999996,"15min":70.510999999999996},"last":5.1639999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":3.6520000000000001,"15min":4.8330000000000002},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":3.2490000000000001,"5min":67.828999999999994,"15min":67.828999999999994},"last":2.9180000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":5.1100000000000003,"15min":3.3239999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.19},"max":{"1min":7.4050000000000002,"5min":147.20099999999999,"15min":147.20099999999999},"last":2.2480000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":4.7240000000000002,"15min":2.988},"min":{"1min":0.28999999999999998,"5min":0.215,"15min":0.214},"max":{"1min":2.3279999999999998,"5min":147.81299999999999,"15min":147.81299999999999},"last":4.9390000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:36:11 2023","interfaces":[{"interface":"back","average":{"1min":2.161,"5min":6.0970000000000004,"15min":5.3760000000000003},"min":{"1min":0.34499999999999997,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":11.15,"5min":138.03299999999999,"15min":138.03299999999999},"last":5.008},{"interface":"front","average":{"1min":3.7360000000000002,"5min":6.3019999999999996,"15min":6.0659999999999998},"min":{"1min":0.29699999999999999,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":17.553999999999998,"5min":138.095,"15min":138.095},"last":2.7730000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.7210000000000001,"15min":2.996},"min":{"1min":0.35899999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":5.9470000000000001,"5min":7.3929999999999998,"15min":224.00999999999999},"last":2.895},{"interface":"front","average":{"1min":1.115,"5min":2.3839999999999999,"15min":3.3599999999999999},"min":{"1min":0.35999999999999999,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":5.7869999999999999,"5min":29.486999999999998,"15min":223.28399999999999},"last":2.2999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.6830000000000001,"15min":1.651},"min":{"1min":0.39300000000000002,"5min":0.379,"15min":0.29699999999999999},"max":{"1min":0.98499999999999999,"5min":10.554,"15min":14.180999999999999},"last":4.843},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.7,"15min":1.8979999999999999},"min":{"1min":0.35499999999999998,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":1.8580000000000001,"5min":7.3200000000000003,"15min":16.125},"last":2.8730000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.7769999999999999,"15min":1.966},"min":{"1min":0.43099999999999999,"5min":0.34200000000000003,"15min":0.34000000000000002},"max":{"1min":1.8859999999999999,"5min":7.4180000000000001,"15min":16.183},"last":2.839},{"interface":"front","average":{"1min":0.91800000000000004,"5min":2.4580000000000002,"15min":2.3620000000000001},"min":{"1min":0.40500000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.1819999999999999,"5min":29.538,"15min":29.538},"last":2.278}]},{"osd":7,"last update":"Sat Jun 10 21:36:41 2023","interfaces":[{"interface":"back","average":{"1min":16.013000000000002,"5min":6.5259999999999998,"15min":3.6400000000000001},"min":{"1min":0.48999999999999999,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":219.131,"5min":219.131,"15min":219.131},"last":4.8949999999999996},{"interface":"front","average":{"1min":16.175999999999998,"5min":6.4640000000000004,"15min":3.9630000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":219.19,"5min":219.19,"15min":219.19},"last":2.8159999999999998}]}]},{"osd":3,"up_from":250,"seq":1073741824100,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2929704,"kb_used_data":2569504,"kb_used_omap":7,"kb_used_meta":360184,"kb_avail":90815448,"statfs":{"total":95995035648,"available":92995018752,"internally_reserved":0,"allocated":2631172096,"data_stored":2630086245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":368828435},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":4.4569999999999999,"15min":4.9820000000000002},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.1890000000000001,"5min":105.09,"15min":105.09},"last":4.5750000000000002},{"interface":"front","average":{"1min":1.472,"5min":4.4160000000000004,"15min":4.7969999999999997},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.216},"max":{"1min":9.6699999999999999,"5min":99.852999999999994,"15min":99.852999999999994},"last":4.4100000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8480000000000001,"5min":4.766,"15min":4.0380000000000003},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":10.419,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":4.6639999999999997,"15min":3.7320000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.9629999999999992,"5min":94.691000000000003,"15min":94.691000000000003},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":3.7690000000000001,"15min":9.1449999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.7480000000000002,"5min":100.676,"15min":100.676},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.1850000000000001,"5min":5.2400000000000002,"15min":10.669},"min":{"1min":0.30399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.835000000000001,"5min":103.032,"15min":103.032},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":1.446,"15min":1.405},"min":{"1min":0.33700000000000002,"5min":0.30199999999999999,"15min":0.29199999999999998},"max":{"1min":3.7770000000000001,"5min":13.417,"15min":13.417},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.244,"5min":2.3780000000000001,"15min":2.6680000000000001},"min":{"1min":0.32300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.2030000000000003,"5min":14.384,"15min":19.728000000000002},"last":0.67700000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.5169999999999999,"15min":1.397},"min":{"1min":0.377,"5min":0.35699999999999998,"15min":0.32400000000000001},"max":{"1min":1.4079999999999999,"5min":9.7889999999999997,"15min":9.7889999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.6060000000000001,"15min":1.4870000000000001},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":3.7999999999999998,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":2.3919999999999999,"15min":2.6899999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.1740000000000004,"5min":14.269,"15min":19.805},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":1.552,"15min":1.4470000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.39100000000000001},"max":{"1min":1.4379999999999999,"5min":9.7289999999999992,"15min":9.7289999999999992},"last":0.74099999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:36:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":3.0760000000000001,"15min":2.9260000000000002},"min":{"1min":0.40000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.1449999999999996,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.72999999999999998,"5min":2.2360000000000002,"15min":1.6579999999999999},"min":{"1min":0.44,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":1.6899999999999999,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.77100000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019365,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1954932,"kb_used_data":1327596,"kb_used_omap":0,"kb_used_meta":627328,"kb_avail":91790220,"statfs":{"total":95995035648,"available":93993185280,"internally_reserved":0,"allocated":1359458304,"data_stored":1358889633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":642383872},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:36:46 2023","interfaces":[{"interface":"back","average":{"1min":5.8570000000000002,"5min":5.1840000000000002,"15min":7.5780000000000003},"min":{"1min":0.26300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":115.831,"5min":234.203,"15min":234.203},"last":0.64800000000000002},{"interface":"front","average":{"1min":6.157,"5min":5.4509999999999996,"15min":7.5700000000000003},"min":{"1min":0.32600000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":117.111,"5min":234.19,"15min":234.19},"last":0.57599999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:36:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":3.0179999999999998,"15min":3.48},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.125,"5min":88.700999999999993,"15min":182.035},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":3.165,"15min":3.3849999999999998},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":3.641,"5min":87.853999999999999,"15min":181.73099999999999},"last":1.9199999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:36:34 2023","interfaces":[{"interface":"back","average":{"1min":5.7279999999999998,"5min":8.4760000000000009,"15min":4.1020000000000003},"min":{"1min":0.35699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":79.097999999999999,"5min":255.386,"15min":255.386},"last":3.504},{"interface":"front","average":{"1min":5.593,"5min":8.3559999999999999,"15min":4.016},"min":{"1min":0.34100000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":79.808999999999997,"5min":255.29599999999999,"15min":255.29599999999999},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3959999999999999,"5min":8.0939999999999994,"15min":4.2489999999999997},"min":{"1min":0.35199999999999998,"5min":0.28499999999999998,"15min":0.28399999999999997},"max":{"1min":36.366999999999997,"5min":234.91900000000001,"15min":234.91900000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":3.6619999999999999,"5min":8.4410000000000007,"15min":4.1900000000000004},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.28499999999999998},"max":{"1min":38.659999999999997,"5min":235.05099999999999,"15min":235.05099999999999},"last":1.224}]},{"osd":5,"last update":"Sat Jun 10 21:36:34 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.38500000000000001,"15min":0.98399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.23999999999999999},"max":{"1min":0.57799999999999996,"5min":0.86299999999999999,"15min":13.941000000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.46999999999999997,"15min":0.997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.247},"max":{"1min":0.95799999999999996,"5min":5.25,"15min":11.904999999999999},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42899999999999999,"15min":1.0149999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25600000000000001},"max":{"1min":0.67700000000000005,"5min":1.7210000000000001,"15min":8.2599999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46600000000000003,"15min":1.079},"min":{"1min":0.35499999999999998,"5min":0.34699999999999998,"15min":0.309},"max":{"1min":0.63400000000000001,"5min":1.71,"15min":13.845000000000001},"last":0.60899999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.77900000000000003,"15min":1.2669999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":0.65300000000000002,"5min":39.713999999999999,"15min":39.713999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.79200000000000004,"15min":1.21},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.28000000000000003},"max":{"1min":0.68500000000000005,"5min":37.429000000000002,"15min":37.429000000000002},"last":0.45200000000000001}]}]},{"osd":5,"up_from":0,"seq":154618823137,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1636499456,"data_stored":1636499456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1955004416,"data_stored":1955004416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2750676992,"data_stored":2750676992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2542993408,"data_stored":2542993408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1357774848,"data_stored":1357774848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2988769280,"data_stored":2988769280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:37:15.794 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:37:15.794 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:37:15.794 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-06-10T21:37:15.794 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:37:15.795 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:37:15.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-06-10T21:37:15.818 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:15.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:15.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:15.963 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:16.021 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:16.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:16.127 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:16.156 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:16 smithi152 ceph-mon[156819]: pgmap v1199: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s 2023-06-10T21:37:16.159 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:16.217 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:16.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:16.225 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:37:16.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:16.358 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:16.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:16 smithi064 ceph-mon[186706]: pgmap v1199: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s 2023-06-10T21:37:16.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:16 smithi064 ceph-mon[180297]: pgmap v1199: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s 2023-06-10T21:37:16.447 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:16.520 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:16.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:16.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:16.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:16.660 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:16.716 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:16.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:16.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 73142 73126 72.4998 56 0.0241801 0.0137872 2023-06-10T21:37:16.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:16.917 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:17.446 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:17.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:17.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:17.587 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:17.628 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:17.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:17.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:17.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:17.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 73927 73911 72.1337 49.0625 0.022618 0.013858 2023-06-10T21:37:17.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:37:17.833 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:18.044 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:18.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:18.152 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:18.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:18 smithi064 ceph-mon[180297]: pgmap v1200: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s 2023-06-10T21:37:18.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:18 smithi064 ceph-mon[186706]: pgmap v1200: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s 2023-06-10T21:37:18.230 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:18.283 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:18 smithi152 ceph-mon[156819]: pgmap v1200: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 818 op/s 2023-06-10T21:37:18.484 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:37:18.484 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "" 2023-06-10T21:37:18.484 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:37:18.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 74608 74592 71.679 42.5625 0.00644759 0.0139425 2023-06-10T21:37:19.025 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:19.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:19.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:19.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:19.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:19.328 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:19.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:19.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:19.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:19.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:19.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:19.449 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:37:19.482 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:19.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:37:19.527 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:37:19.534 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:19.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:19.587 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:37:19.633 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:37:19.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:19.675 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:37:19.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:37:19.727 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:19.742 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:19.769 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:37:19.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 75331 75315 71.2778 45.1875 0.0229412 0.0140227 2023-06-10T21:37:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.787 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.788 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:37:19.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:19.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:19.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:19.991 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:20.002 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:20.002 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:37:20.002 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:37:20.002 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:37:20.002 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:20.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1281124289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:20.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:20 smithi064 ceph-mon[180297]: pgmap v1201: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:37:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1281124289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:20.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:20 smithi064 ceph-mon[186706]: pgmap v1201: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:37:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1281124289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:20 smithi152 ceph-mon[156819]: pgmap v1201: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 822 op/s 2023-06-10T21:37:20.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:20.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:20.773 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 76152 76136 70.9798 51.3125 0.00747369 0.014071 2023-06-10T21:37:20.788 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:20.816 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:21.774 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 76814 76798 70.5442 41.375 0.0210238 0.0141708 2023-06-10T21:37:21.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:37:21.822 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:21.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:21.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:21.960 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:22.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:22.122 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:22.174 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:22 smithi064 ceph-mon[180297]: pgmap v1202: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:37:22.174 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:22 smithi064 ceph-mon[186706]: pgmap v1202: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:37:22.202 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:22.254 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:22.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:22.262 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:22 smithi152 ceph-mon[156819]: pgmap v1202: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 51 MiB/s wr, 814 op/s 2023-06-10T21:37:22.363 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:22.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:22.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:22.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:22.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:22.656 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:22.724 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:22.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:22.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:22.776 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 77480 77464 70.1233 41.625 0.0145713 0.0142557 2023-06-10T21:37:22.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:22.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:23.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:23.083 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:22] "GET /metrics HTTP/1.1" 200 224098 "" "Prometheus/2.33.4" 2023-06-10T21:37:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:23.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:23.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:23.219 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:23.280 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:23.505 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:23.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:23.620 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:23.654 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:23.703 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:23.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:23.776 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 78319 78303 69.8705 52.4375 0.0182294 0.0143063 2023-06-10T21:37:23.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:37:23.808 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:23.922 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:24.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:24 smithi064 ceph-mon[180297]: pgmap v1203: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-06-10T21:37:24.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:24 smithi064 ceph-mon[186706]: pgmap v1203: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-06-10T21:37:24.192 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:24.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:24.219 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 7, 0, 2] out_osds: [5, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:37:24.220 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:37:24.220 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [1, 3, 4, 7, 0, 2] 2023-06-10T21:37:24.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 4 2023-06-10T21:37:24.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:24.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:24.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:24 smithi152 ceph-mon[156819]: pgmap v1203: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 54 MiB/s wr, 866 op/s 2023-06-10T21:37:24.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 78851 78835 69.3543 33.25 0.0136494 0.0144109 2023-06-10T21:37:25.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:25.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:25.116 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:37:25.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:25.190 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:25.237 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:25.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:25.311 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3304261268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.311 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:25 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3304261268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:25 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.345 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:25.443 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3304261268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:25 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:25.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:25.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:25.669 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:25.699 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:25.750 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:25.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:25.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 79652 79636 69.0865 50.0625 0.0163039 0.0144614 2023-06-10T21:37:25.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:25.899 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:26.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:26.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:37:26.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:37:26.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: osdmap e323: 8 total, 8 up, 6 in 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.142 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[186706]: pgmap v1205: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:37:26.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:37:26.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:37:26.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:26.143 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: osdmap e323: 8 total, 8 up, 6 in 2023-06-10T21:37:26.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.144 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:26 smithi064 ceph-mon[180297]: pgmap v1205: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:37:26.147 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:26.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:26.204 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:26.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:26.279 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:26.328 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:26.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:37:26.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:37:26.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:26.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: osdmap e323: 8 total, 8 up, 6 in 2023-06-10T21:37:26.334 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: failed to encode map e323 with expected crc 2023-06-10T21:37:26.335 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:26 smithi152 ceph-mon[156819]: pgmap v1205: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-10T21:37:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:26.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:26.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:26.466 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:26.516 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:26.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:26.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:26.689 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:26.689 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24681 11.5 11:adf48b79:::benchmark_data_smithi064_385725_object24680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:37:26.690 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.56176079900001, 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28889557700000001, 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.691 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24681 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115709+0000", 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115708+0000", 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115733+0000", 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5386000000000001e-05 2023-06-10T21:37:26.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115742+0000", 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0780000000000002e-06 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115749+0000", 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4110000000000001e-06 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115922+0000", 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000173318 2023-06-10T21:37:26.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116070+0000", 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014733500000000001 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116228+0000", 2023-06-10T21:37:26.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000158376 2023-06-10T21:37:26.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120319+0000", 2023-06-10T21:37:26.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040913800000000004 2023-06-10T21:37:26.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324910+0000", 2023-06-10T21:37:26.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.204591211 2023-06-10T21:37:26.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404530+0000", 2023-06-10T21:37:26.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079619326000000004 2023-06-10T21:37:26.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404570+0000", 2023-06-10T21:37:26.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9895999999999999e-05 2023-06-10T21:37:26.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404605+0000", 2023-06-10T21:37:26.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5114999999999998e-05 2023-06-10T21:37:26.705 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.706 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.706 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.707 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24682 11.a 11:59387607:::benchmark_data_smithi064_385725_object24681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.707 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:37:26.707 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.56171563300001, 2023-06-10T21:37:26.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28861932000000001, 2023-06-10T21:37:26.708 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.708 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.709 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24682 2023-06-10T21:37:26.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.710 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:37:26.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:37:26.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115754+0000", 2023-06-10T21:37:26.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115775+0000", 2023-06-10T21:37:26.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1480000000000001e-05 2023-06-10T21:37:26.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115782+0000", 2023-06-10T21:37:26.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5980000000000002e-06 2023-06-10T21:37:26.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115787+0000", 2023-06-10T21:37:26.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2730000000000002e-06 2023-06-10T21:37:26.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115977+0000", 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019004300000000001 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116096+0000", 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119048 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116239+0000", 2023-06-10T21:37:26.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014270900000000001 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117796+0000", 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001557014 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.121220+0000", 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034239399999999999 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404279+0000", 2023-06-10T21:37:26.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.283058645 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404326+0000", 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7493000000000001e-05 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404374+0000", 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7534000000000003e-05 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.724 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24686 11.5 11:aad12278:::benchmark_data_smithi064_385725_object24685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.56092242800003, 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.288163171, 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.725 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24686 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:37:26.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116548+0000", 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116547+0000", 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116571+0000", 2023-06-10T21:37:26.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4745000000000001e-05 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116584+0000", 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2991e-05 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116593+0000", 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2369999999999992e-06 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117598+0000", 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010048850000000001 2023-06-10T21:37:26.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117717+0000", 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119888 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117865+0000", 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014747999999999999 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120325+0000", 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002459934 2023-06-10T21:37:26.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325488+0000", 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20516282499999999 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404638+0000", 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079150140999999993 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404675+0000", 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7067000000000003e-05 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404711+0000", 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5892000000000001e-05 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24687 11.b 11:d00b5318:::benchmark_data_smithi064_385725_object24686:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.56086892600001, 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28619599200000001, 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.736 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24687 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116601+0000", 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116622+0000", 2023-06-10T21:37:26.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1503999999999999e-05 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116631+0000", 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6300000000000004e-06 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116637+0000", 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4819999999999999e-06 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116750+0000", 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011303600000000001 2023-06-10T21:37:26.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116884+0000", 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013408300000000001 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117022+0000", 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013734700000000001 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119935+0000", 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029137130000000001 2023-06-10T21:37:26.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324876+0000", 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20494073099999999 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402697+0000", 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077820749999999994 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402754+0000", 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6580999999999999e-05 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402797+0000", 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3612999999999997e-05 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24688 11.5 11:aaadba38:::benchmark_data_smithi064_385725_object24687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.55871659600001, 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.286064453, 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24688 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118753+0000", 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118752+0000", 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118776+0000", 2023-06-10T21:37:26.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4070000000000002e-05 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118794+0000", 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7810000000000001e-05 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118804+0000", 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0053e-05 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118981+0000", 2023-06-10T21:37:26.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000177503 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119111+0000", 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129488 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.119301+0000", 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018976 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120860+0000", 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015592309999999999 2023-06-10T21:37:26.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325669+0000", 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20480867 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404750+0000", 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.079081311000000001 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404787+0000", 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7070999999999998e-05 2023-06-10T21:37:26.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.404818+0000", 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0985000000000002e-05 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24693 11.9 11:93e9483b:::benchmark_data_smithi064_385725_object24692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.55709505700003, 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.282496517, 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.758 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24693 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120375+0000", 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120374+0000", 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120401+0000", 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7317e-05 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120419+0000", 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7456999999999999e-05 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120427+0000", 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7839999999999992e-06 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120485+0000", 2023-06-10T21:37:26.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7590999999999998e-05 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120609+0000", 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012447700000000001 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120750+0000", 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140925 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.122064+0000", 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001313374 2023-06-10T21:37:26.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.325118+0000", 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.203053815 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402795+0000", 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077677874999999993 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402838+0000", 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2296000000000003e-05 2023-06-10T21:37:26.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402872+0000", 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3819999999999998e-05 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:24695 11.b 11:d780a7ce:::benchmark_data_smithi064_385725_object24694:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e279)", 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 349.55702734699997, 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28245634200000003, 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 24695 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120443+0000", 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120442+0000", 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120465+0000", 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3102e-05 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120474+0000", 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3240000000000003e-06 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120480+0000", 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6449999999999999e-06 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120523+0000", 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2395000000000003e-05 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120636+0000", 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112953 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.120751+0000", 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011537200000000001 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.121926+0000", 2023-06-10T21:37:26.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001175155 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.324953+0000", 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20302672799999999 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402834+0000", 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.077880854999999999 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402870+0000", 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6094999999999999e-05 2023-06-10T21:37:26.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402899+0000", 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9294000000000001e-05 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.779 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99515 11.1 11:8ea0c8ed:::benchmark_data_smithi064_385725_object15242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 244.47400695900001, 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30862035399999999, 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.780 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99515 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:37:26.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203461+0000", 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:37:26.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203469+0000", 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203482+0000", 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3318e-05 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203491+0000", 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7749999999999998e-06 2023-06-10T21:37:26.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203533+0000", 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2716999999999998e-05 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203662+0000", 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012818000000000001 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203771+0000", 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109725 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.213580+0000", 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098082800000000008 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.218941+0000", 2023-06-10T21:37:26.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053615360000000001 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.511973+0000", 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29303174399999998 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512034+0000", 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1147000000000003e-05 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512083+0000", 2023-06-10T21:37:26.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9351000000000002e-05 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99523 11.1 11:88f160b1:::benchmark_data_smithi064_385725_object15250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:37:26.791 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 244.46221877799999, 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.311404917, 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99523 2023-06-10T21:37:26.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215250+0000", 2023-06-10T21:37:26.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215255+0000", 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0019999999999997e-06 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215263+0000", 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1470000000000001e-06 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215270+0000", 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4099999999999996e-06 2023-06-10T21:37:26.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215321+0000", 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1413999999999999e-05 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215475+0000", 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153478 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215610+0000", 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135716 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.219102+0000", 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034921179999999998 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.220787+0000", 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001685099 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526574+0000", 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30578659800000002 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526618+0000", 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4140000000000001e-05 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526656+0000", 2023-06-10T21:37:26.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8003999999999997e-05 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99530 11.1 11:816041a6:::benchmark_data_smithi064_385725_object15257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:37:26.802 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 244.45174698, 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30104990300000001, 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.803 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99530 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225722+0000", 2023-06-10T21:37:26.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225728+0000", 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9259999999999997e-06 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225741+0000", 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2894999999999999e-05 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225749+0000", 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9209999999999992e-06 2023-06-10T21:37:26.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225790+0000", 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1196999999999999e-05 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225895+0000", 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010484699999999999 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.226005+0000", 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110655 2023-06-10T21:37:26.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229259+0000", 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032538469999999998 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229476+0000", 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021646299999999999 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526694+0000", 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29721868499999998 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526739+0000", 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4938999999999998e-05 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526773+0000", 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3716e-05 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101022 11.b 11:d995299d:::benchmark_data_smithi064_385725_object16749:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:37:26.813 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 241.437124221, 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29097264099999998, 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.814 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101022 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240346+0000", 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240344+0000", 2023-06-10T21:37:26.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240352+0000", 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5129999999999999e-06 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240369+0000", 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7329e-05 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240378+0000", 2023-06-10T21:37:26.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8529999999999998e-06 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240590+0000", 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000211561 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240700+0000", 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110667 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.240808+0000", 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010772000000000001 2023-06-10T21:37:26.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.251610+0000", 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010801638000000001 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.255480+0000", 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038706489999999999 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531216+0000", 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27573550899999999 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531274+0000", 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8594000000000002e-05 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531318+0000", 2023-06-10T21:37:26.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4014999999999998e-05 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101024 11.9 11:99976c21:::benchmark_data_smithi064_385725_object16751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:37:26.824 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 241.43059167600001, 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.284438739, 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.825 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101024 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246878+0000", 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246877+0000", 2023-06-10T21:37:26.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246883+0000", 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4069999999999998e-06 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246896+0000", 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459e-05 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246903+0000", 2023-06-10T21:37:26.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4580000000000004e-06 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.248757+0000", 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00185384 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.248899+0000", 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141674 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.249016+0000", 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117834 2023-06-10T21:37:26.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.255313+0000", 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062968360000000001 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.261888+0000", 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065752049999999998 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531237+0000", 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.26934889699999998 2023-06-10T21:37:26.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531279+0000", 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1313999999999998e-05 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531317+0000", 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8510000000000002e-05 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101027 11.1 11:84832f10:::benchmark_data_smithi064_385725_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:37:26.835 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 241.43055306799999, 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29481494899999999, 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.836 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101027 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:37:26.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246919+0000", 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.565e-06 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246925+0000", 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7769999999999999e-06 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246930+0000", 2023-06-10T21:37:26.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6949999999999999e-06 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252174+0000", 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052442779999999998 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252278+0000", 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010410499999999999 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252344+0000", 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6168999999999996e-05 2023-06-10T21:37:26.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.256536+0000", 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041921479999999997 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531254+0000", 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27471757699999999 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541651+0000", 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010397027 2023-06-10T21:37:26.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541697+0000", 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5889000000000001e-05 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541732+0000", 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5098999999999997e-05 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101036 11.9 11:91c4f911:::benchmark_data_smithi064_385725_object16763:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:37:26.846 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 241.41468729900001, 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27844923399999999, 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.847 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101036 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262783+0000", 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262781+0000", 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262788+0000", 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4790000000000001e-06 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262804+0000", 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6654999999999999e-05 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262812+0000", 2023-06-10T21:37:26.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6799999999999993e-06 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.262862+0000", 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9917000000000003e-05 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.263004+0000", 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014181600000000001 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.263145+0000", 2023-06-10T21:37:26.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140937 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.266594+0000", 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003449662 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.266716+0000", 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012186800000000001 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541154+0000", 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27443805900000001 2023-06-10T21:37:26.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541196+0000", 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2249999999999997e-05 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541232+0000", 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5506000000000002e-05 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126745 11.1 11:8766de5a:::benchmark_data_smithi064_385725_object42472:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:37:26.857 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.25388398499999, 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33496997699999997, 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126745 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423584+0000", 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423595+0000", 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1586000000000001e-05 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423618+0000", 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2609999999999999e-05 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423630+0000", 2023-06-10T21:37:26.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2449000000000001e-05 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423686+0000", 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5452999999999999e-05 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423832+0000", 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146155 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423960+0000", 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012785100000000001 2023-06-10T21:37:26.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.434684+0000", 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01072394 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625142+0000", 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19045854700000001 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758437+0000", 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13329444800000001 2023-06-10T21:37:26.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758501+0000", 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3959000000000002e-05 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758556+0000", 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5461000000000003e-05 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126747 11.1 11:8706db2c:::benchmark_data_smithi064_385725_object42474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:37:26.868 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.25382495100001, 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33505084800000001, 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.869 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126747 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423648+0000", 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1719999999999999e-06 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423654+0000", 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8340000000000004e-06 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423660+0000", 2023-06-10T21:37:26.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7059999999999998e-06 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437557+0000", 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013896382000000001 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437673+0000", 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116501 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437787+0000", 2023-06-10T21:37:26.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113354 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625022+0000", 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18723535299999999 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625168+0000", 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146494 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758602+0000", 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13343400399999999 2023-06-10T21:37:26.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758650+0000", 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7052000000000003e-05 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758696+0000", 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6424000000000001e-05 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.879 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126748 11.8 11:1c2ac95f:::benchmark_data_smithi064_385725_object42475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.253786983, 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32430128899999999, 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.880 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126748 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423682+0000", 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423688+0000", 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6160000000000001e-06 2023-06-10T21:37:26.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423699+0000", 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1623e-05 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423707+0000", 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8830000000000006e-06 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423759+0000", 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1573000000000002e-05 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423869+0000", 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109951 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423973+0000", 2023-06-10T21:37:26.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104144 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437147+0000", 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013174289 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.624707+0000", 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.187559904 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747900+0000", 2023-06-10T21:37:26.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12319322200000001 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747947+0000", 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6390000000000001e-05 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747984+0000", 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7703999999999997e-05 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.890 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126751 11.1 11:8ace7e90:::benchmark_data_smithi064_385725_object42478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.238344267, 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31970830500000003, 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.891 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126751 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439125+0000", 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439130+0000", 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5249999999999996e-06 2023-06-10T21:37:26.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439144+0000", 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3719e-05 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439152+0000", 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2449999999999998e-06 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625233+0000", 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.186081301 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625322+0000", 2023-06-10T21:37:26.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8226999999999994e-05 2023-06-10T21:37:26.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625417+0000", 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5429999999999999e-05 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631580+0000", 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061630039999999997 2023-06-10T21:37:26.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631643+0000", 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3408e-05 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758744+0000", 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.127100451 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758794+0000", 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0526999999999999e-05 2023-06-10T21:37:26.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758834+0000", 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9748e-05 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126756 11.1 11:81432d08:::benchmark_data_smithi064_385725_object42483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 192.23830635300001, 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.328752612, 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.905 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126756 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439163+0000", 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439166+0000", 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1990000000000002e-06 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439174+0000", 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439180+0000", 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8000000000000004e-06 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627884+0000", 2023-06-10T21:37:26.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.188704802 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627970+0000", 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5963000000000003e-05 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.628045+0000", 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4239000000000005e-05 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631687+0000", 2023-06-10T21:37:26.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036425799999999999 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631697+0000", 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0166999999999999e-05 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767840+0000", 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13614217200000001 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767882+0000", 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2262000000000003e-05 2023-06-10T21:37:26.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767916+0000", 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4548000000000002e-05 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:138782 11.8 11:10cd1588:::benchmark_data_smithi064_385725_object54509:head [delete] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 167.748330036, 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.288210731, 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:26.916 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 138782 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:26.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929140+0000", 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:26.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929142+0000", 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.667e-06 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929148+0000", 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4040000000000003e-06 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.929153+0000", 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0119999999999996e-06 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:26.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934464+0000", 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053112979999999999 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934571+0000", 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106839 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.934679+0000", 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108268 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:38.941381+0000", 2023-06-10T21:37:26.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0067020270000000002 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.189309+0000", 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24792794200000001 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217251+0000", 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027941506000000001 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217304+0000", 2023-06-10T21:37:26.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3751000000000001e-05 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:39.217351+0000", 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6397000000000003e-05 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.927 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:26.927 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:26.927 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:26.927 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:26.928 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 80367 80351 68.7523 44.6875 0.018217 0.0145403 2023-06-10T21:37:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:26.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:26.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:27.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:27.057 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:27.105 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:27.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:27.216 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:27.249 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:27.301 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:27.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:27.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:27.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:27.438 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:27.497 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:27.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:27.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:27.608 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:27.641 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:27.689 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:27.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:27.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 81264 81248 68.5809 56.0625 0.0197295 0.0145765 2023-06-10T21:37:27.798 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:27.884 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:28.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:28 smithi064 ceph-mon[180297]: pgmap v1206: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 756 op/s 2023-06-10T21:37:28.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:28 smithi064 ceph-mon[186706]: pgmap v1206: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 756 op/s 2023-06-10T21:37:28.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:37:28.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:28 smithi152 ceph-mon[156819]: pgmap v1206: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 47 MiB/s wr, 756 op/s 2023-06-10T21:37:28.777 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 81920 81904 68.2132 41 0.0470712 0.0146457 2023-06-10T21:37:28.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:37:29.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:29.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:29.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:29.647 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:29.755 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:29.781 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 82519 82503 67.8054 37.4375 0.0256073 0.0147411 2023-06-10T21:37:29.858 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:29.876 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:29.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:29.984 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:30.013 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:30.065 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:30.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:30.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:30 smithi064 ceph-mon[180297]: pgmap v1207: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 732 op/s 2023-06-10T21:37:30.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:30 smithi064 ceph-mon[186706]: pgmap v1207: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 732 op/s 2023-06-10T21:37:30.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:30.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:30.389 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:30.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:30.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:30 smithi152 ceph-mon[156819]: pgmap v1207: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 732 op/s 2023-06-10T21:37:30.454 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:30.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:30.526 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:30.578 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:30.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:30.625 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:37:30.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:30.690 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:37:30.715 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:30.742 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:37:30.764 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:30.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:30.783 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 83228 83212 67.4999 44.3125 0.0174352 0.0148082 2023-06-10T21:37:30.788 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:37:30.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:37:30.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:37:30.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:30.901 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:30.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:37:30.946 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:37:30.952 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:30.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:30.978 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.979 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:37:30.980 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:37:31.057 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:31.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:31.091 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3585776277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:31.091 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/685845621' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.091 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.091 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[186706]: Client client.admin marked osd.4 out, while it was still marked up 2023-06-10T21:37:31.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:31 smithi064 conmon[180272]: 2023-06-10T21:37:31.003+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e323 definitely_dead 0 2023-06-10T21:37:31.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3585776277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:31.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/685845621' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:31 smithi064 ceph-mon[180297]: Client client.admin marked osd.4 out, while it was still marked up 2023-06-10T21:37:31.142 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:31.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:31.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3585776277' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/685845621' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:31 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-06-10T21:37:31.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:31 smithi152 ceph-mon[156819]: Client client.admin marked osd.4 out, while it was still marked up 2023-06-10T21:37:31.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:31.284 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:31.342 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:31.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:31.446 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:31.538 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:31.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 84060 84044 67.3012 52 0.0108787 0.0148538 2023-06-10T21:37:32.023 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.4. 2023-06-10T21:37:32.320 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[186706]: pgmap v1208: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 728 op/s 2023-06-10T21:37:32.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-06-10T21:37:32.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[186706]: osdmap e324: 8 total, 8 up, 5 in 2023-06-10T21:37:32.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[180297]: pgmap v1208: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 728 op/s 2023-06-10T21:37:32.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-06-10T21:37:32.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:32 smithi064 ceph-mon[180297]: osdmap e324: 8 total, 8 up, 5 in 2023-06-10T21:37:32.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:32 smithi152 ceph-mon[156819]: pgmap v1208: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 46 MiB/s wr, 728 op/s 2023-06-10T21:37:32.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:32 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-06-10T21:37:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:32 smithi152 ceph-mon[156819]: osdmap e324: 8 total, 8 up, 5 in 2023-06-10T21:37:32.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 84299 84283 66.6387 14.9375 0.00320636 0.0148725 2023-06-10T21:37:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:32] "GET /metrics HTTP/1.1" 200 224098 "" "Prometheus/2.33.4" 2023-06-10T21:37:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[180297]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[180297]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[180297]: osdmap e325: 8 total, 8 up, 5 in 2023-06-10T21:37:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[186706]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[186706]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:33 smithi064 ceph-mon[186706]: osdmap e325: 8 total, 8 up, 5 in 2023-06-10T21:37:33.411 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:33.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:33.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:33 smithi152 ceph-mon[156819]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:33 smithi152 ceph-mon[156819]: failed to encode map e324 with expected crc 2023-06-10T21:37:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:33 smithi152 ceph-mon[156819]: osdmap e325: 8 total, 8 up, 5 in 2023-06-10T21:37:33.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:33.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:33.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:33.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:33.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:33.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:37:33.780251+0000 min lat: 0.00116179 max lat: 0.288432 avg lat: 0.0148725 2023-06-10T21:37:33.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:37:33.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 84299 84283 65.8061 0 - 0.0148725 2023-06-10T21:37:33.848 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:33.875 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:33.916 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:33.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:33.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:34.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:34.049 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:34.094 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:34.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:34.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:34.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:34.194 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: pgmap v1211: 89 pgs: 1 remapped+peering, 88 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 685 B/s, 1 keys/s, 1 objects/s recovering 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: failed to encode map e326 with expected crc 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[186706]: osdmap e326: 8 total, 8 up, 5 in 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.195 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: pgmap v1211: 89 pgs: 1 remapped+peering, 88 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 685 B/s, 1 keys/s, 1 objects/s recovering 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: failed to encode map e326 with expected crc 2023-06-10T21:37:34.196 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:34 smithi064 ceph-mon[180297]: osdmap e326: 8 total, 8 up, 5 in 2023-06-10T21:37:34.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:34.239 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:34.249 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:37:34.249 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:37:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: failed to encode map e325 with expected crc 2023-06-10T21:37:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: pgmap v1211: 89 pgs: 1 remapped+peering, 88 active+clean; 5.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 45 MiB/s wr, 714 op/s; 685 B/s, 1 keys/s, 1 objects/s recovering 2023-06-10T21:37:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: failed to encode map e326 with expected crc 2023-06-10T21:37:34.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:34 smithi152 ceph-mon[156819]: osdmap e326: 8 total, 8 up, 5 in 2023-06-10T21:37:34.782 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 84450 84434 65.1106 4.71875 0.0345352 0.0153515 2023-06-10T21:37:34.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:34.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:34.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:35.019 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:35.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:37:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[186706]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[186706]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[186706]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[180297]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[180297]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:35 smithi064 ceph-mon[180297]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.528 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:35.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:35.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:35 smithi152 ceph-mon[156819]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:35 smithi152 ceph-mon[156819]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:35 smithi152 ceph-mon[156819]: failed to encode map e326 with expected crc 2023-06-10T21:37:35.735 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:35.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 84901 84885 64.6589 28.1875 0.0295006 0.0154588 2023-06-10T21:37:35.827 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:35.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:35.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:35.957 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:36.002 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:36.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:36.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:36.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:36.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:36.273 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:37:36.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2366971054' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:37:36.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[186706]: pgmap v1213: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 448 op/s; 5294/252792 objects degraded (2.094%); 5194/252792 objects misplaced (2.055%); 129 KiB/s, 1 keys/s, 4 objects/s recovering 2023-06-10T21:37:36.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2366971054' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:37:36.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:36 smithi064 ceph-mon[180297]: pgmap v1213: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 448 op/s; 5294/252792 objects degraded (2.094%); 5194/252792 objects misplaced (2.055%); 129 KiB/s, 1 keys/s, 4 objects/s recovering 2023-06-10T21:37:36.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:36.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:37:36.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:36.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2366971054' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:37:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:36 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:37:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:36 smithi152 ceph-mon[156819]: pgmap v1213: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 86 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 448 op/s; 5294/252792 objects degraded (2.094%); 5194/252792 objects misplaced (2.055%); 129 KiB/s, 1 keys/s, 4 objects/s recovering 2023-06-10T21:37:36.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:36.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 85294 85278 64.1761 24.5625 0.059704 0.0155734 2023-06-10T21:37:36.849 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 5294/252792 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:37:37.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:37.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:37:37.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: osdmap e327: 8 total, 8 up, 5 in 2023-06-10T21:37:37.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[180297]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 5294/252792 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: osdmap e327: 8 total, 8 up, 5 in 2023-06-10T21:37:37.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:37 smithi064 ceph-mon[186706]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 5294/252792 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2023-06-10T21:37:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:37:37.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: osdmap e327: 8 total, 8 up, 5 in 2023-06-10T21:37:37.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:37 smithi152 ceph-mon[156819]: failed to encode map e327 with expected crc 2023-06-10T21:37:37.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 85636 85620 63.6668 21.375 0.00793924 0.015697 2023-06-10T21:37:37.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:37.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:37.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:37.976 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:38.028 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:38.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:38.100 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:38.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:38.195 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:38.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:38.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:38.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:38.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:38 smithi064 ceph-mon[186706]: failed to encode map e327 with expected crc 2023-06-10T21:37:38.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:38 smithi064 ceph-mon[186706]: pgmap v1215: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 remapped+peering, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 20 MiB/s wr, 325 op/s; 10552/254013 objects degraded (4.154%); 10408/254013 objects misplaced (4.097%); 645 KiB/s, 1 keys/s, 12 objects/s recovering 2023-06-10T21:37:38.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:38 smithi064 ceph-mon[180297]: failed to encode map e327 with expected crc 2023-06-10T21:37:38.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:38 smithi064 ceph-mon[180297]: pgmap v1215: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 remapped+peering, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 20 MiB/s wr, 325 op/s; 10552/254013 objects degraded (4.154%); 10408/254013 objects misplaced (4.097%); 645 KiB/s, 1 keys/s, 12 objects/s recovering 2023-06-10T21:37:38.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:38.604 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:38 smithi152 ceph-mon[156819]: failed to encode map e327 with expected crc 2023-06-10T21:37:38.605 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:38 smithi152 ceph-mon[156819]: pgmap v1215: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 remapped+peering, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 20 MiB/s wr, 325 op/s; 10552/254013 objects degraded (4.154%); 10408/254013 objects misplaced (4.097%); 645 KiB/s, 1 keys/s, 12 objects/s recovering 2023-06-10T21:37:38.608 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:38.760 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:38.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:38.792 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 86068 86052 63.2356 27 0.0254652 0.0158067 2023-06-10T21:37:38.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:39.000 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:39.672 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2] out_osds: [5, 6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:37:39.672 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:37:39.673 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-06-10T21:37:39.673 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:37:39.673 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:37:39.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-06-10T21:37:39.726 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:39.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:39.784 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 86537 86521 62.8413 29.3125 0.0244681 0.0159061 2023-06-10T21:37:39.844 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:37:39.871 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:39.912 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:37:39.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:39.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:37:39.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:37:40.017 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:40.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:40.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[180297]: pgmap v1216: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 455 op/s; 15833/257694 objects degraded (6.144%); 10324/257694 objects misplaced (4.006%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2023-06-10T21:37:40.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:40 smithi064 ceph-mon[186706]: pgmap v1216: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 455 op/s; 15833/257694 objects degraded (6.144%); 10324/257694 objects misplaced (4.006%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2023-06-10T21:37:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:40.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:40 smithi152 ceph-mon[156819]: pgmap v1216: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 28 MiB/s wr, 455 op/s; 15833/257694 objects degraded (6.144%); 10324/257694 objects misplaced (4.006%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2023-06-10T21:37:40.696 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:40.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:40.791 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 86826 86810 62.3222 18.0625 0.0487431 0.0160378 2023-06-10T21:37:40.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:40.901 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:41.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:41.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:41.798 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 87214 87198 61.8849 24.25 0.0311556 0.0161519 2023-06-10T21:37:41.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:41.905 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:41.950 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:41.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:42.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:42.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:42 smithi064 ceph-mon[180297]: pgmap v1217: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 362 op/s; 15833/259674 objects degraded (6.097%); 10229/259674 objects misplaced (3.939%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:37:42.075 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:42 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15833/259674 objects degraded (6.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:42.143 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:42.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:42 smithi152 ceph-mon[156819]: pgmap v1217: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 362 op/s; 15833/259674 objects degraded (6.097%); 10229/259674 objects misplaced (3.939%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:37:42.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:42 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15833/259674 objects degraded (6.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:42.303 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:37:42.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:42.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:42.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:42 smithi064 ceph-mon[186706]: pgmap v1217: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 23 MiB/s wr, 362 op/s; 15833/259674 objects degraded (6.097%); 10229/259674 objects misplaced (3.939%); 1.9 MiB/s, 30 objects/s recovering 2023-06-10T21:37:42.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:42 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15833/259674 objects degraded (6.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:42.354 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:42.383 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:37:42.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:42.428 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:37:42.478 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:37:42.534 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:37:42.583 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:42.584 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:37:42.644 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:42.667 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:37:42.721 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:42.721 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:37:42.722 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:37:42.722 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:37:42.722 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:42.722 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:37:42.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.763 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.764 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:37:42.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 87603 87587 61.4612 24.3125 0.0290984 0.0162639 2023-06-10T21:37:42.809 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:42.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:42.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:42.945 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:42.989 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:42.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:43.085 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:42] "GET /metrics HTTP/1.1" 200 224129 "" "Prometheus/2.33.4" 2023-06-10T21:37:43.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/51897777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:43.086 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/51897777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:43.097 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:43.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/51897777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:43.199 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:43.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0198 2023-06-10T21:37:43.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 88047 2023-06-10T21:37:43.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:37:43.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:37:43.801 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 61.1303 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 30.9811 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 182.25 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 978 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 495.708 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 2916 2023-06-10T21:37:43.802 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-06-10T21:37:43.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0163535 2023-06-10T21:37:43.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0377745 2023-06-10T21:37:43.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.52381 2023-06-10T21:37:43.803 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00116179 2023-06-10T21:37:43.808 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:37:44.401 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:44 smithi064 ceph-mon[180297]: pgmap v1218: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 23 MiB/s wr, 362 op/s; 15833/261747 objects degraded (6.049%); 10229/261747 objects misplaced (3.908%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:37:44.401 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:44 smithi064 ceph-mon[186706]: pgmap v1218: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 23 MiB/s wr, 362 op/s; 15833/261747 objects degraded (6.049%); 10229/261747 objects misplaced (3.908%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:37:44.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:44 smithi152 ceph-mon[156819]: pgmap v1218: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 23 MiB/s wr, 362 op/s; 15833/261747 objects degraded (6.049%); 10229/261747 objects misplaced (3.908%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:37:45.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:45.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:45.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:45.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:37:45.703 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:46.086 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:46 smithi064 ceph-mon[186706]: pgmap v1219: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 22 MiB/s wr, 362 op/s; 15833/262776 objects degraded (6.025%); 10146/262776 objects misplaced (3.861%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:37:46.086 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:46 smithi064 ceph-mon[180297]: pgmap v1219: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 22 MiB/s wr, 362 op/s; 15833/262776 objects degraded (6.025%); 10146/262776 objects misplaced (3.861%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:37:46.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:46 smithi152 ceph-mon[156819]: pgmap v1219: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 102 B/s rd, 22 MiB/s wr, 362 op/s; 15833/262776 objects degraded (6.025%); 10146/262776 objects misplaced (3.861%); 1.9 MiB/s, 31 objects/s recovering 2023-06-10T21:37:46.667 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:46.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:46.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:37:46.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:47.142 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:37:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:47.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:37:47.252 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:47.279 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:47.323 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:47.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:47.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:47.527 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:48.084 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:37:48.085 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:48.095 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:48 smithi064 ceph-mon[186706]: pgmap v1220: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 87 B/s rd, 20 MiB/s wr, 369 op/s; 15833/262233 objects degraded (6.038%); 10040/262233 objects misplaced (3.829%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:37:48.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:48 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15833/262233 objects degraded (6.038%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:48.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:48 smithi064 ceph-mon[180297]: pgmap v1220: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 87 B/s rd, 20 MiB/s wr, 369 op/s; 15833/262233 objects degraded (6.038%); 10040/262233 objects misplaced (3.829%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:37:48.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:48 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15833/262233 objects degraded (6.038%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:48 smithi152 ceph-mon[156819]: pgmap v1220: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 87 B/s rd, 20 MiB/s wr, 369 op/s; 15833/262233 objects degraded (6.038%); 10040/262233 objects misplaced (3.829%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:37:48.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:48 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15833/262233 objects degraded (6.038%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:48.539 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:37:48.608 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:48.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:48.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:48.749 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:48.793 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:48.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:48.900 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:48.927 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:48.972 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:48.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:48.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:49.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:49.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:49.572 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:49.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:49.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:49.573 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:37:49.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:49.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:49.885 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:50.010 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:50 smithi152 ceph-mon[156819]: pgmap v1221: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 18 MiB/s wr, 454 op/s; 15833/258048 objects degraded (6.136%); 9958/258048 objects misplaced (3.859%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:50.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:50 smithi064 ceph-mon[180297]: pgmap v1221: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 18 MiB/s wr, 454 op/s; 15833/258048 objects degraded (6.136%); 9958/258048 objects misplaced (3.859%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:50 smithi064 ceph-mon[186706]: pgmap v1221: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 18 MiB/s wr, 454 op/s; 15833/258048 objects degraded (6.136%); 9958/258048 objects misplaced (3.859%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:51.459 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:51.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:51.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:51.582 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:37:51.608 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:51.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:51.660 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:37:51.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:51.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:37:51.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:37:51.793 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:51.838 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:37:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:51.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:37:51.990 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:52.125 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:52.306 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:52 smithi064 ceph-mon[180297]: pgmap v1222: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 11 MiB/s wr, 409 op/s; 15833/255990 objects degraded (6.185%); 9854/255990 objects misplaced (3.849%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:37:52.306 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:52 smithi064 ceph-mon[186706]: pgmap v1222: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 11 MiB/s wr, 409 op/s; 15833/255990 objects degraded (6.185%); 9854/255990 objects misplaced (3.849%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:37:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:52 smithi152 ceph-mon[156819]: pgmap v1222: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 85 B/s rd, 11 MiB/s wr, 409 op/s; 15833/255990 objects degraded (6.185%); 9854/255990 objects misplaced (3.849%); 2.4 MiB/s, 39 objects/s recovering 2023-06-10T21:37:52.771 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:37:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:37:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:37:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:37:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.772 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24678 11.6 e279/269 11:677a8f62:::benchmark_data_smithi064_385725_object24677:head v 279'1564, mlcod=279'1562)", 2023-06-10T21:37:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:37:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.65151885199998, 2023-06-10T21:37:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.284365914, 2023-06-10T21:37:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.773 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:37:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114444+0000", 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114466+0000", 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2447e-05 2023-06-10T21:37:52.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114476+0000", 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9420000000000007e-06 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.114483+0000", 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7329999999999997e-06 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115009+0000", 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052553099999999998 2023-06-10T21:37:52.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115028+0000", 2023-06-10T21:37:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8950999999999999e-05 2023-06-10T21:37:52.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116760+0000", 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001732063 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398810+0000", 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28205071100000001 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.782 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24683 11.e e279/269 11:7b1e2b59:::benchmark_data_smithi064_385725_object24682:head v 279'1506, mlcod=279'1502)", 2023-06-10T21:37:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:37:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.65077662300001, 2023-06-10T21:37:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28366207199999999, 2023-06-10T21:37:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.783 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115187+0000", 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115186+0000", 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115216+0000", 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9923000000000001e-05 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115227+0000", 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0413e-05 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115233+0000", 2023-06-10T21:37:52.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4470000000000003e-06 2023-06-10T21:37:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115458+0000", 2023-06-10T21:37:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022539899999999999 2023-06-10T21:37:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.115476+0000", 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7776999999999999e-05 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398819+0000", 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28334313500000002 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398849+0000", 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9493999999999999e-05 2023-06-10T21:37:52.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24682 11.a e279/269 11:59387607:::benchmark_data_smithi064_385725_object24681:head v 279'1501, mlcod=279'1498)", 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:37:52.793 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.64962208999998, 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28636024700000001, 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:37:52.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116341+0000", 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116403+0000", 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2799999999999995e-05 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116415+0000", 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1375e-05 2023-06-10T21:37:52.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116422+0000", 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6580000000000002e-06 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398803+0000", 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28238095299999999 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398836+0000", 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2336999999999999e-05 2023-06-10T21:37:52.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402680+0000", 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038444540000000002 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402701+0000", 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1370999999999999e-05 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.801 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24681 11.5 e279/272 11:adf48b79:::benchmark_data_smithi064_385725_object24680:head v 279'1595, mlcod=279'1591)", 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.64952703199998, 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28619006400000002, 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:37:52.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116436+0000", 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116461+0000", 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4776000000000001e-05 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116469+0000", 2023-06-10T21:37:52.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0469999999999994e-06 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.116474+0000", 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7200000000000003e-06 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398798+0000", 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28232310799999999 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398829+0000", 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1575000000000002e-05 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402581+0000", 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037519649999999999 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402626+0000", 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5272999999999998e-05 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24684 11.3 e279/272 11:cb1db951:::benchmark_data_smithi064_385725_object24683:head v 279'1548, mlcod=279'1546)", 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.64881459100002, 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.285234922, 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.810 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117149+0000", 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117148+0000", 2023-06-10T21:37:52.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.117171+0000", 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2493999999999998e-05 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398742+0000", 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28157169199999998 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398762+0000", 2023-06-10T21:37:52.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9312000000000001e-05 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398806+0000", 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4169999999999999e-05 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398833+0000", 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7229999999999998e-05 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402321+0000", 2023-06-10T21:37:52.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034878449999999998 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402384+0000", 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2801999999999999e-05 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.817 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:24687 11.b e279/269 11:d00b5318:::benchmark_data_smithi064_385725_object24686:head v 279'1542, mlcod=279'1540)", 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.64763237, 2023-06-10T21:37:52.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28407989, 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:37:52.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118331+0000", 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118330+0000", 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.118361+0000", 2023-06-10T21:37:52.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0694000000000001e-05 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398783+0000", 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.280422161 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398792+0000", 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9940000000000006e-06 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398895+0000", 2023-06-10T21:37:52.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103235 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.398921+0000", 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5743000000000001e-05 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402386+0000", 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034653610000000001 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:37.402411+0000", 2023-06-10T21:37:52.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4371000000000001e-05 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:25920 11.5 e280/272 11:a9b9458c:::benchmark_data_smithi064_385725_object25919:head v 280'1670, mlcod=280'1667)", 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:37:52.826 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 374.379534197, 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22712411299999999, 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386429+0000", 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386450+0000", 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1576000000000001e-05 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386457+0000", 2023-06-10T21:37:52.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0119999999999999e-06 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386463+0000", 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6799999999999998e-06 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.387488+0000", 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001024552 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.387505+0000", 2023-06-10T21:37:52.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7699000000000001e-05 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613522+0000", 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22601626399999999 2023-06-10T21:37:52.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613553+0000", 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1594999999999998e-05 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.834 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:25926 11.4 e280/269 11:2939867b:::benchmark_data_smithi064_385725_object25925:head v 280'1586, mlcod=280'1583)", 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 374.37922893000001, 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.226442118, 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.835 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386734+0000", 2023-06-10T21:37:52.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386819+0000", 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5186999999999996e-05 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386828+0000", 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2189999999999993e-06 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.386835+0000", 2023-06-10T21:37:52.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3579999999999999e-06 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.388201+0000", 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013661770000000001 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.388214+0000", 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3159999999999999e-05 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613149+0000", 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22493552 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:31:38.613177+0000", 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7114000000000002e-05 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.842 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166752 11.a e307/269 11:53b08c96:::benchmark_data_smithi064_385725_object82479:head v 307'10239, mlcod=307'10237)", 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.249458+0000", 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.51650529899999, 2023-06-10T21:37:52.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16893702699999999, 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249458+0000", 2023-06-10T21:37:52.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249458+0000", 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249458+0000", 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249462+0000", 2023-06-10T21:37:52.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2300000000000002e-06 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249468+0000", 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6559999999999997e-06 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.249472+0000", 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3420000000000001e-06 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.252677+0000", 2023-06-10T21:37:52.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003204812 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.252696+0000", 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8737999999999999e-05 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418364+0000", 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.165668076 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418395+0000", 2023-06-10T21:37:52.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1470000000000002e-05 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166757 11.c e307/269 11:3d7af98d:::benchmark_data_smithi064_385725_object82484:head v 307'10442, mlcod=307'10440)", 2023-06-10T21:37:52.851 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.254411+0000", 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.51155268299999, 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16865375899999999, 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254411+0000", 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254411+0000", 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254410+0000", 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254418+0000", 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3000000000000002e-06 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254430+0000", 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2082999999999999e-05 2023-06-10T21:37:52.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254439+0000", 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5459999999999992e-06 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254479+0000", 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9857999999999997e-05 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.254501+0000", 2023-06-10T21:37:52.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2466000000000001e-05 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.423027+0000", 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16852651900000001 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.423064+0000", 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6915000000000001e-05 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.859 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166756 11.3 e307/305 11:cf179f45:::benchmark_data_smithi064_385725_object82483:head v 307'10278, mlcod=307'10276)", 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.258342+0000", 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.50762095499999, 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16394403599999999, 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258342+0000", 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258342+0000", 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258341+0000", 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258352+0000", 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0587999999999999e-05 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258375+0000", 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2979000000000001e-05 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258384+0000", 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5070000000000006e-06 2023-06-10T21:37:52.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258443+0000", 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8758000000000002e-05 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258469+0000", 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5979e-05 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422248+0000", 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16377929299999999 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422286+0000", 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8226999999999998e-05 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166760 11.3 e307/305 11:c2c15377:::benchmark_data_smithi064_385725_object82487:head v 307'10279, mlcod=307'10276)", 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.258404+0000", 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.50755947100001, 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16421524500000001, 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.868 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258404+0000", 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258404+0000", 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258404+0000", 2023-06-10T21:37:52.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258408+0000", 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0520000000000003e-06 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258414+0000", 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8880000000000002e-06 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258418+0000", 2023-06-10T21:37:52.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.301e-06 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.261629+0000", 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003210816 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.261646+0000", 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7679000000000001e-05 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422596+0000", 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16094989300000001 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422619+0000", 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2920000000000001e-05 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.875 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166762 11.e e307/269 11:7a72cbe4:::benchmark_data_smithi064_385725_object82489:head v 307'10251, mlcod=307'10249)", 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.258437+0000", 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.507526015, 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17444915599999999, 2023-06-10T21:37:52.876 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258437+0000", 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258437+0000", 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258437+0000", 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258445+0000", 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3980000000000002e-06 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258450+0000", 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5799999999999999e-06 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258454+0000", 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2849999999999996e-06 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258587+0000", 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000132667 2023-06-10T21:37:52.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258607+0000", 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9680000000000001e-05 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.432847+0000", 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.17424019299999999 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.432886+0000", 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9533999999999998e-05 2023-06-10T21:37:52.883 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166755 11.a e307/269 11:5ae28e7a:::benchmark_data_smithi064_385725_object82482:head v 307'10240, mlcod=307'10237)", 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.258483+0000", 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.50748041700001, 2023-06-10T21:37:52.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.163114483, 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258483+0000", 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258483+0000", 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258483+0000", 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258487+0000", 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7709999999999997e-06 2023-06-10T21:37:52.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258494+0000", 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4559999999999997e-06 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258498+0000", 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3819999999999997e-06 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258549+0000", 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0441e-05 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.258571+0000", 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2753000000000001e-05 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.421544+0000", 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16297239299999999 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.421597+0000", 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3693000000000002e-05 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.891 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166754 11.b e307/269 11:da493436:::benchmark_data_smithi064_385725_object82481:head v 307'10423, mlcod=307'10420)", 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.259404+0000", 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.50655890100001, 2023-06-10T21:37:52.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16318692900000001, 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259404+0000", 2023-06-10T21:37:52.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259404+0000", 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259404+0000", 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259410+0000", 2023-06-10T21:37:52.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7599999999999997e-06 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259421+0000", 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0855e-05 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.259429+0000", 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9179999999999994e-06 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.261623+0000", 2023-06-10T21:37:52.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021941080000000002 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.261644+0000", 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0411000000000002e-05 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422538+0000", 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16089381899999999 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.422591+0000", 2023-06-10T21:37:52.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3780999999999999e-05 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.36159.0:166758 11.c e307/269 11:31b2b2bf:::benchmark_data_smithi064_385725_object82485:head v 307'10443, mlcod=307'10440)", 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:35:35.267768+0000", 2023-06-10T21:37:52.900 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 137.49819529600001, 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16222557900000001, 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.267768+0000", 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.267768+0000", 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.267767+0000", 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.267776+0000", 2023-06-10T21:37:52.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8030000000000003e-06 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418190+0000", 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.15041445000000001 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418207+0000", 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6259000000000001e-05 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418248+0000", 2023-06-10T21:37:52.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1610000000000003e-05 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.418266+0000", 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8045000000000001e-05 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.429949+0000", 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011683157 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:35:35.429994+0000", 2023-06-10T21:37:52.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4229e-05 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:87622 12.2 e327/325 12:4bddfeef:::benchmark_data_smithi064_442287_object87621:head v 327'5495, mlcod=0'0)", 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:42.866784+0000", 2023-06-10T21:37:52.908 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 9.8991795039999992, 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.050047361999999998, 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866784+0000", 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866784+0000", 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866783+0000", 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866815+0000", 2023-06-10T21:37:52.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1798999999999998e-05 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866833+0000", 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8646000000000001e-05 2023-06-10T21:37:52.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866842+0000", 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7450000000000001e-06 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866919+0000", 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6543000000000002e-05 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.866951+0000", 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2098000000000003e-05 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.916799+0000", 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049848759999999999 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.916831+0000", 2023-06-10T21:37:52.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1798000000000003e-05 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:87625 12.f e327/325 12:f954e71e:::benchmark_data_smithi064_442287_object87624:head v 327'5505, mlcod=0'0)", 2023-06-10T21:37:52.916 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:42.884082+0000", 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 9.8818809739999995, 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.061396565, 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884082+0000", 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884082+0000", 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884082+0000", 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884105+0000", 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3020999999999999e-05 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884113+0000", 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6170000000000001e-06 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.884133+0000", 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0415999999999999e-05 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.891796+0000", 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076624299999999996 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.891827+0000", 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1661000000000001e-05 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.945280+0000", 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.053453301000000002 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:42.945479+0000", 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019844900000000001 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:89579 12.f e327/325 12:f7ac19dc:::benchmark_data_smithi064_442287_object1529:head v 327'5621, mlcod=0'0)", 2023-06-10T21:37:52.924 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:47.139143+0000", 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 5.6268207950000004, 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049620795000000002, 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139143+0000", 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139143+0000", 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139142+0000", 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139149+0000", 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6680000000000001e-06 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139166+0000", 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6696000000000001e-05 2023-06-10T21:37:52.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139173+0000", 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1409999999999999e-06 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139263+0000", 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0088999999999998e-05 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.139288+0000", 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4722000000000001e-05 2023-06-10T21:37:52.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.188709+0000", 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049421184 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:47.188763+0000", 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4171e-05 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.932 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:91561 12.b e327/325 12:dea7bd6b:::benchmark_data_smithi064_442287_object3511:head v 327'5732, mlcod=315'7)", 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:51.587496+0000", 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 1.178466985, 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047614097000000001, 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:37:52.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587496+0000", 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587496+0000", 2023-06-10T21:37:52.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587496+0000", 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587501+0000", 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7650000000000001e-06 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587508+0000", 2023-06-10T21:37:52.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0099999999999998e-06 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.587513+0000", 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4060000000000004e-06 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.595350+0000", 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078365329999999997 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.603740+0000", 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083899109999999999 2023-06-10T21:37:52.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.635069+0000", 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031329598 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:37:52.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:51.635110+0000", 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1254999999999997e-05 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:37:52.940 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:37:52.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:52.942 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:52.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:53.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:53.096 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:37:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:37:52] "GET /metrics HTTP/1.1" 200 224200 "" "Prometheus/2.33.4" 2023-06-10T21:37:53.097 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1086044762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:53.098 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:53.173 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:37:53.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:53.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1086044762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:53.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1086044762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:53.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:37:54.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:54.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: osdmap e328: 8 total, 8 up, 5 in 2023-06-10T21:37:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[186706]: pgmap v1224: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 5.1 MiB/s wr, 440 op/s; 15833/253389 objects degraded (6.248%); 9854/253389 objects misplaced (3.889%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:54.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:54.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:54.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: osdmap e328: 8 total, 8 up, 5 in 2023-06-10T21:37:54.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:54 smithi064 ceph-mon[180297]: pgmap v1224: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 5.1 MiB/s wr, 440 op/s; 15833/253389 objects degraded (6.248%); 9854/253389 objects misplaced (3.889%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:37:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:37:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: osdmap e328: 8 total, 8 up, 5 in 2023-06-10T21:37:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: failed to encode map e328 with expected crc 2023-06-10T21:37:54.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:54 smithi152 ceph-mon[156819]: pgmap v1224: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 5.1 MiB/s wr, 440 op/s; 15833/253389 objects degraded (6.248%); 9854/253389 objects misplaced (3.889%); 2.3 MiB/s, 37 objects/s recovering 2023-06-10T21:37:55.061 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2] out_osds: [5, 6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:37:55.061 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:37:55.062 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-06-10T21:37:55.062 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:37:55.062 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:37:55.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-06-10T21:37:55.284 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:55.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:55.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[180297]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15833/253389 objects degraded (6.248%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[180297]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[186706]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15833/253389 objects degraded (6.248%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:55.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:55 smithi064 ceph-mon[186706]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:55 smithi152 ceph-mon[156819]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:55 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15833/253389 objects degraded (6.248%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:37:55.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:55 smithi152 ceph-mon[156819]: failed to encode map e328 with expected crc 2023-06-10T21:37:55.491 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:57.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:57 smithi064 ceph-mon[180297]: pgmap v1225: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 2.1 MiB/s wr, 450 op/s; 15833/251244 objects degraded (6.302%); 9764/251244 objects misplaced (3.886%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:57.286 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:57 smithi064 ceph-mon[186706]: pgmap v1225: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 2.1 MiB/s wr, 450 op/s; 15833/251244 objects degraded (6.302%); 9764/251244 objects misplaced (3.886%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:57.332 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:57.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:57.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:37:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:57 smithi152 ceph-mon[156819]: pgmap v1225: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 2.1 MiB/s wr, 450 op/s; 15833/251244 objects degraded (6.302%); 9764/251244 objects misplaced (3.886%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:57.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:57.555 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:57.791 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:37:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:57.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:37:57.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:37:58.062 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:58.098 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:58.240 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:37:58.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:58.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:37:58.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:37:58.339 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:37:58.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:37:58.406 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:37:58.461 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:37:58.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:58.510 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:37:58.547 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:37:58.580 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:37:58.597 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:37:58.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:58.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:37:58.623 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:37:58.637 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.637 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.637 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.638 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:37:58.711 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:37:58.743 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:58.789 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:37:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:37:58.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:37:58.822 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:37:58.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[186706]: pgmap v1226: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 416 KiB/s wr, 445 op/s; 15833/249261 objects degraded (6.352%); 9653/249261 objects misplaced (3.873%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:58.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/873246122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:58.870 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.870 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[180297]: pgmap v1226: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 416 KiB/s wr, 445 op/s; 15833/249261 objects degraded (6.352%); 9653/249261 objects misplaced (3.873%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:58.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/873246122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:58.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:37:58.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:58 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:37:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:58 smithi152 ceph-mon[156819]: pgmap v1226: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.1 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 416 KiB/s wr, 445 op/s; 15833/249261 objects degraded (6.352%); 9653/249261 objects misplaced (3.873%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:37:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/873246122' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:37:58.993 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:37:59.333 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:38:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:38:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:38:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:38:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:38:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:37:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:00.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:38:00.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:38:00.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:37:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:01.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:00 smithi064 ceph-mon[186706]: pgmap v1227: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 442 op/s; 15833/244779 objects degraded (6.468%); 9572/244779 objects misplaced (3.910%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:01.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:00 smithi064 ceph-mon[180297]: pgmap v1227: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 442 op/s; 15833/244779 objects degraded (6.468%); 9572/244779 objects misplaced (3.910%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:01.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:00 smithi152 ceph-mon[156819]: pgmap v1227: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 5.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 442 op/s; 15833/244779 objects degraded (6.468%); 9572/244779 objects misplaced (3.910%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:01.271 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:01.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:01.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:01.383 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:01.491 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:01.730 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:01.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:01.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:01.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15833/244779 objects degraded (6.468%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:01.856 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15833/244779 objects degraded (6.468%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:01.962 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:02.170 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:02.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15833/244779 objects degraded (6.468%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:02.362 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:02.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:02.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:02.511 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:02.553 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:02.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:02.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:02.688 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:38:02.689 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-06-10T21:38:02.689 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:38:02.690 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:02.732 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:02.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:02.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:02.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:02 smithi152 ceph-mon[156819]: pgmap v1228: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 439 op/s; 15833/242784 objects degraded (6.521%); 9467/242784 objects misplaced (3.899%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:02.947 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:02] "GET /metrics HTTP/1.1" 200 224200 "" "Prometheus/2.33.4" 2023-06-10T21:38:03.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:02 smithi064 ceph-mon[186706]: pgmap v1228: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 439 op/s; 15833/242784 objects degraded (6.521%); 9467/242784 objects misplaced (3.899%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:03.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:02 smithi064 ceph-mon[180297]: pgmap v1228: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 439 op/s; 15833/242784 objects degraded (6.521%); 9467/242784 objects misplaced (3.899%); 2.4 MiB/s, 38 objects/s recovering 2023-06-10T21:38:03.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:38:03.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:03.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:04.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:04.045 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:04.046 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:04.173 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:04.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:05.030 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:05.046 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:04 smithi064 ceph-mon[180297]: pgmap v1229: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 480 op/s; 15833/240309 objects degraded (6.589%); 9467/240309 objects misplaced (3.940%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:38:05.047 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:04 smithi064 ceph-mon[186706]: pgmap v1229: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 480 op/s; 15833/240309 objects degraded (6.589%); 9467/240309 objects misplaced (3.940%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:38:05.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:05.148 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:05.172 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:04 smithi152 ceph-mon[156819]: pgmap v1229: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 480 op/s; 15833/240309 objects degraded (6.589%); 9467/240309 objects misplaced (3.940%); 2.2 MiB/s, 35 objects/s recovering 2023-06-10T21:38:05.174 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:05.217 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:05.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:05.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:05.349 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:05.390 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:05.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:05.496 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:05.522 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:05.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:05.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:05.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:05.799 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:05.832 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:05.833 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:06.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:06 smithi064 ceph-mon[186706]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 418 op/s; 15864/238317 objects degraded (6.657%); 9341/238317 objects misplaced (3.920%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:38:06.085 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15833/240309 objects degraded (6.589%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:06.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:06 smithi064 ceph-mon[180297]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 418 op/s; 15864/238317 objects degraded (6.657%); 9341/238317 objects misplaced (3.920%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:38:06.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15833/240309 objects degraded (6.589%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:06 smithi152 ceph-mon[156819]: pgmap v1230: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 418 op/s; 15864/238317 objects degraded (6.657%); 9341/238317 objects misplaced (3.920%); 2.5 MiB/s, 40 objects/s recovering 2023-06-10T21:38:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15833/240309 objects degraded (6.589%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:06.984 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:06.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:07.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:07.127 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:07.890 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:07.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:08.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:08.036 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:08.085 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:08.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:08.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:08 smithi064 ceph-mon[186706]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 15864/236115 objects degraded (6.719%); 9203/236115 objects misplaced (3.898%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:08.163 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:08 smithi064 ceph-mon[180297]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 15864/236115 objects degraded (6.719%); 9203/236115 objects misplaced (3.898%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:08.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:08.218 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:08.235 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:08.264 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:08.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:08.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:08.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:38:08.369 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:08.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:08 smithi152 ceph-mon[156819]: pgmap v1231: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 15864/236115 objects degraded (6.719%); 9203/236115 objects misplaced (3.898%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:08.445 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:08.708 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:08.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:08.816 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:08.844 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:08.888 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:08.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:08.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:09.020 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:09.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:09.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:09.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:09.199 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:09.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:09.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:09.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:09.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:09.588 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:09.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:09.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:09.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:09.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:09.816 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:38:09.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:09.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:09.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:09.992 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:10.036 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:10.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:10.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:10.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3009056904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:10.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3009056904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.165 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:09 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:10.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3009056904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:09 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:10.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:09 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:10.230 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:10.378 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2] out_osds: [5, 6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:38:10.379 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:38:10.379 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.23010933957063198 2023-06-10T21:38:10.379 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 0.23010933957063198 2023-06-10T21:38:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[180297]: osdmap e329: 8 total, 8 up, 5 in 2023-06-10T21:38:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:10.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[180297]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15888/231753 objects degraded (6.856%); 9075/231753 objects misplaced (3.916%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[186706]: osdmap e329: 8 total, 8 up, 5 in 2023-06-10T21:38:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:10 smithi064 ceph-mon[186706]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15888/231753 objects degraded (6.856%); 9075/231753 objects misplaced (3.916%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:11.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:11.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:10 smithi152 ceph-mon[156819]: osdmap e329: 8 total, 8 up, 5 in 2023-06-10T21:38:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:10 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:10 smithi152 ceph-mon[156819]: pgmap v1233: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15888/231753 objects degraded (6.856%); 9075/231753 objects misplaced (3.916%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:11.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:11.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:11.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:11.618 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:11.649 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:11.660 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:11.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:11.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:11.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15888/231753 objects degraded (6.856%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:11.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.830 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.831 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15888/231753 objects degraded (6.856%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:11.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:11.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: failed to encode map e329 with expected crc 2023-06-10T21:38:11.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15888/231753 objects degraded (6.856%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:12.059 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:12.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:12.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:12.199 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:12.245 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:12.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:12.316 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:38:12.351 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:12.377 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:12.383 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:38:12.421 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:12.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:12.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:38:12.507 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:38:12.529 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:12.536 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:38:12.557 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:12.577 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:38:12.600 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:12.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:12.610 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:38:12.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:38:12.669 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:12.681 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.681 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.681 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.682 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.683 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:38:12.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:12.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:12.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:12 smithi064 ceph-mon[180297]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 432 op/s; 15888/229812 objects degraded (6.913%); 8973/229812 objects misplaced (3.904%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:12.946 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/565059560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:12.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:12 smithi064 ceph-mon[186706]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 432 op/s; 15888/229812 objects degraded (6.913%); 8973/229812 objects misplaced (3.904%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:12.947 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/565059560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:12.947 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:12] "GET /metrics HTTP/1.1" 200 224186 "" "Prometheus/2.33.4" 2023-06-10T21:38:12.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:12.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:13.068 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:13.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:13.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:12 smithi152 ceph-mon[156819]: pgmap v1234: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 432 op/s; 15888/229812 objects degraded (6.913%); 8973/229812 objects misplaced (3.904%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:13.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/565059560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:13.521 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:38:14.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:14 smithi064 ceph-mon[180297]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 428 op/s; 15888/227445 objects degraded (6.985%); 8973/227445 objects misplaced (3.945%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:14.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:14 smithi064 ceph-mon[186706]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 428 op/s; 15888/227445 objects degraded (6.985%); 8973/227445 objects misplaced (3.945%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:14 smithi152 ceph-mon[156819]: pgmap v1235: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 428 op/s; 15888/227445 objects degraded (6.985%); 8973/227445 objects misplaced (3.945%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:38:14.504 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:14.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:14.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:14.644 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:14.687 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:14.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:14.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:14.819 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:14.868 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:14.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:14.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:15.001 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:15.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:15.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:15.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:15.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:15.648 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:15.824 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:15.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:15.935 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:15.962 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:16.007 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:16.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:16.114 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:16.140 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:16.185 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:16.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:16.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:16.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:16 smithi064 ceph-mon[186706]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15923/225300 objects degraded (7.067%); 8833/225300 objects misplaced (3.921%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:16.390 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15888/227445 objects degraded (6.985%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:16.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:16 smithi064 ceph-mon[180297]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15923/225300 objects degraded (7.067%); 8833/225300 objects misplaced (3.921%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:16.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15888/227445 objects degraded (6.985%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:16.400 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:16.434 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:38:16.435 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:38:16.435 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:38:16.435 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:38:16.435 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:38:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:16 smithi152 ceph-mon[156819]: pgmap v1236: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 15923/225300 objects degraded (7.067%); 8833/225300 objects misplaced (3.921%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:38:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15888/227445 objects degraded (6.985%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:17.369 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:17.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:17.478 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:17.505 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:17.547 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:17.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:17.605 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:38:17.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:17.680 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:17.724 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:17.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:17.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:17.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:17.860 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:17.879 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:17.903 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:17.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:18.009 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:18.035 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:18.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:18.083 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:18 smithi064 ceph-mon[180297]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 15923/223185 objects degraded (7.134%); 8723/223185 objects misplaced (3.908%); 2.6 MiB/s, 42 objects/s recovering 2023-06-10T21:38:18.083 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:18 smithi064 ceph-mon[186706]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 15923/223185 objects degraded (7.134%); 8723/223185 objects misplaced (3.908%); 2.6 MiB/s, 42 objects/s recovering 2023-06-10T21:38:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:18.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:18.185 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:18.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:18 smithi152 ceph-mon[156819]: pgmap v1237: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 15923/223185 objects degraded (7.134%); 8723/223185 objects misplaced (3.908%); 2.6 MiB/s, 42 objects/s recovering 2023-06-10T21:38:18.211 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:18.254 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:18.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:18.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:18.436 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:18.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:38:19.015 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:19.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:19.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:19.181 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:19.225 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:19.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:19.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:19.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:19.353 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:19.356 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:19.424 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:19.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:19.977 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:19.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:19.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:20.082 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:20.159 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:20.376 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:20.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:20 smithi064 ceph-mon[180297]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 15947/218508 objects degraded (7.298%); 8597/218508 objects misplaced (3.934%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:20.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:20.387 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:20 smithi064 ceph-mon[186706]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 15947/218508 objects degraded (7.298%); 8597/218508 objects misplaced (3.934%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:20 smithi152 ceph-mon[156819]: pgmap v1238: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 500 op/s; 15947/218508 objects degraded (7.298%); 8597/218508 objects misplaced (3.934%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:20.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:20.514 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:20.559 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:20.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:20.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:20.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:21.026 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:21.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:21.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:21.046 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:21.132 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:21.211 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 0.230109 (3ae8) 2023-06-10T21:38:21.260 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:21.280 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/419630177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15947/218508 objects degraded (7.298%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/419630177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15947/218508 objects degraded (7.298%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:21.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/419630177' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]: dispatch 2023-06-10T21:38:21.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15947/218508 objects degraded (7.298%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:22.181 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:22.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:22.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:22.314 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:22.361 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:22.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:22.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:22.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]': finished 2023-06-10T21:38:22.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: osdmap e330: 8 total, 8 up, 5 in 2023-06-10T21:38:22.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[180297]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 10689/217761 objects degraded (4.909%); 8447/217761 objects misplaced (3.879%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]': finished 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: osdmap e330: 8 total, 8 up, 5 in 2023-06-10T21:38:22.499 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.500 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.500 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.500 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:22 smithi064 ceph-mon[186706]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 10689/217761 objects degraded (4.909%); 8447/217761 objects misplaced (3.879%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:38:22.502 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.23010933957063198}]': finished 2023-06-10T21:38:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: osdmap e330: 8 total, 8 up, 5 in 2023-06-10T21:38:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: failed to encode map e330 with expected crc 2023-06-10T21:38:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:22 smithi152 ceph-mon[156819]: pgmap v1240: 89 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 401 op/s; 10689/217761 objects degraded (4.909%); 8447/217761 objects misplaced (3.879%); 2.9 MiB/s, 46 objects/s recovering 2023-06-10T21:38:23.084 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:23.088 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:22] "GET /metrics HTTP/1.1" 200 224187 "" "Prometheus/2.33.4" 2023-06-10T21:38:23.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:23.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:23.236 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[186706]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[186706]: osdmap e331: 8 total, 8 up, 5 in 2023-06-10T21:38:23.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[186706]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[186706]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[186706]: failed to encode map e330 with expected crc 2023-06-10T21:38:23.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[180297]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[180297]: osdmap e331: 8 total, 8 up, 5 in 2023-06-10T21:38:23.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[180297]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[180297]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:23 smithi064 ceph-mon[180297]: failed to encode map e330 with expected crc 2023-06-10T21:38:23.663 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:23.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:23.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:23 smithi152 ceph-mon[156819]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:23 smithi152 ceph-mon[156819]: osdmap e331: 8 total, 8 up, 5 in 2023-06-10T21:38:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:23 smithi152 ceph-mon[156819]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:23 smithi152 ceph-mon[156819]: failed to encode map e331 with expected crc 2023-06-10T21:38:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:23 smithi152 ceph-mon[156819]: failed to encode map e330 with expected crc 2023-06-10T21:38:23.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:23.856 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:24.294 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:38:24.326 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: failed to encode map e331 with expected crc 2023-06-10T21:38:24.326 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: osdmap e332: 8 total, 8 up, 5 in 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 329'6617 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.2 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6582 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6574 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 329'6603 2023-06-10T21:38:24.327 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.e starting backfill to osd.1 from (0'0,0'0] MAX to 329'6563 2023-06-10T21:38:24.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 329'6411 2023-06-10T21:38:24.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/490197892' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.328 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[186706]: pgmap v1243: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 14302/215637 objects degraded (6.632%); 27074/215637 objects misplaced (12.555%); 2.7 MiB/s, 45 objects/s recovering 2023-06-10T21:38:24.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: failed to encode map e331 with expected crc 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: osdmap e332: 8 total, 8 up, 5 in 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 329'6617 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.2 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6582 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6574 2023-06-10T21:38:24.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 329'6603 2023-06-10T21:38:24.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.e starting backfill to osd.1 from (0'0,0'0] MAX to 329'6563 2023-06-10T21:38:24.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 329'6411 2023-06-10T21:38:24.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/490197892' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:24 smithi064 ceph-mon[180297]: pgmap v1243: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 14302/215637 objects degraded (6.632%); 27074/215637 objects misplaced (12.555%); 2.7 MiB/s, 45 objects/s recovering 2023-06-10T21:38:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: failed to encode map e331 with expected crc 2023-06-10T21:38:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: osdmap e332: 8 total, 8 up, 5 in 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: failed to encode map e332 with expected crc 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.8 starting backfill to osd.7 from (0'0,0'0] MAX to 329'6617 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.2 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6582 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.4 starting backfill to osd.0 from (0'0,0'0] MAX to 329'6574 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.3 starting backfill to osd.1 from (0'0,0'0] MAX to 329'6603 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.e starting backfill to osd.1 from (0'0,0'0] MAX to 329'6563 2023-06-10T21:38:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: 12.1 starting backfill to osd.2 from (0'0,0'0] MAX to 329'6411 2023-06-10T21:38:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/490197892' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:24 smithi152 ceph-mon[156819]: pgmap v1243: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 14302/215637 objects degraded (6.632%); 27074/215637 objects misplaced (12.555%); 2.7 MiB/s, 45 objects/s recovering 2023-06-10T21:38:25.408 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:25.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:25.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:25.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:38:25.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[180297]: failed to encode map e333 with expected crc 2023-06-10T21:38:25.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:25.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[180297]: osdmap e333: 8 total, 8 up, 5 in 2023-06-10T21:38:25.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[180297]: failed to encode map e333 with expected crc 2023-06-10T21:38:25.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:38:25.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[186706]: failed to encode map e333 with expected crc 2023-06-10T21:38:25.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:25.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[186706]: osdmap e333: 8 total, 8 up, 5 in 2023-06-10T21:38:25.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:25 smithi064 ceph-mon[186706]: failed to encode map e333 with expected crc 2023-06-10T21:38:25.624 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:25 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:38:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:25 smithi152 ceph-mon[156819]: failed to encode map e333 with expected crc 2023-06-10T21:38:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:25 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:25 smithi152 ceph-mon[156819]: osdmap e333: 8 total, 8 up, 5 in 2023-06-10T21:38:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:25 smithi152 ceph-mon[156819]: failed to encode map e333 with expected crc 2023-06-10T21:38:26.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[186706]: failed to encode map e333 with expected crc 2023-06-10T21:38:26.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[186706]: pgmap v1245: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 304 op/s; 26414/214101 objects degraded (12.337%); 31529/214101 objects misplaced (14.726%); 3.5 MiB/s, 58 objects/s recovering 2023-06-10T21:38:26.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14302/215637 objects degraded (6.632%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:26.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[180297]: failed to encode map e333 with expected crc 2023-06-10T21:38:26.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[180297]: pgmap v1245: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 304 op/s; 26414/214101 objects degraded (12.337%); 31529/214101 objects misplaced (14.726%); 3.5 MiB/s, 58 objects/s recovering 2023-06-10T21:38:26.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14302/215637 objects degraded (6.632%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:26 smithi152 ceph-mon[156819]: failed to encode map e333 with expected crc 2023-06-10T21:38:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:26 smithi152 ceph-mon[156819]: pgmap v1245: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 304 op/s; 26414/214101 objects degraded (12.337%); 31529/214101 objects misplaced (14.726%); 3.5 MiB/s, 58 objects/s recovering 2023-06-10T21:38:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14302/215637 objects degraded (6.632%), 4 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:26.885 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:26.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:26.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:27.021 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:27.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:27.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:27.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:27.279 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:27.383 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:27.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:27.508 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:27.537 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:27.584 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:27.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:27.693 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:27.837 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:27.934 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:38:27.935 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:38:27.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:38:27.935 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85133 11.3 11:cc56e7c0:::benchmark_data_smithi064_385725_object860:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508726+0000", 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41257149900002, 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7029576049999999, 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:27.936 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85133 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:27.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:27.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508726+0000", 2023-06-10T21:38:27.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:27.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508726+0000", 2023-06-10T21:38:27.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508724+0000", 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:27.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508732+0000", 2023-06-10T21:38:27.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1949999999999997e-06 2023-06-10T21:38:27.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508745+0000", 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3040000000000001e-05 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:27.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508755+0000", 2023-06-10T21:38:27.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0771000000000001e-05 2023-06-10T21:38:27.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510296+0000", 2023-06-10T21:38:27.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015404959999999999 2023-06-10T21:38:27.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:27.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510306+0000", 2023-06-10T21:38:27.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0180000000000001e-05 2023-06-10T21:38:27.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.189804+0000", 2023-06-10T21:38:27.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6794979030000001 2023-06-10T21:38:27.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:38:27.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.189982+0000", 2023-06-10T21:38:27.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017799 2023-06-10T21:38:27.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.211527+0000", 2023-06-10T21:38:27.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021545251000000001 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.211683+0000", 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015623500000000001 2023-06-10T21:38:27.953 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.954 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:27.954 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.954 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85160 11.3 11:caac7e6f:::benchmark_data_smithi064_385725_object887:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508763+0000", 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.412534483, 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.719386168, 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:27.955 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:27.956 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:27.956 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85160 2023-06-10T21:38:27.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.956 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:27.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:27.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508763+0000", 2023-06-10T21:38:27.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:27.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508763+0000", 2023-06-10T21:38:27.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:27.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508762+0000", 2023-06-10T21:38:27.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:27.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508765+0000", 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0359999999999998e-06 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:27.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508784+0000", 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8249000000000001e-05 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508790+0000", 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7290000000000003e-06 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510344+0000", 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001553142 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:27.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510350+0000", 2023-06-10T21:38:27.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8249999999999999e-06 2023-06-10T21:38:27.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.194623+0000", 2023-06-10T21:38:27.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6842729489999999 2023-06-10T21:38:27.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:27.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.194804+0000", 2023-06-10T21:38:27.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018102799999999999 2023-06-10T21:38:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.194978+0000", 2023-06-10T21:38:27.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000174029 2023-06-10T21:38:27.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.211816+0000", 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016837947999999998 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.211864+0000", 2023-06-10T21:38:27.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7274999999999997e-05 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223914+0000", 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012050735 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:27.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223981+0000", 2023-06-10T21:38:27.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7113999999999995e-05 2023-06-10T21:38:27.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228149+0000", 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041676250000000003 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85195 11.3 11:cecdd989:::benchmark_data_smithi064_385725_object922:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508796+0000", 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41250172700001, 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.758871547, 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:27.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85195 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508796+0000", 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508796+0000", 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508795+0000", 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508798+0000", 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0809999999999998e-06 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508805+0000", 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7270000000000001e-06 2023-06-10T21:38:27.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508812+0000", 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6660000000000002e-06 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510382+0000", 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015701140000000001 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510390+0000", 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5490000000000006e-06 2023-06-10T21:38:27.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.211893+0000", 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7015028989999998 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:27.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.212077+0000", 2023-06-10T21:38:27.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018351199999999999 2023-06-10T21:38:27.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.212263+0000", 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018638300000000001 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223855+0000", 2023-06-10T21:38:27.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011592196000000001 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223898+0000", 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2419999999999997e-05 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263179+0000", 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039281571000000001 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263236+0000", 2023-06-10T21:38:27.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6799999999999998e-05 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.267667+0000", 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044311630000000001 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:27.988 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85196 11.3 11:c45bb3e6:::benchmark_data_smithi064_385725_object923:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508817+0000", 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41248079399998, 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.763177177, 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:27.989 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85196 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:27.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508817+0000", 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508817+0000", 2023-06-10T21:38:27.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508816+0000", 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508819+0000", 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4480000000000001e-06 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508827+0000", 2023-06-10T21:38:27.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2479999999999996e-06 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508833+0000", 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6980000000000003e-06 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510451+0000", 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016182950000000001 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510458+0000", 2023-06-10T21:38:27.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8560000000000001e-06 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.216001+0000", 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7055432530000001 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.216180+0000", 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179166 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.216345+0000", 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016455699999999999 2023-06-10T21:38:27.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228222+0000", 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011877194000000001 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228271+0000", 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8724999999999999e-05 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.267677+0000", 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.039406412000000002 2023-06-10T21:38:27.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.267723+0000", 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6236999999999998e-05 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.271994+0000", 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042704939999999997 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85222 11.3 11:cb8993b4:::benchmark_data_smithi064_385725_object949:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.001 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508837+0000", 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41246074600002, 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7676521919999999, 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85222 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508837+0000", 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508837+0000", 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508836+0000", 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508839+0000", 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3319999999999999e-06 2023-06-10T21:38:28.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508844+0000", 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8529999999999997e-06 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508850+0000", 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3600000000000004e-06 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510502+0000", 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001651918 2023-06-10T21:38:28.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510511+0000", 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4290000000000007e-06 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228301+0000", 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7177903080000001 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228476+0000", 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017505299999999999 2023-06-10T21:38:28.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228632+0000", 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015559800000000001 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.262819+0000", 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.034187039000000002 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.262858+0000", 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9097000000000002e-05 2023-06-10T21:38:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.272047+0000", 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091888620000000008 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.272099+0000", 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2027999999999997e-05 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.276489+0000", 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043897060000000002 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.013 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85223 11.3 11:c01fefd1:::benchmark_data_smithi064_385725_object950:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508854+0000", 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41244319600003, 2023-06-10T21:38:28.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7719092079999998, 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85223 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.015 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508854+0000", 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508854+0000", 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508854+0000", 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508856+0000", 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.463e-06 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508862+0000", 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6670000000000001e-06 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508867+0000", 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3870000000000003e-06 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510544+0000", 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016770719999999999 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510552+0000", 2023-06-10T21:38:28.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5070000000000003e-06 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231042+0000", 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7204901860000001 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231200+0000", 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015799999999999999 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231340+0000", 2023-06-10T21:38:28.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139529 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.262895+0000", 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031555295999999997 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.262932+0000", 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6882999999999999e-05 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.276498+0000", 2023-06-10T21:38:28.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013566477 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.276551+0000", 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3094999999999999e-05 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.280763+0000", 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042120309999999998 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85224 11.3 11:c5136d5c:::benchmark_data_smithi064_385725_object951:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508871+0000", 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.412426272, 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.776760519, 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.027 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85224 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508871+0000", 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508871+0000", 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508871+0000", 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508873+0000", 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.384e-06 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508879+0000", 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9410000000000004e-06 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508885+0000", 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5539999999999997e-06 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510615+0000", 2023-06-10T21:38:28.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017308410000000001 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510631+0000", 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5379999999999998e-05 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234160+0000", 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7235292609999999 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234267+0000", 2023-06-10T21:38:28.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010730699999999999 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234391+0000", 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012374099999999999 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.262970+0000", 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028579317 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263011+0000", 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.002e-05 2023-06-10T21:38:28.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.280775+0000", 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017764317000000002 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.280835+0000", 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0488000000000001e-05 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.285632+0000", 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.004796308 2023-06-10T21:38:28.038 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85227 11.3 11:cd02d7b7:::benchmark_data_smithi064_385725_object954:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508889+0000", 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41240837200002, 2023-06-10T21:38:28.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.780961534, 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85227 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.040 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508889+0000", 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508889+0000", 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508889+0000", 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508891+0000", 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.244e-06 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508896+0000", 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4840000000000003e-06 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508903+0000", 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8399999999999997e-06 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510662+0000", 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017587449999999999 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510670+0000", 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6699999999999994e-06 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.236886+0000", 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.726215963 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.237066+0000", 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018083500000000001 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.237235+0000", 2023-06-10T21:38:28.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168498 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263049+0000", 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025814494 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.272039+0000", 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089895830000000006 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.285647+0000", 2023-06-10T21:38:28.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013608442 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.285705+0000", 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7960000000000001e-05 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.289851+0000", 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041451700000000001 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.051 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85276 11.3 11:c383c98a:::benchmark_data_smithi064_385725_object1003:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508907+0000", 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.412390372, 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.785276598, 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.052 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85276 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508907+0000", 2023-06-10T21:38:28.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508907+0000", 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508907+0000", 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508909+0000", 2023-06-10T21:38:28.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4229999999999999e-06 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508915+0000", 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0120000000000002e-06 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508920+0000", 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4269999999999999e-06 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510724+0000", 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018036650000000001 2023-06-10T21:38:28.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510731+0000", 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1239999999999999e-06 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240254+0000", 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.729522746 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240433+0000", 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179066 2023-06-10T21:38:28.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240600+0000", 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016677700000000001 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263087+0000", 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022487150000000001 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.289867+0000", 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026779977999999999 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.289931+0000", 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3709999999999998e-05 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.289992+0000", 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1666000000000002e-05 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.294184+0000", 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041912390000000002 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.063 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85286 11.3 11:c0b32301:::benchmark_data_smithi064_385725_object1013:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.508925+0000", 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 332.41237272900003, 2023-06-10T21:38:28.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.789132202, 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85286 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.065 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508925+0000", 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508925+0000", 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508924+0000", 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508927+0000", 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2809999999999998e-06 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508932+0000", 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1900000000000003e-06 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.508937+0000", 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1699999999999996e-06 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510753+0000", 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018160030000000001 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510759+0000", 2023-06-10T21:38:28.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2500000000000003e-06 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.243645+0000", 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7328862229999999 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.243798+0000", 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015242399999999999 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.243946+0000", 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148667 2023-06-10T21:38:28.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.263125+0000", 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019179036 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.289874+0000", 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026748480000000002 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.294254+0000", 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043800130000000003 2023-06-10T21:38:28.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.294302+0000", 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8446999999999999e-05 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.298057+0000", 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037544229999999998 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104494 12.9 12:9ca311c5:::benchmark_data_smithi064_442287_object16444:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:38:28.076 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.7433002589999997, 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6361784579999998, 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.077 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104494 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177995+0000", 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178006+0000", 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1514e-05 2023-06-10T21:38:28.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178024+0000", 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7651000000000001e-05 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178035+0000", 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0857e-05 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.188934+0000", 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010898873 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189075+0000", 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014062699999999999 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:38:28.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189180+0000", 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105855 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.196756+0000", 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075760269999999999 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207957+0000", 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011200676999999999 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245880+0000", 2023-06-10T21:38:28.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037923327999999999 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245895+0000", 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4575e-05 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200093+0000", 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95419813399999998 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200108+0000", 2023-06-10T21:38:28.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5024e-05 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754247+0000", 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5541385060000001 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754441+0000", 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019410900000000001 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814067+0000", 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059626237999999998 2023-06-10T21:38:28.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814176+0000", 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010860400000000001 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104503 12.a 12:53f91a42:::benchmark_data_smithi064_442287_object16453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.7153998169999998, 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.527545849, 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.091 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104503 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205896+0000", 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205904+0000", 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9699999999999999e-06 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205916+0000", 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2194999999999999e-05 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205924+0000", 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218750+0000", 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012825721999999999 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218889+0000", 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138617 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.219004+0000", 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011551 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.224321+0000", 2023-06-10T21:38:28.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053164609999999998 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246026+0000", 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021705486 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246036+0000", 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0294e-05 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202839+0000", 2023-06-10T21:38:28.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95680261799999999 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202852+0000", 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459999999999999e-05 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636477+0000", 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.433625159 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636649+0000", 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017257199999999999 2023-06-10T21:38:28.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733342+0000", 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096692527 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733443+0000", 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101641 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.104 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104492 12.8 12:145682b0:::benchmark_data_smithi064_442287_object16442:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.6906043789999998, 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1766732329999998, 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.105 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104492 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230691+0000", 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230699+0000", 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-06-10T21:38:28.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.237985+0000", 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072853450000000004 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.238000+0000", 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5784999999999999e-05 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258465+0000", 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020465041999999999 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258473+0000", 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9489999999999996e-06 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257243+0000", 2023-06-10T21:38:28.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99876928499999995 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257255+0000", 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2431e-05 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.405789+0000", 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.148533523 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407366+0000", 2023-06-10T21:38:28.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577572 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104510 12.9 12:91eab998:::benchmark_data_smithi064_442287_object16460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:38:28.114 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.6565238779999998, 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5536998, 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104510 2023-06-10T21:38:28.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264772+0000", 2023-06-10T21:38:28.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264780+0000", 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264792+0000", 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1987999999999999e-05 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264800+0000", 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8189999999999998e-06 2023-06-10T21:38:28.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264846+0000", 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5782000000000003e-05 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264860+0000", 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4267000000000001e-05 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200139+0000", 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93527848099999999 2023-06-10T21:38:28.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200148+0000", 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758081+0000", 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5579332859999999 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758234+0000", 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015339999999999999 2023-06-10T21:38:28.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758918+0000", 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068356899999999997 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.784273+0000", 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025354811000000001 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.785804+0000", 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00153134 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814218+0000", 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028414053000000002 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814269+0000", 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.075e-05 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818473+0000", 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042044639999999998 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.127 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104512 12.8 12:1b4cbe21:::benchmark_data_smithi064_442287_object16462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.6499547979999996, 2023-06-10T21:38:28.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1733947009999999, 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104512 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.129 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271341+0000", 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271350+0000", 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8789999999999998e-06 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271394+0000", 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.384e-05 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271404+0000", 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6309999999999998e-06 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271472+0000", 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7986999999999998e-05 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271488+0000", 2023-06-10T21:38:28.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5911000000000001e-05 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257279+0000", 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98579087099999996 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257287+0000", 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9410000000000007e-06 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407591+0000", 2023-06-10T21:38:28.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15030481 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408064+0000", 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047220399999999999 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408146+0000", 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2446000000000003e-05 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420391+0000", 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012245285999999999 2023-06-10T21:38:28.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423446+0000", 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030550500000000001 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429481+0000", 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060345260000000001 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444541+0000", 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015059771 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444604+0000", 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3081000000000001e-05 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444737+0000", 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133468 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104514 12.8 12:1ddd6575:::benchmark_data_smithi064_442287_object16464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:38:28.142 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 6.6423912329999997, 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1660305000000002, 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.143 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104514 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278905+0000", 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278910+0000", 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278921+0000", 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278927+0000", 2023-06-10T21:38:28.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5470000000000004e-06 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278994+0000", 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6595999999999997e-05 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.279006+0000", 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2479e-05 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257307+0000", 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97830109899999995 2023-06-10T21:38:28.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257315+0000", 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3119999999999994e-06 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410688+0000", 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15337278 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410832+0000", 2023-06-10T21:38:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014341000000000001 2023-06-10T21:38:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410967+0000", 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135816 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420402+0000", 2023-06-10T21:38:28.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094344049999999999 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423489+0000", 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030871060000000001 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429518+0000", 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060294930000000004 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444782+0000", 2023-06-10T21:38:28.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015263465 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444832+0000", 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9935999999999998e-05 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444937+0000", 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010481599999999999 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.157 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104517 12.9 12:938977ad:::benchmark_data_smithi064_442287_object16467:head [delete] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:23.415439+0000", 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.5058584609999999, 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.40644339699999998, 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.158 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104517 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415439+0000", 2023-06-10T21:38:28.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415439+0000", 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415437+0000", 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415446+0000", 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8769999999999999e-06 2023-06-10T21:38:28.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415460+0000", 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4012000000000001e-05 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.415470+0000", 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210999999999999e-05 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606825+0000", 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19135460300000001 2023-06-10T21:38:28.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606838+0000", 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3246e-05 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.773947+0000", 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16710897999999999 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.774132+0000", 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018511600000000001 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.774314+0000", 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018203100000000001 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.784281+0000", 2023-06-10T21:38:28.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0099670499999999999 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.787767+0000", 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034859019999999999 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818589+0000", 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030821459999999998 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818651+0000", 2023-06-10T21:38:28.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2125000000000006e-05 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.821882+0000", 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032314750000000001 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104524 12.9 12:9a4edbfc:::benchmark_data_smithi064_442287_object16474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:23.462618+0000", 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.4586791100000003, 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.36527274399999998, 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.170 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104524 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462618+0000", 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462618+0000", 2023-06-10T21:38:28.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462617+0000", 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462626+0000", 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9600000000000006e-06 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462640+0000", 2023-06-10T21:38:28.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4044e-05 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.462649+0000", 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7880000000000001e-06 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606860+0000", 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14421108399999999 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606868+0000", 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0990000000000001e-06 2023-06-10T21:38:28.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.787799+0000", 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18093142600000001 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.788005+0000", 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00020622899999999999 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.788192+0000", 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000186577 2023-06-10T21:38:28.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818529+0000", 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030337631 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823849+0000", 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053193629999999997 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823903+0000", 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4076000000000002e-05 2023-06-10T21:38:28.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823951+0000", 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8161999999999997e-05 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.827891+0000", 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039400160000000002 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104527 12.9 12:9559c84e:::benchmark_data_smithi064_442287_object16477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-06-10T21:38:28.182 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:23.499833+0000", 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.4214646789999996, 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33185326399999998, 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.183 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104527 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499833+0000", 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499833+0000", 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499830+0000", 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499841+0000", 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0448999999999999e-05 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499856+0000", 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4935e-05 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.499866+0000", 2023-06-10T21:38:28.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.076e-05 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606888+0000", 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.107021557 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606896+0000", 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6380000000000004e-06 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.795717+0000", 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18882133400000001 2023-06-10T21:38:28.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.795905+0000", 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000187839 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.796065+0000", 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016063899999999999 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823781+0000", 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027715653 2023-06-10T21:38:28.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823862+0000", 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1057999999999995e-05 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.827972+0000", 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041100920000000001 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.828045+0000", 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2942999999999996e-05 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.831686+0000", 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036407309999999999 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104529 12.9 12:94108bde:::benchmark_data_smithi064_442287_object16479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e332)", 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:23.511893+0000", 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 4.4094046779999996, 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32356031400000002, 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:28.195 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104529 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:38:28.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511893+0000", 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511893+0000", 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511891+0000", 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:38:28.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511901+0000", 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0210999999999999e-05 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511915+0000", 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4279e-05 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.511927+0000", 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1430999999999999e-05 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606917+0000", 2023-06-10T21:38:28.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.09499051 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.606924+0000", 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1470000000000004e-06 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.802785+0000", 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19586076699999999 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.802988+0000", 2023-06-10T21:38:28.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000202411 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.803186+0000", 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019800199999999999 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823825+0000", 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020639595 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.823869+0000", 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3618000000000001e-05 2023-06-10T21:38:28.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.831795+0000", 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079266860000000005 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:38:28.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.831856+0000", 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0207999999999997e-05 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.835453+0000", 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035973239999999998 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.206 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:38:28.207 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:38:28.207 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:38:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:28.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:28.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:28.214 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2] out_osds: [5, 6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:38:28.214 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:38:28.214 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-06-10T21:38:28.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 5 2023-06-10T21:38:28.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:28.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:28 smithi064 ceph-mon[180297]: pgmap v1246: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 30582/213882 objects degraded (14.299%); 27336/213882 objects misplaced (12.781%); 1.5 MiB/s, 26 objects/s recovering 2023-06-10T21:38:28.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:28 smithi064 ceph-mon[186706]: pgmap v1246: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 30582/213882 objects degraded (14.299%); 27336/213882 objects misplaced (12.781%); 1.5 MiB/s, 26 objects/s recovering 2023-06-10T21:38:28.411 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:28.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:28 smithi152 ceph-mon[156819]: pgmap v1246: 89 pgs: 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 4.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 215 op/s; 30582/213882 objects degraded (14.299%); 27336/213882 objects misplaced (12.781%); 1.5 MiB/s, 26 objects/s recovering 2023-06-10T21:38:28.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:38:28.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:38:28.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:38:28.499 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:38:28.538 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:38:28.591 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:38:28.625 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:28.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:38:28.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:28.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:28.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:28.776 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:28.860 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:38:28.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.860 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:38:28.861 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:38:29.102 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2176447177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:29.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:38:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:29.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:29.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:29.362 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2176447177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2176447177' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:29.460 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:29.487 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:29.519 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:38:29.530 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:29.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:29.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:29.635 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:29.740 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:29.906 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:30.400 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:30.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:30 smithi064 ceph-mon[186706]: pgmap v1247: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 30358/208164 objects degraded (14.584%); 26639/208164 objects misplaced (12.797%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:38:30.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:30 smithi064 ceph-mon[180297]: pgmap v1247: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 30358/208164 objects degraded (14.584%); 26639/208164 objects misplaced (12.797%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:38:30.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:30.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:30 smithi152 ceph-mon[156819]: pgmap v1247: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 409 op/s; 30358/208164 objects degraded (14.584%); 26639/208164 objects misplaced (12.797%); 3.6 MiB/s, 58 objects/s recovering 2023-06-10T21:38:30.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:30.627 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:30.747 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:30.750 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:30.751 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:30.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:30.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:30.908 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:30.967 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:30.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:31.072 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:31.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:31.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 30358/208164 objects degraded (14.584%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:31.224 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:31.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:31.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:31.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:31.368 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 30358/208164 objects degraded (14.584%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:31.370 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 30358/208164 objects degraded (14.584%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:38:31.452 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:32.110 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:32.110 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:32.135 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:32 smithi064 ceph-mon[180297]: pgmap v1248: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 30358/207762 objects degraded (14.612%); 26639/207762 objects misplaced (12.822%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:32.141 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:32 smithi064 ceph-mon[186706]: pgmap v1248: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 30358/207762 objects degraded (14.612%); 26639/207762 objects misplaced (12.822%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:32.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:32 smithi152 ceph-mon[156819]: pgmap v1248: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 377 op/s; 30358/207762 objects degraded (14.612%); 26639/207762 objects misplaced (12.822%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:38:32.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:32.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:32.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:32.429 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:32.460 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:32.543 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:38:32.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:32.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:32.665 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:32.716 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:32.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:38:33.169 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:32] "GET /metrics HTTP/1.1" 200 224187 "" "Prometheus/2.33.4" 2023-06-10T21:38:33.301 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:33.559 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:33.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:33.671 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:33.701 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:33.799 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:33.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:33.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:33.932 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:33.979 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:33.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:34.037 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:34.083 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:34.110 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:34.362 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:34.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:34.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:34.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:34 smithi152 ceph-mon[156819]: pgmap v1249: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 30246/206322 objects degraded (14.660%); 26527/206322 objects misplaced (12.857%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:38:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:34.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:34.464 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:34.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:34.465 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[180297]: pgmap v1249: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 30246/206322 objects degraded (14.660%); 26527/206322 objects misplaced (12.857%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:38:34.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:34.465 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:34.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:34 smithi064 ceph-mon[186706]: pgmap v1249: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 310 op/s; 30246/206322 objects degraded (14.660%); 26527/206322 objects misplaced (12.857%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:38:34.467 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:34.495 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:34.580 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:34.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:34.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:34.714 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:34.811 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:34.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:34.898 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:34.899 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:34.916 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:35.064 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:35.223 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:38:35.263 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:35.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:35.368 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:35.394 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:38:35.397 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:38:35.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:35 smithi064 conmon[180272]: 2023-06-10T21:38:35.235+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e334 definitely_dead 0 2023-06-10T21:38:35.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:38:35.631 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:35.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:35.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:35.687 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:38:35.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:35.764 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:35.814 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:35.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:35.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:35.921 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:35.948 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:36.049 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:36.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:36.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:36.067 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:36.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:36.183 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:36.282 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.5. 2023-06-10T21:38:36.378 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:36.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:36.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:36.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:36.491 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:36.492 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:38:36.492 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: osdmap e334: 8 total, 8 up, 5 in 2023-06-10T21:38:36.492 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:38:36.492 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.492 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: pgmap v1251: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 396 op/s; 30132/202182 objects degraded (14.903%); 25870/202182 objects misplaced (12.795%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 30246/206322 objects degraded (14.660%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:36.493 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: osdmap e334: 8 total, 8 up, 5 in 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.494 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: pgmap v1251: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 396 op/s; 30132/202182 objects degraded (14.903%); 25870/202182 objects misplaced (12.795%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:38:36.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 30246/206322 objects degraded (14.660%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:36.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:36.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1586734963' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: osdmap e334: 8 total, 8 up, 5 in 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:36.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: pgmap v1251: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 78 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 396 op/s; 30132/202182 objects degraded (14.903%); 25870/202182 objects misplaced (12.795%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:38:36.506 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 30246/206322 objects degraded (14.660%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:36.513 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:37.467 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:37.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:37.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: failed to encode map e334 with expected crc 2023-06-10T21:38:37.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:38:37.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: osdmap e335: 8 total, 8 up, 6 in 2023-06-10T21:38:37.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[186706]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: failed to encode map e334 with expected crc 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: osdmap e335: 8 total, 8 up, 6 in 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:37 smithi064 ceph-mon[180297]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.527 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: failed to encode map e334 with expected crc 2023-06-10T21:38:37.527 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/517219403' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:38:37.527 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: osdmap e335: 8 total, 8 up, 6 in 2023-06-10T21:38:37.528 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.528 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.528 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:37 smithi152 ceph-mon[156819]: failed to encode map e335 with expected crc 2023-06-10T21:38:37.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:37.613 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:37.670 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:37.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:37.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:37.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:37.801 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:37.802 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:37.806 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e335 with expected crc 2023-06-10T21:38:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: osdmap e336: 8 total, 8 up, 6 in 2023-06-10T21:38:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 334'6847 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6922 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6709 2023-06-10T21:38:38.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[186706]: pgmap v1254: 89 pgs: 2 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 325 op/s; 26145/201897 objects degraded (12.950%); 29798/201897 objects misplaced (14.759%); 4.4 MiB/s, 69 objects/s recovering 2023-06-10T21:38:38.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e335 with expected crc 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: osdmap e336: 8 total, 8 up, 6 in 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 334'6847 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6922 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6709 2023-06-10T21:38:38.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:38 smithi064 ceph-mon[180297]: pgmap v1254: 89 pgs: 2 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 325 op/s; 26145/201897 objects degraded (12.950%); 29798/201897 objects misplaced (14.759%); 4.4 MiB/s, 69 objects/s recovering 2023-06-10T21:38:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e335 with expected crc 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: osdmap e336: 8 total, 8 up, 6 in 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: 12.e starting backfill to osd.5 from (0'0,0'0] MAX to 334'6847 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: 12.3 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6922 2023-06-10T21:38:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: failed to encode map e336 with expected crc 2023-06-10T21:38:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: 12.1 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6709 2023-06-10T21:38:38.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:38 smithi152 ceph-mon[156819]: pgmap v1254: 89 pgs: 2 peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 76 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 325 op/s; 26145/201897 objects degraded (12.950%); 29798/201897 objects misplaced (14.759%); 4.4 MiB/s, 69 objects/s recovering 2023-06-10T21:38:38.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:38:39.376 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:39.395 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:39.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:39.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:39.506 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-06-10T21:38:39.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:38:39.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: 12.f starting backfill to osd.5 from (0'0,0'0] MAX to 334'6856 2023-06-10T21:38:39.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: 12.9 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6867 2023-06-10T21:38:39.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6753 2023-06-10T21:38:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: osdmap e337: 8 total, 8 up, 6 in 2023-06-10T21:38:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: 12.c starting backfill to osd.5 from (0'0,0'0] MAX to 334'7035 2023-06-10T21:38:39.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:39 smithi152 ceph-mon[156819]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.540 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: 12.f starting backfill to osd.5 from (0'0,0'0] MAX to 334'6856 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: 12.9 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6867 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6753 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: osdmap e337: 8 total, 8 up, 6 in 2023-06-10T21:38:39.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: 12.c starting backfill to osd.5 from (0'0,0'0] MAX to 334'7035 2023-06-10T21:38:39.571 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[186706]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.575 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: 12.f starting backfill to osd.5 from (0'0,0'0] MAX to 334'6856 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: 12.9 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6867 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: 12.0 starting backfill to osd.5 from (0'0,0'0] MAX to 334'6753 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: osdmap e337: 8 total, 8 up, 6 in 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: 12.c starting backfill to osd.5 from (0'0,0'0] MAX to 334'7035 2023-06-10T21:38:39.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:39 smithi064 ceph-mon[180297]: failed to encode map e337 with expected crc 2023-06-10T21:38:39.606 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:39.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:39.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:39.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:39.842 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:39.922 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:38:39.964 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:39.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:39.983 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:38:40.037 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:38:40.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:38:40.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:40.114 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:40.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:38:40.173 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:40.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:40.181 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:38:40.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:38:40.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:38:40.280 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:40.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2703834315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:40.287 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:40 smithi064 ceph-mon[180297]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 22338/198798 objects degraded (11.237%); 31559/198798 objects misplaced (15.875%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:38:40.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.307 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.308 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.308 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.308 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.308 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:38:40.311 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:40.366 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:40.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:40.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2703834315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:40 smithi152 ceph-mon[156819]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 22338/198798 objects degraded (11.237%); 31559/198798 objects misplaced (15.875%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:38:40.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:40.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2703834315' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:40.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:40 smithi064 ceph-mon[186706]: pgmap v1256: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 234 op/s; 22338/198798 objects degraded (11.237%); 31559/198798 objects misplaced (15.875%); 14 MiB/s, 232 objects/s recovering 2023-06-10T21:38:40.570 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:40.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:40.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:40.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:40.926 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:41.092 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:41.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:41.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:41.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:41.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:41.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:41.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:41.408 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:41.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:41.494 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22338/198798 objects degraded (11.237%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:41.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:41.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:41.496 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22338/198798 objects degraded (11.237%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:41.509 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22338/198798 objects degraded (11.237%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:41.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:41.635 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:41.712 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:41.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:41.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:41.777 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:38:41.778 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:38:41.778 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:38:41.778 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:38:41.820 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:41.854 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:42.024 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:42.025 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:42.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:42.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:42.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:38:42.160 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:42.212 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:38:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:42.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:38:42.313 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:42 smithi064 ceph-mon[180297]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 22338/198552 objects degraded (11.250%); 31351/198552 objects misplaced (15.790%); 14 MiB/s, 220 objects/s recovering 2023-06-10T21:38:42.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:42.413 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:42.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:42 smithi152 ceph-mon[156819]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 22338/198552 objects degraded (11.250%); 31351/198552 objects misplaced (15.790%); 14 MiB/s, 220 objects/s recovering 2023-06-10T21:38:42.552 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:42.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:42.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:42.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:42 smithi064 ceph-mon[186706]: pgmap v1257: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 22338/198552 objects degraded (11.250%); 31351/198552 objects misplaced (15.790%); 14 MiB/s, 220 objects/s recovering 2023-06-10T21:38:42.663 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:42.695 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:42.750 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:42.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:42.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:42.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:42.950 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:43.118 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:43.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:43.135 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:42] "GET /metrics HTTP/1.1" 200 224225 "" "Prometheus/2.33.4" 2023-06-10T21:38:43.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:43.259 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:43.288 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:38:43.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:43.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:43.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:43.435 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:43.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:43.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:43.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:43.730 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:43.749 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:43.983 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:43.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:44.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:38:44.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:44.222 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:44.338 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:44 smithi152 ceph-mon[156819]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 22233/197277 objects degraded (11.270%); 31130/197277 objects misplaced (15.780%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:38:44.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:44 smithi064 ceph-mon[186706]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 22233/197277 objects degraded (11.270%); 31130/197277 objects misplaced (15.780%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:38:44.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:44 smithi064 ceph-mon[180297]: pgmap v1258: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 22233/197277 objects degraded (11.270%); 31130/197277 objects misplaced (15.780%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:38:44.466 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:44.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:44.507 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:38:44.507 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:38:44.508 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2023-06-10T21:38:44.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:38:44.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:44.684 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:44.869 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:44.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:44.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:45.024 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:45.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:45.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:45.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:45.106 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:45.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:45.215 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:45.268 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:45.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:45.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:45.403 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:45.459 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:45.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:45.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:45.486 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:38:45.541 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:45.564 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:45.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:45.596 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:45.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:45.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:38:45.777 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:45.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:45.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:38:45.921 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:46.003 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:38:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:46.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:38:46.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:46.299 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:46.313 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:38:46.313 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:38:46.313 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:38:46.313 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:38:46.314 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:38:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: osdmap e338: 8 total, 8 up, 6 in 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21930/193716 objects degraded (11.321%); 28325/193716 objects misplaced (14.622%); 32 MiB/s, 512 objects/s recovering 2023-06-10T21:38:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22233/197277 objects degraded (11.270%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: osdmap e338: 8 total, 8 up, 6 in 2023-06-10T21:38:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21930/193716 objects degraded (11.321%); 28325/193716 objects misplaced (14.622%); 32 MiB/s, 512 objects/s recovering 2023-06-10T21:38:46.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22233/197277 objects degraded (11.270%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3038004758' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:38:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: osdmap e338: 8 total, 8 up, 6 in 2023-06-10T21:38:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: failed to encode map e338 with expected crc 2023-06-10T21:38:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: pgmap v1260: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 21930/193716 objects degraded (11.321%); 28325/193716 objects misplaced (14.622%); 32 MiB/s, 512 objects/s recovering 2023-06-10T21:38:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22233/197277 objects degraded (11.270%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:47.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:38:47.972 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:48.129 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:48.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:48.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:48.345 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[180297]: osdmap e339: 8 total, 8 up, 6 in 2023-06-10T21:38:48.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[180297]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 21930/193410 objects degraded (11.339%); 28189/193410 objects misplaced (14.575%); 25 MiB/s, 397 objects/s recovering 2023-06-10T21:38:48.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[186706]: osdmap e339: 8 total, 8 up, 6 in 2023-06-10T21:38:48.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.348 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:48 smithi064 ceph-mon[186706]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 21930/193410 objects degraded (11.339%); 28189/193410 objects misplaced (14.575%); 25 MiB/s, 397 objects/s recovering 2023-06-10T21:38:48.381 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:48.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:48 smithi152 ceph-mon[156819]: osdmap e339: 8 total, 8 up, 6 in 2023-06-10T21:38:48.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:48 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:48 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:48.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:48 smithi152 ceph-mon[156819]: pgmap v1262: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 77 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 21930/193410 objects degraded (11.339%); 28189/193410 objects misplaced (14.575%); 25 MiB/s, 397 objects/s recovering 2023-06-10T21:38:48.749 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:48.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:48.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:48.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:49.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:49.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:49.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:49.183 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: osdmap e340: 8 total, 8 up, 6 in 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.184 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.185 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: osdmap e340: 8 total, 8 up, 6 in 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:49.186 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:49.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:49.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:49.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:49.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:49.423 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e339 with expected crc 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: osdmap e340: 8 total, 8 up, 6 in 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: failed to encode map e340 with expected crc 2023-06-10T21:38:49.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:38:49.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:38:49.451 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:49.482 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:49.535 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:49.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:49.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:49.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:49.905 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:38:49.905 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:38:49.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:38:49.906 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:38:49.906 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.906 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110270 12.1 e337/335 12:80f8c927:::benchmark_data_smithi064_442287_object22220:head v 337'6768, mlcod=337'6765)", 2023-06-10T21:38:49.907 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.310712+0000", 2023-06-10T21:38:49.907 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.5896521660000005, 2023-06-10T21:38:49.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089504449999999996, 2023-06-10T21:38:49.907 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310712+0000", 2023-06-10T21:38:49.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310712+0000", 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.909 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310712+0000", 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6210000000000004e-06 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310719+0000", 2023-06-10T21:38:49.910 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1214e-05 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310730+0000", 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5339999999999999e-06 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310736+0000", 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5429000000000002e-05 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310781+0000", 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0451000000000001e-05 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.310802+0000", 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088329250000000002 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.319635+0000", 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7898999999999998e-05 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.319663+0000", 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089504449999999996 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110466 12.3 e337/335 12:c841d5ee:::benchmark_data_smithi064_442287_object22416:head v 337'6975, mlcod=337'6973)", 2023-06-10T21:38:49.915 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.808206+0000", 2023-06-10T21:38:49.916 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0921583600000009, 2023-06-10T21:38:49.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083673833000000003, 2023-06-10T21:38:49.916 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.916 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.916 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808206+0000", 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808206+0000", 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808206+0000", 2023-06-10T21:38:49.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6500000000000004e-06 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808210+0000", 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7490000000000004e-06 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808216+0000", 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2540000000000001e-06 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808219+0000", 2023-06-10T21:38:49.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8945000000000001e-05 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808258+0000", 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0072e-05 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.808278+0000", 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083564271999999995 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.922 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891843+0000", 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7323999999999997e-05 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891880+0000", 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083673833000000003 2023-06-10T21:38:49.923 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110453 12.9 e337/336 12:9ba27293:::benchmark_data_smithi064_442287_object22403:head v 337'6919, mlcod=316'1390)", 2023-06-10T21:38:49.924 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.826389+0000", 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0739754240000003, 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.071210829000000003, 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.826389+0000", 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.826389+0000", 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:38:49.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.826388+0000", 2023-06-10T21:38:49.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8200000000000008e-06 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.826398+0000", 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.065429184000000001 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891827+0000", 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3200000000000001e-05 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891840+0000", 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4369000000000002e-05 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891884+0000", 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2289999999999998e-05 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891907+0000", 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056674999999999998 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897574+0000", 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5565999999999999e-05 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897600+0000", 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.071210829000000003 2023-06-10T21:38:49.932 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110472 12.3 e337/335 12:ce82edd4:::benchmark_data_smithi064_442287_object22422:head v 337'6976, mlcod=337'6973)", 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.830182+0000", 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0701827809999997, 2023-06-10T21:38:49.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.064866579999999993, 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.830182+0000", 2023-06-10T21:38:49.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.830182+0000", 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.830181+0000", 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0710000000000001e-06 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.830190+0000", 2023-06-10T21:38:49.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.061660903000000003 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891851+0000", 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2810000000000008e-06 2023-06-10T21:38:49.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891859+0000", 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6675999999999998e-05 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891905+0000", 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.878e-05 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891924+0000", 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030849359999999999 2023-06-10T21:38:49.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.895009+0000", 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8983999999999998e-05 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.895048+0000", 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.064866579999999993 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110483 12.1 e337/335 12:8a943759:::benchmark_data_smithi064_442287_object22433:head v 337'6784, mlcod=337'6782)", 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.868948+0000", 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0314160010000002, 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028623454999999999, 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.942 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.868948+0000", 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.868948+0000", 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.868947+0000", 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2717e-05 2023-06-10T21:38:49.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.868960+0000", 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.022872809000000001 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891833+0000", 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0505e-05 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891843+0000", 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5226999999999998e-05 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891898+0000", 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1078999999999999e-05 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.947 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891920+0000", 2023-06-10T21:38:49.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056170949999999999 2023-06-10T21:38:49.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897537+0000", 2023-06-10T21:38:49.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5179000000000003e-05 2023-06-10T21:38:49.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897572+0000", 2023-06-10T21:38:49.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.028623454999999999 2023-06-10T21:38:49.951 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.951 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.951 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.952 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110460 12.0 e337/336 12:080dc56e:::benchmark_data_smithi064_442287_object22410:head v 337'6804, mlcod=316'1358)", 2023-06-10T21:38:49.952 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.891905+0000", 2023-06-10T21:38:49.952 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0084594839999994, 2023-06-10T21:38:49.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096378230000000002, 2023-06-10T21:38:49.953 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.953 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.953 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891905+0000", 2023-06-10T21:38:49.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891905+0000", 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891904+0000", 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4669999999999998e-06 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891909+0000", 2023-06-10T21:38:49.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9599999999999997e-06 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891915+0000", 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3290000000000004e-06 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891919+0000", 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00019321799999999999 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.958 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892112+0000", 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8539999999999999e-05 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892131+0000", 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093809140000000006 2023-06-10T21:38:49.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901512+0000", 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0837999999999998e-05 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901543+0000", 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096378230000000002 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110462 12.0 e337/336 12:0b371683:::benchmark_data_smithi064_442287_object22412:head v 337'6805, mlcod=316'1358)", 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.891931+0000", 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0084335220000007, 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096385399999999993, 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.962 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891931+0000", 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891931+0000", 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891930+0000", 2023-06-10T21:38:49.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.493e-06 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891935+0000", 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4500000000000003e-06 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891940+0000", 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8249999999999998e-06 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891944+0000", 2023-06-10T21:38:49.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028427209999999999 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.894787+0000", 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.143e-05 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.894808+0000", 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067369159999999999 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901545+0000", 2023-06-10T21:38:49.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4155000000000002e-05 2023-06-10T21:38:49.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901569+0000", 2023-06-10T21:38:49.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096385399999999993 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110465 12.0 e337/336 12:0fab8df0:::benchmark_data_smithi064_442287_object22415:head v 337'6806, mlcod=316'1358)", 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.891999+0000", 2023-06-10T21:38:49.970 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0083652109999992, 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095994389999999995, 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891999+0000", 2023-06-10T21:38:49.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891999+0000", 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.891999+0000", 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0219999999999998e-06 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892003+0000", 2023-06-10T21:38:49.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3700000000000003e-06 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892008+0000", 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2100000000000002e-06 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892012+0000", 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054394650000000001 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897451+0000", 2023-06-10T21:38:49.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8958e-05 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.897470+0000", 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041013619999999999 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901571+0000", 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7263e-05 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901599+0000", 2023-06-10T21:38:49.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095994389999999995 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:110487 12.0 e337/336 12:04442309:::benchmark_data_smithi064_442287_object22437:head v 337'6807, mlcod=316'1358)", 2023-06-10T21:38:49.978 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:40.892035+0000", 2023-06-10T21:38:49.979 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 9.0083295200000002, 2023-06-10T21:38:49.979 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096312419999999999, 2023-06-10T21:38:49.979 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.979 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.979 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892035+0000", 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892035+0000", 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892035+0000", 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9359999999999999e-06 2023-06-10T21:38:49.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892038+0000", 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6990000000000002e-06 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892042+0000", 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9529999999999999e-06 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.892045+0000", 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078781690000000008 2023-06-10T21:38:49.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.899923+0000", 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5187999999999999e-05 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.899939+0000", 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017114470000000001 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901650+0000", 2023-06-10T21:38:49.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6030000000000001e-05 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:40.901666+0000", 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096312419999999999 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.986 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112013 12.9 e338/336 12:92675e63:::benchmark_data_smithi064_442287_object23963:head v 338'7009, mlcod=316'1479)", 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:45.599403+0000", 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4.3009610509999998, 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085922689999999996, 2023-06-10T21:38:49.987 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599403+0000", 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599403+0000", 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599402+0000", 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0458999999999999e-05 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599413+0000", 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6299000000000001e-05 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599429+0000", 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7340000000000002e-06 2023-06-10T21:38:49.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599436+0000", 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3316000000000002e-05 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:49.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599499+0000", 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0168999999999999e-05 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.599529+0000", 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0084201970000000008 2023-06-10T21:38:49.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.607950+0000", 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5945000000000001e-05 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:45.607996+0000", 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085922689999999996 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.995 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "pg_backfill(progress 12.3 e 338/338 lb 12:c145284f:::benchmark_data_smithi064_442287_object40547:head)", 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:46.950702+0000", 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.9496621420000002, 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16255845599999999, 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:49.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950702+0000", 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950702+0000", 2023-06-10T21:38:49.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950702+0000", 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0110000000000002e-06 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.998 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950706+0000", 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0960000000000003e-06 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950710+0000", 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.689e-06 2023-06-10T21:38:49.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:46.950713+0000", 2023-06-10T21:38:50.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16248891600000001 2023-06-10T21:38:50.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113202+0000", 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.17e-05 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113213+0000", 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7345000000000002e-05 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113261+0000", 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16255845599999999 2023-06-10T21:38:50.002 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112436 12.f e339/336 12:ff4cc2a7:::benchmark_data_smithi064_442287_object24386:head v 339'7023, mlcod=316'1490)", 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.113449+0000", 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7869154260000002, 2023-06-10T21:38:50.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016130979, 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113449+0000", 2023-06-10T21:38:50.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113449+0000", 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113449+0000", 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4499999999999997e-06 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113453+0000", 2023-06-10T21:38:50.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3289999999999999e-06 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113459+0000", 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9049999999999999e-06 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113463+0000", 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043894939999999999 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.117853+0000", 2023-06-10T21:38:50.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3699000000000001e-05 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.117877+0000", 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011667049000000001 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129544+0000", 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6334000000000001e-05 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129580+0000", 2023-06-10T21:38:50.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016130979 2023-06-10T21:38:50.011 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.011 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.011 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112440 12.f e339/336 12:f6ef8f8c:::benchmark_data_smithi064_442287_object24390:head v 339'7024, mlcod=316'1490)", 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.113468+0000", 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7868960880000002, 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016145459000000001, 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.012 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113468+0000", 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113468+0000", 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113468+0000", 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8990000000000003e-06 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113472+0000", 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7419999999999997e-06 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113477+0000", 2023-06-10T21:38:50.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1250000000000003e-06 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113481+0000", 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064623390000000001 2023-06-10T21:38:50.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119943+0000", 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6946e-05 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119960+0000", 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096268559999999996 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129587+0000", 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6747999999999999e-05 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129614+0000", 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016145459000000001 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112446 12.f e339/336 12:f7192498:::benchmark_data_smithi064_442287_object24396:head v 339'7025, mlcod=316'1490)", 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.113485+0000", 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7868795770000001, 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016154175999999999, 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.020 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113485+0000", 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113485+0000", 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.022 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113485+0000", 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.024e-06 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113492+0000", 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6909999999999996e-06 2023-06-10T21:38:50.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113496+0000", 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4000000000000001e-06 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113500+0000", 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088225300000000003 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.122322+0000", 2023-06-10T21:38:50.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9458000000000002e-05 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.122352+0000", 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072689089999999996 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129621+0000", 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8355999999999998e-05 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129639+0000", 2023-06-10T21:38:50.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016154175999999999 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112447 12.f e339/336 12:f1811aef:::benchmark_data_smithi064_442287_object24397:head v 339'7026, mlcod=316'1490)", 2023-06-10T21:38:50.028 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.113528+0000", 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7868365900000001, 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016137008000000001, 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113528+0000", 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113528+0000", 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:50.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113527+0000", 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6160000000000004e-06 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113532+0000", 2023-06-10T21:38:50.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.28e-06 2023-06-10T21:38:50.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.032 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.032 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113538+0000", 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0609999999999997e-06 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.113542+0000", 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011248009 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124790+0000", 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3626e-05 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124814+0000", 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048283429999999997 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129642+0000", 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2476e-05 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129665+0000", 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016137008000000001 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112456 12.f e339/336 12:f321521b:::benchmark_data_smithi064_442287_object24406:head v 339'7027, mlcod=316'1490)", 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.115568+0000", 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7847968010000002, 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015388195, 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.037 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115568+0000", 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115568+0000", 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115567+0000", 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5719999999999999e-06 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115574+0000", 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4050000000000006e-06 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115583+0000", 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6529999999999999e-06 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.115589+0000", 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012429315999999999 2023-06-10T21:38:50.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.128018+0000", 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6395000000000001e-05 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.128034+0000", 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028889839999999998 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.043 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.130923+0000", 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2509999999999999e-05 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.130956+0000", 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015388195 2023-06-10T21:38:50.044 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112462 12.f e339/336 12:f95e1ca6:::benchmark_data_smithi064_442287_object24412:head v 339'7028, mlcod=316'1490)", 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.124539+0000", 2023-06-10T21:38:50.045 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.7758252099999998, 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091149390000000007, 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124539+0000", 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124539+0000", 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124538+0000", 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5869999999999993e-06 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124547+0000", 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3716e-05 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124561+0000", 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7180000000000001e-06 2023-06-10T21:38:50.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.124567+0000", 2023-06-10T21:38:50.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051218369999999997 2023-06-10T21:38:50.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129688+0000", 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6212999999999999e-05 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.129705+0000", 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039137269999999997 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.133618+0000", 2023-06-10T21:38:50.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5818999999999997e-05 2023-06-10T21:38:50.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.133654+0000", 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091149390000000007 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112960 12.e e340/335 12:70ee2228:::benchmark_data_smithi064_442287_object24910:head v 340'7040, mlcod=0'0)", 2023-06-10T21:38:50.054 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:48.928856+0000", 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.97150870600000006, 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.102426188, 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.055 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928856+0000", 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928856+0000", 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:38:50.056 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928854+0000", 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1763000000000001e-05 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.057 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928866+0000", 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7676000000000002e-05 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928884+0000", 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5300000000000002e-06 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.058 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.928893+0000", 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107054 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.929000+0000", 2023-06-10T21:38:50.059 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0106999999999999e-05 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.929020+0000", 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001081156 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.060 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930102+0000", 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101180305 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031282+0000", 2023-06-10T21:38:50.061 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.102426188 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.062 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "pg_backfill(progress 12.3 e 340/340 lb 12:c303602f:::benchmark_data_smithi064_442287_object68268:head)", 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:48.930096+0000", 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.970268837, 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101259219, 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:38:50.063 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930096+0000", 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.064 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930096+0000", 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930095+0000", 2023-06-10T21:38:50.065 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5440000000000004e-06 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930101+0000", 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.156e-06 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.066 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.930108+0000", 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101155723 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031263+0000", 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9206999999999998e-05 2023-06-10T21:38:50.067 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031313+0000", 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0115000000000001e-05 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.068 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031323+0000", 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2073999999999998e-05 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031355+0000", 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.101259219 2023-06-10T21:38:50.069 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.070 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.070 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.070 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.070 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:112983 12.0 e340/336 12:046288e5:::benchmark_data_smithi064_442287_object24933:head v 340'6972, mlcod=316'1526)", 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:48.943832+0000", 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.95653275500000001, 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090178770000000005, 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:38:50.071 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:38:50.072 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:38:50.072 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.072 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:38:50.072 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.943832+0000", 2023-06-10T21:38:50.073 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:38:50.073 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.073 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.073 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:38:50.074 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.943832+0000", 2023-06-10T21:38:50.074 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:38:50.074 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.075 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.075 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:38:50.075 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.943830+0000", 2023-06-10T21:38:50.075 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4953000000000001e-05 2023-06-10T21:38:50.076 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.076 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.076 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:38:50.077 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.943845+0000", 2023-06-10T21:38:50.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.087418170000000003 2023-06-10T21:38:50.077 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.078 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:38:50.078 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031263+0000", 2023-06-10T21:38:50.078 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1392999999999999e-05 2023-06-10T21:38:50.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:38:50.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031274+0000", 2023-06-10T21:38:50.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6455000000000002e-05 2023-06-10T21:38:50.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:38:50.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031321+0000", 2023-06-10T21:38:50.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9663000000000001e-05 2023-06-10T21:38:50.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:38:50.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.031341+0000", 2023-06-10T21:38:50.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026228979999999998 2023-06-10T21:38:50.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.033963+0000", 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6971000000000001e-05 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:49.034010+0000", 2023-06-10T21:38:50.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.090178770000000005 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:38:50.084 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:38:50.211 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:50.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:50.331 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:50.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:50 smithi064 ceph-mon[186706]: pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 443 op/s; 21575/189291 objects degraded (11.398%); 25917/189291 objects misplaced (13.692%); 52 MiB/s, 834 objects/s recovering 2023-06-10T21:38:50.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:50 smithi064 ceph-mon[180297]: pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 443 op/s; 21575/189291 objects degraded (11.398%); 25917/189291 objects misplaced (13.692%); 52 MiB/s, 834 objects/s recovering 2023-06-10T21:38:50.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:50 smithi152 ceph-mon[156819]: pgmap v1264: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 443 op/s; 21575/189291 objects degraded (11.398%); 25917/189291 objects misplaced (13.692%); 52 MiB/s, 834 objects/s recovering 2023-06-10T21:38:50.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:50.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:50.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:50.610 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:38:50.656 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:50.734 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:38:50.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:38:50.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:50.837 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:38:50.874 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:38:50.905 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:50.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:50.917 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:38:50.963 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:38:51.016 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:51.023 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:38:51.046 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:51.063 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:38:51.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:51.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:51.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2069092345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:51.119 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21575/189291 objects degraded (11.398%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:51.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:38:51.136 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.136 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.136 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:38:51.220 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:51.286 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:51.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:51.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2069092345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:51.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21575/189291 objects degraded (11.398%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2069092345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:38:51.415 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21575/189291 objects degraded (11.398%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:51.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:51.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:51.624 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:38:51.664 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:38:51.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:51.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:51.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:52.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:52.099 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:52.108 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:52 smithi064 ceph-mon[180297]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:38:52.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:52 smithi064 ceph-mon[180297]: pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 21575/189033 objects degraded (11.413%); 25722/189033 objects misplaced (13.607%); 23 MiB/s, 375 objects/s recovering 2023-06-10T21:38:52.147 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:52.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:52.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:52.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:52.279 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:52 smithi064 ceph-mon[186706]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:38:52.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:52 smithi064 ceph-mon[186706]: pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 21575/189033 objects degraded (11.413%); 25722/189033 objects misplaced (13.607%); 23 MiB/s, 375 objects/s recovering 2023-06-10T21:38:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:52 smithi152 ceph-mon[156819]: from='client.56452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:38:52.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:52 smithi152 ceph-mon[156819]: pgmap v1265: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 236 op/s; 21575/189033 objects degraded (11.413%); 25722/189033 objects misplaced (13.607%); 23 MiB/s, 375 objects/s recovering 2023-06-10T21:38:53.108 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:53.108 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:53.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:38:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:38:52] "GET /metrics HTTP/1.1" 200 224253 "" "Prometheus/2.33.4" 2023-06-10T21:38:53.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:53.228 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:53.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:53.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:53.428 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:53.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:38:54.159 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:54.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:54.249 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:54 smithi064 ceph-mon[180297]: pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 239 op/s; 21472/187971 objects degraded (11.423%); 24704/187971 objects misplaced (13.142%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:38:54.249 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:54 smithi064 ceph-mon[186706]: pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 239 op/s; 21472/187971 objects degraded (11.423%); 24704/187971 objects misplaced (13.142%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:38:54.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:54.275 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1264,"stamp":"2023-06-10T21:38:50.018759+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4134082470,"num_objects":63097,"num_object_clones":0,"num_object_copies":189291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21575,"num_objects_misplaced":25917,"num_objects_unfound":0,"num_objects_dirty":63097,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":113284,"num_write_kb":5637455,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9222,"num_bytes_recovered":601045210,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84853,"ondisk_log_size":84853,"up":267,"acting":265,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14786260,"kb_used_data":11395108,"kb_used_omap":30,"kb_used_meta":3391009,"kb_avail":547684652,"statfs":{"total":575970213888,"available":560829083648,"internally_reserved":0,"allocated":11668590592,"data_stored":11662950614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31706,"internal_metadata":3472393254},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,2,0,2,1,0,0,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-174456832,"num_objects":-2662,"num_object_clones":0,"num_object_copies":-7986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-658,"num_objects_misplaced":-5213,"num_objects_unfound":0,"num_objects_dirty":-2662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2662,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5011,"num_bytes_recovered":328400896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002252"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":344,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058566+0000","last_change":"2023-06-10T21:37:33.031289+0000","last_active":"2023-06-10T21:38:47.058566+0000","last_peered":"2023-06-10T21:38:47.058566+0000","last_clean":"2023-06-10T21:38:47.058566+0000","last_became_active":"2023-06-10T21:37:33.030930+0000","last_became_peered":"2023-06-10T21:37:33.030930+0000","last_unstale":"2023-06-10T21:38:47.058566+0000","last_undegraded":"2023-06-10T21:38:47.058566+0000","last_fullsized":"2023-06-10T21:38:47.058566+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:51:14.255181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116880+0000","last_change":"2023-06-10T21:38:37.275718+0000","last_active":"2023-06-10T21:38:47.116880+0000","last_peered":"2023-06-10T21:38:47.116880+0000","last_clean":"2023-06-10T21:38:47.116880+0000","last_became_active":"2023-06-10T21:38:37.274832+0000","last_became_peered":"2023-06-10T21:38:37.274832+0000","last_unstale":"2023-06-10T21:38:47.116880+0000","last_undegraded":"2023-06-10T21:38:47.116880+0000","last_fullsized":"2023-06-10T21:38:47.116880+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":258,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.273374+0000","last_change":"2023-06-10T21:37:33.029694+0000","last_active":"2023-06-10T21:38:38.273374+0000","last_peered":"2023-06-10T21:38:38.273374+0000","last_clean":"2023-06-10T21:38:38.273374+0000","last_became_active":"2023-06-10T21:37:33.029367+0000","last_became_peered":"2023-06-10T21:37:33.029367+0000","last_unstale":"2023-06-10T21:38:38.273374+0000","last_undegraded":"2023-06-10T21:38:38.273374+0000","last_fullsized":"2023-06-10T21:38:38.273374+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:39:58.502583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":208,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.086150+0000","last_change":"2023-06-10T21:38:22.232678+0000","last_active":"2023-06-10T21:38:47.086150+0000","last_peered":"2023-06-10T21:38:47.086150+0000","last_clean":"2023-06-10T21:38:47.086150+0000","last_became_active":"2023-06-10T21:38:22.232397+0000","last_became_peered":"2023-06-10T21:38:22.232397+0000","last_unstale":"2023-06-10T21:38:47.086150+0000","last_undegraded":"2023-06-10T21:38:47.086150+0000","last_fullsized":"2023-06-10T21:38:47.086150+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:21:31.860905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":235,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.080332+0000","last_change":"2023-06-10T21:38:22.276292+0000","last_active":"2023-06-10T21:38:47.080332+0000","last_peered":"2023-06-10T21:38:47.080332+0000","last_clean":"2023-06-10T21:38:47.080332+0000","last_became_active":"2023-06-10T21:38:22.275373+0000","last_became_peered":"2023-06-10T21:38:22.275373+0000","last_unstale":"2023-06-10T21:38:47.080332+0000","last_undegraded":"2023-06-10T21:38:47.080332+0000","last_fullsized":"2023-06-10T21:38:47.080332+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:23:37.345308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.119969+0000","last_change":"2023-06-10T21:38:37.276136+0000","last_active":"2023-06-10T21:38:47.119969+0000","last_peered":"2023-06-10T21:38:47.119969+0000","last_clean":"2023-06-10T21:38:47.119969+0000","last_became_active":"2023-06-10T21:38:37.274298+0000","last_became_peered":"2023-06-10T21:38:37.274298+0000","last_unstale":"2023-06-10T21:38:47.119969+0000","last_undegraded":"2023-06-10T21:38:47.119969+0000","last_fullsized":"2023-06-10T21:38:47.119969+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081284+0000","last_change":"2023-06-10T21:38:37.284899+0000","last_active":"2023-06-10T21:38:47.081284+0000","last_peered":"2023-06-10T21:38:47.081284+0000","last_clean":"2023-06-10T21:38:47.081284+0000","last_became_active":"2023-06-10T21:38:37.283647+0000","last_became_peered":"2023-06-10T21:38:37.283647+0000","last_unstale":"2023-06-10T21:38:47.081284+0000","last_undegraded":"2023-06-10T21:38:47.081284+0000","last_fullsized":"2023-06-10T21:38:47.081284+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:28:21.990548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116552+0000","last_change":"2023-06-10T21:38:37.273936+0000","last_active":"2023-06-10T21:38:47.116552+0000","last_peered":"2023-06-10T21:38:47.116552+0000","last_clean":"2023-06-10T21:38:47.116552+0000","last_became_active":"2023-06-10T21:38:37.273477+0000","last_became_peered":"2023-06-10T21:38:37.273477+0000","last_unstale":"2023-06-10T21:38:47.116552+0000","last_undegraded":"2023-06-10T21:38:47.116552+0000","last_fullsized":"2023-06-10T21:38:47.116552+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.117016+0000","last_change":"2023-06-10T21:38:37.274583+0000","last_active":"2023-06-10T21:38:47.117016+0000","last_peered":"2023-06-10T21:38:47.117016+0000","last_clean":"2023-06-10T21:38:47.117016+0000","last_became_active":"2023-06-10T21:38:37.274248+0000","last_became_peered":"2023-06-10T21:38:37.274248+0000","last_unstale":"2023-06-10T21:38:47.117016+0000","last_undegraded":"2023-06-10T21:38:47.117016+0000","last_fullsized":"2023-06-10T21:38:47.117016+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.273235+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:38:38.273235+0000","last_peered":"2023-06-10T21:38:38.273235+0000","last_clean":"2023-06-10T21:38:38.273235+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:38:38.273235+0000","last_undegraded":"2023-06-10T21:38:38.273235+0000","last_fullsized":"2023-06-10T21:38:38.273235+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":82,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121574+0000","last_change":"2023-06-10T21:37:33.585956+0000","last_active":"2023-06-10T21:38:47.121574+0000","last_peered":"2023-06-10T21:38:47.121574+0000","last_clean":"2023-06-10T21:38:47.121574+0000","last_became_active":"2023-06-10T21:37:33.034733+0000","last_became_peered":"2023-06-10T21:37:33.034733+0000","last_unstale":"2023-06-10T21:38:47.121574+0000","last_undegraded":"2023-06-10T21:38:47.121574+0000","last_fullsized":"2023-06-10T21:38:47.121574+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058493+0000","last_change":"2023-06-10T21:28:34.508824+0000","last_active":"2023-06-10T21:38:47.058493+0000","last_peered":"2023-06-10T21:38:47.058493+0000","last_clean":"2023-06-10T21:38:47.058493+0000","last_became_active":"2023-06-10T21:28:34.508511+0000","last_became_peered":"2023-06-10T21:28:34.508511+0000","last_unstale":"2023-06-10T21:38:47.058493+0000","last_undegraded":"2023-06-10T21:38:47.058493+0000","last_fullsized":"2023-06-10T21:38:47.058493+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:42:08.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":229,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121574+0000","last_change":"2023-06-10T21:37:33.406411+0000","last_active":"2023-06-10T21:38:47.121574+0000","last_peered":"2023-06-10T21:38:47.121574+0000","last_clean":"2023-06-10T21:38:47.121574+0000","last_became_active":"2023-06-10T21:37:33.032211+0000","last_became_peered":"2023-06-10T21:37:33.032211+0000","last_unstale":"2023-06-10T21:38:47.121574+0000","last_undegraded":"2023-06-10T21:38:47.121574+0000","last_fullsized":"2023-06-10T21:38:47.121574+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":93,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121145+0000","last_change":"2023-06-10T21:38:37.277220+0000","last_active":"2023-06-10T21:38:47.121145+0000","last_peered":"2023-06-10T21:38:47.121145+0000","last_clean":"2023-06-10T21:38:47.121145+0000","last_became_active":"2023-06-10T21:38:37.270008+0000","last_became_peered":"2023-06-10T21:38:37.270008+0000","last_unstale":"2023-06-10T21:38:47.121145+0000","last_undegraded":"2023-06-10T21:38:47.121145+0000","last_fullsized":"2023-06-10T21:38:47.121145+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":354,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121166+0000","last_change":"2023-06-10T21:37:33.135804+0000","last_active":"2023-06-10T21:38:47.121166+0000","last_peered":"2023-06-10T21:38:47.121166+0000","last_clean":"2023-06-10T21:38:47.121166+0000","last_became_active":"2023-06-10T21:37:33.023662+0000","last_became_peered":"2023-06-10T21:37:33.023662+0000","last_unstale":"2023-06-10T21:38:47.121166+0000","last_undegraded":"2023-06-10T21:38:47.121166+0000","last_fullsized":"2023-06-10T21:38:47.121166+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":274,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.274028+0000","last_change":"2023-06-10T21:28:57.895763+0000","last_active":"2023-06-10T21:38:38.274028+0000","last_peered":"2023-06-10T21:38:38.274028+0000","last_clean":"2023-06-10T21:38:38.274028+0000","last_became_active":"2023-06-10T21:28:57.894920+0000","last_became_peered":"2023-06-10T21:28:57.894920+0000","last_unstale":"2023-06-10T21:38:38.274028+0000","last_undegraded":"2023-06-10T21:38:38.274028+0000","last_fullsized":"2023-06-10T21:38:38.274028+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:38:58.371412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":207,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.080806+0000","last_change":"2023-06-10T21:38:22.241373+0000","last_active":"2023-06-10T21:38:47.080806+0000","last_peered":"2023-06-10T21:38:47.080806+0000","last_clean":"2023-06-10T21:38:47.080806+0000","last_became_active":"2023-06-10T21:38:22.240817+0000","last_became_peered":"2023-06-10T21:38:22.240817+0000","last_unstale":"2023-06-10T21:38:47.080806+0000","last_undegraded":"2023-06-10T21:38:47.080806+0000","last_fullsized":"2023-06-10T21:38:47.080806+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:36:26.399798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116944+0000","last_change":"2023-06-10T21:38:37.272249+0000","last_active":"2023-06-10T21:38:47.116944+0000","last_peered":"2023-06-10T21:38:47.116944+0000","last_clean":"2023-06-10T21:38:47.116944+0000","last_became_active":"2023-06-10T21:38:37.271772+0000","last_became_peered":"2023-06-10T21:38:37.271772+0000","last_unstale":"2023-06-10T21:38:47.116944+0000","last_undegraded":"2023-06-10T21:38:47.116944+0000","last_fullsized":"2023-06-10T21:38:47.116944+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":334,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.120284+0000","last_change":"2023-06-10T21:38:22.212951+0000","last_active":"2023-06-10T21:38:47.120284+0000","last_peered":"2023-06-10T21:38:47.120284+0000","last_clean":"2023-06-10T21:38:47.120284+0000","last_became_active":"2023-06-10T21:38:22.212451+0000","last_became_peered":"2023-06-10T21:38:22.212451+0000","last_unstale":"2023-06-10T21:38:47.120284+0000","last_undegraded":"2023-06-10T21:38:47.120284+0000","last_fullsized":"2023-06-10T21:38:47.120284+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.086300+0000","last_change":"2023-06-10T21:38:22.221508+0000","last_active":"2023-06-10T21:38:47.086300+0000","last_peered":"2023-06-10T21:38:47.086300+0000","last_clean":"2023-06-10T21:38:47.086300+0000","last_became_active":"2023-06-10T21:38:22.221190+0000","last_became_peered":"2023-06-10T21:38:22.221190+0000","last_unstale":"2023-06-10T21:38:47.086300+0000","last_undegraded":"2023-06-10T21:38:47.086300+0000","last_fullsized":"2023-06-10T21:38:47.086300+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:18:00.248557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":355,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.080759+0000","last_change":"2023-06-10T21:38:37.391451+0000","last_active":"2023-06-10T21:38:47.080759+0000","last_peered":"2023-06-10T21:38:47.080759+0000","last_clean":"2023-06-10T21:38:47.080759+0000","last_became_active":"2023-06-10T21:38:37.269231+0000","last_became_peered":"2023-06-10T21:38:37.269231+0000","last_unstale":"2023-06-10T21:38:47.080759+0000","last_undegraded":"2023-06-10T21:38:47.080759+0000","last_fullsized":"2023-06-10T21:38:47.080759+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:18:13.380586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":252,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.122907+0000","last_change":"2023-06-10T21:38:22.230697+0000","last_active":"2023-06-10T21:38:47.122907+0000","last_peered":"2023-06-10T21:38:47.122907+0000","last_clean":"2023-06-10T21:38:47.122907+0000","last_became_active":"2023-06-10T21:38:22.230217+0000","last_became_peered":"2023-06-10T21:38:22.230217+0000","last_unstale":"2023-06-10T21:38:47.122907+0000","last_undegraded":"2023-06-10T21:38:47.122907+0000","last_fullsized":"2023-06-10T21:38:47.122907+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.0","version":"340'6983","reported_seq":9241,"reported_epoch":340,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:38:49.916615+0000","last_change":"2023-06-10T21:38:38.581414+0000","last_active":"2023-06-10T21:38:49.916615+0000","last_peered":"2023-06-10T21:38:49.916615+0000","last_clean":"2023-06-10T21:37:31.991292+0000","last_became_active":"2023-06-10T21:38:38.500267+0000","last_became_peered":"2023-06-10T21:38:38.500267+0000","last_unstale":"2023-06-10T21:38:49.916615+0000","last_undegraded":"2023-06-10T21:38:38.461712+0000","last_fullsized":"2023-06-10T21:38:49.916615+0000","mapping_epoch":336,"log_start":"316'1280","ondisk_log_start":"316'1280","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:39:29.617179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256049152,"num_objects":3907,"num_object_clones":0,"num_object_copies":11721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3907,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6983,"num_write_kb":348480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71761920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,0],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"0,2,7","objects":1},{"shards":"2,3,7","objects":239},{"shards":"2,4,7","objects":3667}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":78,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.122824+0000","last_change":"2023-06-10T21:37:33.590010+0000","last_active":"2023-06-10T21:38:47.122824+0000","last_peered":"2023-06-10T21:38:47.122824+0000","last_clean":"2023-06-10T21:38:47.122824+0000","last_became_active":"2023-06-10T21:37:33.035785+0000","last_became_peered":"2023-06-10T21:37:33.035785+0000","last_unstale":"2023-06-10T21:38:47.122824+0000","last_undegraded":"2023-06-10T21:38:47.122824+0000","last_fullsized":"2023-06-10T21:38:47.122824+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116570+0000","last_change":"2023-06-10T21:38:37.276468+0000","last_active":"2023-06-10T21:38:47.116570+0000","last_peered":"2023-06-10T21:38:47.116570+0000","last_clean":"2023-06-10T21:38:47.116570+0000","last_became_active":"2023-06-10T21:38:37.273971+0000","last_became_peered":"2023-06-10T21:38:37.273971+0000","last_unstale":"2023-06-10T21:38:47.116570+0000","last_undegraded":"2023-06-10T21:38:47.116570+0000","last_fullsized":"2023-06-10T21:38:47.116570+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"340'6948","reported_seq":15159,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:49.061926+0000","last_change":"2023-06-10T21:38:48.093922+0000","last_active":"2023-06-10T21:38:49.061926+0000","last_peered":"2023-06-10T21:38:49.061926+0000","last_clean":"2023-06-10T21:38:49.061926+0000","last_became_active":"2023-06-10T21:38:48.075850+0000","last_became_peered":"2023-06-10T21:38:48.075850+0000","last_unstale":"2023-06-10T21:38:49.061926+0000","last_undegraded":"2023-06-10T21:38:49.061926+0000","last_fullsized":"2023-06-10T21:38:49.061926+0000","mapping_epoch":339,"log_start":"319'2066","ondisk_log_start":"319'2066","created":311,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4882,"log_dups_size":0,"ondisk_log_size":4882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257425408,"num_objects":3928,"num_object_clones":0,"num_object_copies":11784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6948,"num_write_kb":348032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4070,"num_bytes_recovered":266534912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":78,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.122050+0000","last_change":"2023-06-10T21:37:33.588114+0000","last_active":"2023-06-10T21:38:47.122050+0000","last_peered":"2023-06-10T21:38:47.122050+0000","last_clean":"2023-06-10T21:38:47.122050+0000","last_became_active":"2023-06-10T21:37:33.033269+0000","last_became_peered":"2023-06-10T21:37:33.033269+0000","last_unstale":"2023-06-10T21:38:47.122050+0000","last_undegraded":"2023-06-10T21:38:47.122050+0000","last_fullsized":"2023-06-10T21:38:47.122050+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":220,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.080281+0000","last_change":"2023-06-10T21:38:22.233384+0000","last_active":"2023-06-10T21:38:47.080281+0000","last_peered":"2023-06-10T21:38:47.080281+0000","last_clean":"2023-06-10T21:38:47.080281+0000","last_became_active":"2023-06-10T21:38:22.232613+0000","last_became_peered":"2023-06-10T21:38:22.232613+0000","last_unstale":"2023-06-10T21:38:47.080281+0000","last_undegraded":"2023-06-10T21:38:47.080281+0000","last_fullsized":"2023-06-10T21:38:47.080281+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:29:41.778097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.2","version":"340'7119","reported_seq":8312,"reported_epoch":340,"state":"active+recovering+remapped","last_fresh":"2023-06-10T21:38:49.063359+0000","last_change":"2023-06-10T21:38:37.427870+0000","last_active":"2023-06-10T21:38:49.063359+0000","last_peered":"2023-06-10T21:38:49.063359+0000","last_clean":"2023-06-10T21:37:32.009051+0000","last_became_active":"2023-06-10T21:38:23.537596+0000","last_became_peered":"2023-06-10T21:38:23.537596+0000","last_unstale":"2023-06-10T21:38:49.063359+0000","last_undegraded":"2023-06-10T21:38:49.063359+0000","last_fullsized":"2023-06-10T21:38:49.063359+0000","mapping_epoch":331,"log_start":"319'2337","ondisk_log_start":"319'2337","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256311296,"num_objects":3911,"num_object_clones":0,"num_object_copies":11733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7242,"num_objects_unfound":0,"num_objects_dirty":3911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7119,"num_write_kb":352960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":570,"num_bytes_recovered":37355520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,3],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"1,3,7","objects":290},{"shards":"3,4,7","objects":3621}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":382,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121732+0000","last_change":"2023-06-10T21:38:37.345732+0000","last_active":"2023-06-10T21:38:47.121732+0000","last_peered":"2023-06-10T21:38:47.121732+0000","last_clean":"2023-06-10T21:38:47.121732+0000","last_became_active":"2023-06-10T21:38:37.292594+0000","last_became_peered":"2023-06-10T21:38:37.292594+0000","last_unstale":"2023-06-10T21:38:47.121732+0000","last_undegraded":"2023-06-10T21:38:47.121732+0000","last_fullsized":"2023-06-10T21:38:47.121732+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.119051+0000","last_change":"2023-06-10T21:38:37.271349+0000","last_active":"2023-06-10T21:38:47.119051+0000","last_peered":"2023-06-10T21:38:47.119051+0000","last_clean":"2023-06-10T21:38:47.119051+0000","last_became_active":"2023-06-10T21:38:37.271011+0000","last_became_peered":"2023-06-10T21:38:37.271011+0000","last_unstale":"2023-06-10T21:38:47.119051+0000","last_undegraded":"2023-06-10T21:38:47.119051+0000","last_fullsized":"2023-06-10T21:38:47.119051+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"340'7132","reported_seq":8601,"reported_epoch":340,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:38:48.717782+0000","last_change":"2023-06-10T21:38:46.217144+0000","last_active":"2023-06-10T21:38:48.717782+0000","last_peered":"2023-06-10T21:38:48.717782+0000","last_clean":"2023-06-10T21:38:21.185185+0000","last_became_active":"2023-06-10T21:38:37.376770+0000","last_became_peered":"2023-06-10T21:38:37.376770+0000","last_unstale":"2023-06-10T21:38:48.717782+0000","last_undegraded":"2023-06-10T21:38:48.717782+0000","last_fullsized":"2023-06-10T21:38:48.717782+0000","mapping_epoch":335,"log_start":"316'1985","ondisk_log_start":"316'1985","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:03:59.048913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260833280,"num_objects":3980,"num_object_clones":0,"num_object_copies":11940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3284,"num_objects_unfound":0,"num_objects_dirty":3980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7132,"num_write_kb":355584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":704,"num_bytes_recovered":46137344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":3980}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.117018+0000","last_change":"2023-06-10T21:38:37.273465+0000","last_active":"2023-06-10T21:38:47.117018+0000","last_peered":"2023-06-10T21:38:47.117018+0000","last_clean":"2023-06-10T21:38:47.117018+0000","last_became_active":"2023-06-10T21:38:37.273068+0000","last_became_peered":"2023-06-10T21:38:37.273068+0000","last_unstale":"2023-06-10T21:38:47.117018+0000","last_undegraded":"2023-06-10T21:38:47.117018+0000","last_fullsized":"2023-06-10T21:38:47.117018+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":192,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081312+0000","last_change":"2023-06-10T21:38:37.286475+0000","last_active":"2023-06-10T21:38:47.081312+0000","last_peered":"2023-06-10T21:38:47.081312+0000","last_clean":"2023-06-10T21:38:47.081312+0000","last_became_active":"2023-06-10T21:38:37.285955+0000","last_became_peered":"2023-06-10T21:38:37.285955+0000","last_unstale":"2023-06-10T21:38:47.081312+0000","last_undegraded":"2023-06-10T21:38:47.081312+0000","last_fullsized":"2023-06-10T21:38:47.081312+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:48:45.543767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"340'7090","reported_seq":7145,"reported_epoch":340,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:38:49.081210+0000","last_change":"2023-06-10T21:38:23.550404+0000","last_active":"2023-06-10T21:38:49.081210+0000","last_peered":"2023-06-10T21:38:49.081210+0000","last_clean":"2023-06-10T21:38:21.136064+0000","last_became_active":"2023-06-10T21:38:23.385464+0000","last_became_peered":"2023-06-10T21:38:23.385464+0000","last_unstale":"2023-06-10T21:38:49.081210+0000","last_undegraded":"2023-06-10T21:38:49.081210+0000","last_fullsized":"2023-06-10T21:38:49.081210+0000","mapping_epoch":331,"log_start":"319'2329","ondisk_log_start":"319'2329","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4761,"log_dups_size":0,"ondisk_log_size":4761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255983616,"num_objects":3906,"num_object_clones":0,"num_object_copies":11718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3906,"num_objects_unfound":0,"num_objects_dirty":3906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7090,"num_write_kb":351872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,3],"avail_no_missing":["7","1","3"],"object_location_counts":[{"shards":"1,3,7","objects":3906}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":175,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081407+0000","last_change":"2023-06-10T21:38:22.344743+0000","last_active":"2023-06-10T21:38:47.081407+0000","last_peered":"2023-06-10T21:38:47.081407+0000","last_clean":"2023-06-10T21:38:47.081407+0000","last_became_active":"2023-06-10T21:38:22.241704+0000","last_became_peered":"2023-06-10T21:38:22.241704+0000","last_unstale":"2023-06-10T21:38:47.081407+0000","last_undegraded":"2023-06-10T21:38:47.081407+0000","last_fullsized":"2023-06-10T21:38:47.081407+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:33:26.417533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":218,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121349+0000","last_change":"2023-06-10T21:38:22.230969+0000","last_active":"2023-06-10T21:38:47.121349+0000","last_peered":"2023-06-10T21:38:47.121349+0000","last_clean":"2023-06-10T21:38:47.121349+0000","last_became_active":"2023-06-10T21:38:22.230537+0000","last_became_peered":"2023-06-10T21:38:22.230537+0000","last_unstale":"2023-06-10T21:38:47.121349+0000","last_undegraded":"2023-06-10T21:38:47.121349+0000","last_fullsized":"2023-06-10T21:38:47.121349+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.5","version":"340'7159","reported_seq":7198,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:49.818622+0000","last_change":"2023-06-10T21:36:04.289746+0000","last_active":"2023-06-10T21:38:49.818622+0000","last_peered":"2023-06-10T21:38:49.818622+0000","last_clean":"2023-06-10T21:38:49.818622+0000","last_became_active":"2023-06-10T21:36:04.289434+0000","last_became_peered":"2023-06-10T21:36:04.289434+0000","last_unstale":"2023-06-10T21:38:49.818622+0000","last_undegraded":"2023-06-10T21:38:49.818622+0000","last_fullsized":"2023-06-10T21:38:49.818622+0000","mapping_epoch":311,"log_start":"316'1467","ondisk_log_start":"316'1467","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5692,"log_dups_size":0,"ondisk_log_size":5692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:39.985524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255787008,"num_objects":3903,"num_object_clones":0,"num_object_copies":11709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7159,"num_write_kb":353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":132,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.274416+0000","last_change":"2023-06-10T21:38:37.272391+0000","last_active":"2023-06-10T21:38:38.274416+0000","last_peered":"2023-06-10T21:38:38.274416+0000","last_clean":"2023-06-10T21:38:38.274416+0000","last_became_active":"2023-06-10T21:38:37.272031+0000","last_became_peered":"2023-06-10T21:38:37.272031+0000","last_unstale":"2023-06-10T21:38:38.274416+0000","last_undegraded":"2023-06-10T21:38:38.274416+0000","last_fullsized":"2023-06-10T21:38:38.274416+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:07:49.456274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058313+0000","last_change":"2023-06-10T21:37:33.031981+0000","last_active":"2023-06-10T21:38:47.058313+0000","last_peered":"2023-06-10T21:38:47.058313+0000","last_clean":"2023-06-10T21:38:47.058313+0000","last_became_active":"2023-06-10T21:37:33.031672+0000","last_became_peered":"2023-06-10T21:37:33.031672+0000","last_unstale":"2023-06-10T21:38:47.058313+0000","last_undegraded":"2023-06-10T21:38:47.058313+0000","last_fullsized":"2023-06-10T21:38:47.058313+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:39:54.124068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"340'6936","reported_seq":6975,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:48.650643+0000","last_change":"2023-06-10T21:36:04.281931+0000","last_active":"2023-06-10T21:38:48.650643+0000","last_peered":"2023-06-10T21:38:48.650643+0000","last_clean":"2023-06-10T21:38:48.650643+0000","last_became_active":"2023-06-10T21:36:04.281632+0000","last_became_peered":"2023-06-10T21:36:04.281632+0000","last_unstale":"2023-06-10T21:38:48.650643+0000","last_undegraded":"2023-06-10T21:38:48.650643+0000","last_fullsized":"2023-06-10T21:38:48.650643+0000","mapping_epoch":311,"log_start":"316'1801","ondisk_log_start":"316'1801","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5135,"log_dups_size":0,"ondisk_log_size":5135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:57.076427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258736128,"num_objects":3948,"num_object_clones":0,"num_object_copies":11844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6936,"num_write_kb":348288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":328,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.060350+0000","last_change":"2023-06-10T21:37:33.023771+0000","last_active":"2023-06-10T21:38:47.060350+0000","last_peered":"2023-06-10T21:38:47.060350+0000","last_clean":"2023-06-10T21:38:47.060350+0000","last_became_active":"2023-06-10T21:37:33.022937+0000","last_became_peered":"2023-06-10T21:37:33.022937+0000","last_unstale":"2023-06-10T21:38:47.060350+0000","last_undegraded":"2023-06-10T21:38:47.060350+0000","last_fullsized":"2023-06-10T21:38:47.060350+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:19:17.233295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116937+0000","last_change":"2023-06-10T21:38:37.274545+0000","last_active":"2023-06-10T21:38:47.116937+0000","last_peered":"2023-06-10T21:38:47.116937+0000","last_clean":"2023-06-10T21:38:47.116937+0000","last_became_active":"2023-06-10T21:38:37.273801+0000","last_became_peered":"2023-06-10T21:38:37.273801+0000","last_unstale":"2023-06-10T21:38:47.116937+0000","last_undegraded":"2023-06-10T21:38:47.116937+0000","last_fullsized":"2023-06-10T21:38:47.116937+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.7","version":"340'6908","reported_seq":6945,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:48.722357+0000","last_change":"2023-06-10T21:36:04.289902+0000","last_active":"2023-06-10T21:38:48.722357+0000","last_peered":"2023-06-10T21:38:48.722357+0000","last_clean":"2023-06-10T21:38:48.722357+0000","last_became_active":"2023-06-10T21:36:04.289647+0000","last_became_peered":"2023-06-10T21:36:04.289647+0000","last_unstale":"2023-06-10T21:38:48.722357+0000","last_undegraded":"2023-06-10T21:38:48.722357+0000","last_fullsized":"2023-06-10T21:38:48.722357+0000","mapping_epoch":311,"log_start":"319'2118","ondisk_log_start":"319'2118","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253624320,"num_objects":3870,"num_object_clones":0,"num_object_copies":11610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6908,"num_write_kb":344896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":263,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.118989+0000","last_change":"2023-06-10T21:38:37.269625+0000","last_active":"2023-06-10T21:38:47.118989+0000","last_peered":"2023-06-10T21:38:47.118989+0000","last_clean":"2023-06-10T21:38:47.118989+0000","last_became_active":"2023-06-10T21:38:37.269141+0000","last_became_peered":"2023-06-10T21:38:37.269141+0000","last_unstale":"2023-06-10T21:38:47.118989+0000","last_undegraded":"2023-06-10T21:38:47.118989+0000","last_fullsized":"2023-06-10T21:38:47.118989+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":259,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.273725+0000","last_change":"2023-06-10T21:37:33.025775+0000","last_active":"2023-06-10T21:38:38.273725+0000","last_peered":"2023-06-10T21:38:38.273725+0000","last_clean":"2023-06-10T21:38:38.273725+0000","last_became_active":"2023-06-10T21:37:33.025410+0000","last_became_peered":"2023-06-10T21:37:33.025410+0000","last_unstale":"2023-06-10T21:38:38.273725+0000","last_undegraded":"2023-06-10T21:38:38.273725+0000","last_fullsized":"2023-06-10T21:38:38.273725+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:32:50.026586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"340'7162","reported_seq":7212,"reported_epoch":340,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:38:49.897792+0000","last_change":"2023-06-10T21:38:23.632128+0000","last_active":"2023-06-10T21:38:49.897792+0000","last_peered":"2023-06-10T21:38:49.897792+0000","last_clean":"2023-06-10T21:38:21.223983+0000","last_became_active":"2023-06-10T21:38:23.609937+0000","last_became_peered":"2023-06-10T21:38:23.609937+0000","last_unstale":"2023-06-10T21:38:49.897792+0000","last_undegraded":"2023-06-10T21:38:23.593196+0000","last_fullsized":"2023-06-10T21:38:23.199810+0000","mapping_epoch":331,"log_start":"316'1434","ondisk_log_start":"316'1434","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5728,"log_dups_size":0,"ondisk_log_size":5728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:56:41.470932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258605056,"num_objects":3946,"num_object_clones":0,"num_object_copies":11838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5553,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7162,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2","objects":-568},{"shards":"0,2,3","objects":3946}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":165,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081155+0000","last_change":"2023-06-10T21:28:34.532576+0000","last_active":"2023-06-10T21:38:47.081155+0000","last_peered":"2023-06-10T21:38:47.081155+0000","last_clean":"2023-06-10T21:38:47.081155+0000","last_became_active":"2023-06-10T21:28:34.532165+0000","last_became_peered":"2023-06-10T21:28:34.532165+0000","last_unstale":"2023-06-10T21:38:47.081155+0000","last_undegraded":"2023-06-10T21:38:47.081155+0000","last_fullsized":"2023-06-10T21:38:47.081155+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:32:53.827714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116866+0000","last_change":"2023-06-10T21:38:37.269405+0000","last_active":"2023-06-10T21:38:47.116866+0000","last_peered":"2023-06-10T21:38:47.116866+0000","last_clean":"2023-06-10T21:38:47.116866+0000","last_became_active":"2023-06-10T21:38:37.269101+0000","last_became_peered":"2023-06-10T21:38:37.269101+0000","last_unstale":"2023-06-10T21:38:47.116866+0000","last_undegraded":"2023-06-10T21:38:47.116866+0000","last_fullsized":"2023-06-10T21:38:47.116866+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116545+0000","last_change":"2023-06-10T21:38:37.274310+0000","last_active":"2023-06-10T21:38:47.116545+0000","last_peered":"2023-06-10T21:38:47.116545+0000","last_clean":"2023-06-10T21:38:47.116545+0000","last_became_active":"2023-06-10T21:38:37.273703+0000","last_became_peered":"2023-06-10T21:38:37.273703+0000","last_unstale":"2023-06-10T21:38:47.116545+0000","last_undegraded":"2023-06-10T21:38:47.116545+0000","last_fullsized":"2023-06-10T21:38:47.116545+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"340'7263","reported_seq":7310,"reported_epoch":340,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:38:48.692306+0000","last_change":"2023-06-10T21:38:38.376675+0000","last_active":"2023-06-10T21:38:48.692306+0000","last_peered":"2023-06-10T21:38:48.692306+0000","last_clean":"2023-06-10T21:38:36.226527+0000","last_became_active":"2023-06-10T21:38:38.363343+0000","last_became_peered":"2023-06-10T21:38:38.363343+0000","last_unstale":"2023-06-10T21:38:48.692306+0000","last_undegraded":"2023-06-10T21:38:48.692306+0000","last_fullsized":"2023-06-10T21:38:48.692306+0000","mapping_epoch":336,"log_start":"316'2098","ondisk_log_start":"316'2098","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5165,"log_dups_size":0,"ondisk_log_size":5165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:15:53.850039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256966656,"num_objects":3921,"num_object_clones":0,"num_object_copies":11763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3921,"num_objects_unfound":0,"num_objects_dirty":3921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7263,"num_write_kb":357888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":3921}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116667+0000","last_change":"2023-06-10T21:38:37.274045+0000","last_active":"2023-06-10T21:38:47.116667+0000","last_peered":"2023-06-10T21:38:47.116667+0000","last_clean":"2023-06-10T21:38:47.116667+0000","last_became_active":"2023-06-10T21:38:37.273208+0000","last_became_peered":"2023-06-10T21:38:37.273208+0000","last_unstale":"2023-06-10T21:38:47.116667+0000","last_undegraded":"2023-06-10T21:38:47.116667+0000","last_fullsized":"2023-06-10T21:38:47.116667+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":312,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081457+0000","last_change":"2023-06-10T21:38:37.286639+0000","last_active":"2023-06-10T21:38:47.081457+0000","last_peered":"2023-06-10T21:38:47.081457+0000","last_clean":"2023-06-10T21:38:47.081457+0000","last_became_active":"2023-06-10T21:38:37.286218+0000","last_became_peered":"2023-06-10T21:38:37.286218+0000","last_unstale":"2023-06-10T21:38:47.081457+0000","last_undegraded":"2023-06-10T21:38:47.081457+0000","last_fullsized":"2023-06-10T21:38:47.081457+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:12:28.583145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":184,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058225+0000","last_change":"2023-06-10T21:38:22.227077+0000","last_active":"2023-06-10T21:38:47.058225+0000","last_peered":"2023-06-10T21:38:47.058225+0000","last_clean":"2023-06-10T21:38:47.058225+0000","last_became_active":"2023-06-10T21:38:22.226085+0000","last_became_peered":"2023-06-10T21:38:22.226085+0000","last_unstale":"2023-06-10T21:38:47.058225+0000","last_undegraded":"2023-06-10T21:38:47.058225+0000","last_fullsized":"2023-06-10T21:38:47.058225+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:46:52.919584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"340'7076","reported_seq":8158,"reported_epoch":340,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:38:49.929881+0000","last_change":"2023-06-10T21:38:38.536556+0000","last_active":"2023-06-10T21:38:49.929881+0000","last_peered":"2023-06-10T21:38:49.929881+0000","last_clean":"2023-06-10T21:37:32.008522+0000","last_became_active":"2023-06-10T21:38:38.451692+0000","last_became_peered":"2023-06-10T21:38:38.451692+0000","last_unstale":"2023-06-10T21:38:49.929881+0000","last_undegraded":"2023-06-10T21:38:38.439159+0000","last_fullsized":"2023-06-10T21:38:49.929881+0000","mapping_epoch":336,"log_start":"316'1365","ondisk_log_start":"316'1365","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5711,"log_dups_size":0,"ondisk_log_size":5711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:54:43.930675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261095424,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7076,"num_write_kb":353920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":513,"num_bytes_recovered":33619968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7,1],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"1,2,7","objects":237},{"shards":"2,4,7","objects":3747}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":195,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.060259+0000","last_change":"2023-06-10T21:38:22.222741+0000","last_active":"2023-06-10T21:38:47.060259+0000","last_peered":"2023-06-10T21:38:47.060259+0000","last_clean":"2023-06-10T21:38:47.060259+0000","last_became_active":"2023-06-10T21:38:22.222447+0000","last_became_peered":"2023-06-10T21:38:22.222447+0000","last_unstale":"2023-06-10T21:38:47.060259+0000","last_undegraded":"2023-06-10T21:38:47.060259+0000","last_fullsized":"2023-06-10T21:38:47.060259+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:35:04.574363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":184,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.057291+0000","last_change":"2023-06-10T21:28:34.502213+0000","last_active":"2023-06-10T21:38:47.057291+0000","last_peered":"2023-06-10T21:38:47.057291+0000","last_clean":"2023-06-10T21:38:47.057291+0000","last_became_active":"2023-06-10T21:28:34.498313+0000","last_became_peered":"2023-06-10T21:28:34.498313+0000","last_unstale":"2023-06-10T21:38:47.057291+0000","last_undegraded":"2023-06-10T21:38:47.057291+0000","last_fullsized":"2023-06-10T21:38:47.057291+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:59.041440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.119904+0000","last_change":"2023-06-10T21:38:37.274199+0000","last_active":"2023-06-10T21:38:47.119904+0000","last_peered":"2023-06-10T21:38:47.119904+0000","last_clean":"2023-06-10T21:38:47.119904+0000","last_became_active":"2023-06-10T21:38:37.273853+0000","last_became_peered":"2023-06-10T21:38:37.273853+0000","last_unstale":"2023-06-10T21:38:47.119904+0000","last_undegraded":"2023-06-10T21:38:47.119904+0000","last_fullsized":"2023-06-10T21:38:47.119904+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"340'6970","reported_seq":7007,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:49.087344+0000","last_change":"2023-06-10T21:36:04.289917+0000","last_active":"2023-06-10T21:38:49.087344+0000","last_peered":"2023-06-10T21:38:49.087344+0000","last_clean":"2023-06-10T21:38:49.087344+0000","last_became_active":"2023-06-10T21:36:04.289698+0000","last_became_peered":"2023-06-10T21:36:04.289698+0000","last_unstale":"2023-06-10T21:38:49.087344+0000","last_undegraded":"2023-06-10T21:38:49.087344+0000","last_fullsized":"2023-06-10T21:38:49.087344+0000","mapping_epoch":311,"log_start":"319'2188","ondisk_log_start":"319'2188","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253820952,"num_objects":3874,"num_object_clones":0,"num_object_copies":11622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3874,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6970,"num_write_kb":346945,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.117176+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:38:47.117176+0000","last_peered":"2023-06-10T21:38:47.117176+0000","last_clean":"2023-06-10T21:38:47.117176+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:38:47.117176+0000","last_undegraded":"2023-06-10T21:38:47.117176+0000","last_fullsized":"2023-06-10T21:38:47.117176+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":503,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.118867+0000","last_change":"2023-06-10T21:32:58.602882+0000","last_active":"2023-06-10T21:38:47.118867+0000","last_peered":"2023-06-10T21:38:47.118867+0000","last_clean":"2023-06-10T21:38:47.118867+0000","last_became_active":"2023-06-10T21:32:58.602565+0000","last_became_peered":"2023-06-10T21:32:58.602565+0000","last_unstale":"2023-06-10T21:38:47.118867+0000","last_undegraded":"2023-06-10T21:38:47.118867+0000","last_fullsized":"2023-06-10T21:38:47.118867+0000","mapping_epoch":291,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":292,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116759+0000","last_change":"2023-06-10T21:38:37.280585+0000","last_active":"2023-06-10T21:38:47.116759+0000","last_peered":"2023-06-10T21:38:47.116759+0000","last_clean":"2023-06-10T21:38:47.116759+0000","last_became_active":"2023-06-10T21:38:37.280270+0000","last_became_peered":"2023-06-10T21:38:37.280270+0000","last_unstale":"2023-06-10T21:38:47.116759+0000","last_undegraded":"2023-06-10T21:38:47.116759+0000","last_fullsized":"2023-06-10T21:38:47.116759+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":181,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121884+0000","last_change":"2023-06-10T21:38:37.278241+0000","last_active":"2023-06-10T21:38:47.121884+0000","last_peered":"2023-06-10T21:38:47.121884+0000","last_clean":"2023-06-10T21:38:47.121884+0000","last_became_active":"2023-06-10T21:38:37.277605+0000","last_became_peered":"2023-06-10T21:38:37.277605+0000","last_unstale":"2023-06-10T21:38:47.121884+0000","last_undegraded":"2023-06-10T21:38:47.121884+0000","last_fullsized":"2023-06-10T21:38:47.121884+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"340'7034","reported_seq":8047,"reported_epoch":340,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:38:48.705071+0000","last_change":"2023-06-10T21:38:37.549207+0000","last_active":"2023-06-10T21:38:48.705071+0000","last_peered":"2023-06-10T21:38:48.705071+0000","last_clean":"2023-06-10T21:37:31.983708+0000","last_became_active":"2023-06-10T21:38:37.446638+0000","last_became_peered":"2023-06-10T21:38:37.446638+0000","last_unstale":"2023-06-10T21:38:48.705071+0000","last_undegraded":"2023-06-10T21:38:37.432761+0000","last_fullsized":"2023-06-10T21:38:48.705071+0000","mapping_epoch":335,"log_start":"316'1845","ondisk_log_start":"316'1845","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:17:58.673919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261095424,"num_objects":3984,"num_object_clones":0,"num_object_copies":11952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7034,"num_write_kb":352576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":473,"num_bytes_recovered":30998528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,3],"avail_no_missing":["0","1","4","7"],"object_location_counts":[{"shards":"0,3,7","objects":229},{"shards":"0,4,7","objects":3755}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":359,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.118920+0000","last_change":"2023-06-10T21:28:34.522313+0000","last_active":"2023-06-10T21:38:47.118920+0000","last_peered":"2023-06-10T21:38:47.118920+0000","last_clean":"2023-06-10T21:38:47.118920+0000","last_became_active":"2023-06-10T21:28:34.521946+0000","last_became_peered":"2023-06-10T21:28:34.521946+0000","last_unstale":"2023-06-10T21:38:47.118920+0000","last_undegraded":"2023-06-10T21:38:47.118920+0000","last_fullsized":"2023-06-10T21:38:47.118920+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":207,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.080651+0000","last_change":"2023-06-10T21:38:37.294605+0000","last_active":"2023-06-10T21:38:47.080651+0000","last_peered":"2023-06-10T21:38:47.080651+0000","last_clean":"2023-06-10T21:38:47.080651+0000","last_became_active":"2023-06-10T21:38:37.294228+0000","last_became_peered":"2023-06-10T21:38:37.294228+0000","last_unstale":"2023-06-10T21:38:47.080651+0000","last_undegraded":"2023-06-10T21:38:47.080651+0000","last_fullsized":"2023-06-10T21:38:47.080651+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:21:40.876830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":165,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.087611+0000","last_change":"2023-06-10T21:28:34.502249+0000","last_active":"2023-06-10T21:38:47.087611+0000","last_peered":"2023-06-10T21:38:47.087611+0000","last_clean":"2023-06-10T21:38:47.087611+0000","last_became_active":"2023-06-10T21:28:34.501706+0000","last_became_peered":"2023-06-10T21:28:34.501706+0000","last_unstale":"2023-06-10T21:38:47.087611+0000","last_undegraded":"2023-06-10T21:38:47.087611+0000","last_fullsized":"2023-06-10T21:38:47.087611+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:56:30.103557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"338'6983","reported_seq":10032,"reported_epoch":338,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:38:46.806978+0000","last_change":"2023-06-10T21:38:37.482519+0000","last_active":"2023-06-10T21:38:46.806978+0000","last_peered":"2023-06-10T21:38:46.806978+0000","last_clean":"2023-06-10T21:37:32.003891+0000","last_became_active":"2023-06-10T21:37:34.477833+0000","last_became_peered":"2023-06-10T21:37:34.477833+0000","last_unstale":"2023-06-10T21:38:46.806978+0000","last_undegraded":"2023-06-10T21:38:22.386111+0000","last_fullsized":"2023-06-10T21:37:34.010923+0000","mapping_epoch":325,"log_start":"316'1200","ondisk_log_start":"316'1200","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:17:25.018747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266403840,"num_objects":4065,"num_object_clones":0,"num_object_copies":12195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":181,"num_objects_misplaced":3584,"num_objects_unfound":0,"num_objects_dirty":4065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6983,"num_write_kb":353536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1497,"num_bytes_recovered":98107392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":300},{"shards":"2,3,4","objects":3584}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":353,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.121792+0000","last_change":"2023-06-10T21:37:33.035001+0000","last_active":"2023-06-10T21:38:47.121792+0000","last_peered":"2023-06-10T21:38:47.121792+0000","last_clean":"2023-06-10T21:38:47.121792+0000","last_became_active":"2023-06-10T21:37:33.033707+0000","last_became_peered":"2023-06-10T21:37:33.033707+0000","last_unstale":"2023-06-10T21:38:47.121792+0000","last_undegraded":"2023-06-10T21:38:47.121792+0000","last_fullsized":"2023-06-10T21:38:47.121792+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":193,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058312+0000","last_change":"2023-06-10T21:38:37.269262+0000","last_active":"2023-06-10T21:38:47.058312+0000","last_peered":"2023-06-10T21:38:47.058312+0000","last_clean":"2023-06-10T21:38:47.058312+0000","last_became_active":"2023-06-10T21:38:37.268935+0000","last_became_peered":"2023-06-10T21:38:37.268935+0000","last_unstale":"2023-06-10T21:38:47.058312+0000","last_undegraded":"2023-06-10T21:38:47.058312+0000","last_fullsized":"2023-06-10T21:38:47.058312+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:19:38.850445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":170,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081485+0000","last_change":"2023-06-10T21:38:37.292289+0000","last_active":"2023-06-10T21:38:47.081485+0000","last_peered":"2023-06-10T21:38:47.081485+0000","last_clean":"2023-06-10T21:38:47.081485+0000","last_became_active":"2023-06-10T21:38:37.291912+0000","last_became_peered":"2023-06-10T21:38:37.291912+0000","last_unstale":"2023-06-10T21:38:47.081485+0000","last_undegraded":"2023-06-10T21:38:47.081485+0000","last_fullsized":"2023-06-10T21:38:47.081485+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:14:01.193855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"340'7170","reported_seq":7213,"reported_epoch":340,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:38:49.883818+0000","last_change":"2023-06-10T21:38:23.407505+0000","last_active":"2023-06-10T21:38:49.883818+0000","last_peered":"2023-06-10T21:38:49.883818+0000","last_clean":"2023-06-10T21:38:21.137340+0000","last_became_active":"2023-06-10T21:38:23.386178+0000","last_became_peered":"2023-06-10T21:38:23.386178+0000","last_unstale":"2023-06-10T21:38:49.883818+0000","last_undegraded":"2023-06-10T21:38:49.883818+0000","last_fullsized":"2023-06-10T21:38:49.883818+0000","mapping_epoch":331,"log_start":"316'1466","ondisk_log_start":"316'1466","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:44:54.811516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260833280,"num_objects":3980,"num_object_clones":0,"num_object_copies":11940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3980,"num_objects_unfound":0,"num_objects_dirty":3980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":3980}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":317,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.273463+0000","last_change":"2023-06-10T21:32:55.530585+0000","last_active":"2023-06-10T21:38:38.273463+0000","last_peered":"2023-06-10T21:38:38.273463+0000","last_clean":"2023-06-10T21:38:38.273463+0000","last_became_active":"2023-06-10T21:32:55.523583+0000","last_became_peered":"2023-06-10T21:32:55.523583+0000","last_unstale":"2023-06-10T21:38:38.273463+0000","last_undegraded":"2023-06-10T21:38:38.273463+0000","last_fullsized":"2023-06-10T21:38:38.273463+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:33:48.047836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":123,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.276096+0000","last_change":"2023-06-10T21:37:33.033971+0000","last_active":"2023-06-10T21:38:38.276096+0000","last_peered":"2023-06-10T21:38:38.276096+0000","last_clean":"2023-06-10T21:38:38.276096+0000","last_became_active":"2023-06-10T21:37:33.033628+0000","last_became_peered":"2023-06-10T21:37:33.033628+0000","last_unstale":"2023-06-10T21:38:38.276096+0000","last_undegraded":"2023-06-10T21:38:38.276096+0000","last_fullsized":"2023-06-10T21:38:38.276096+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:02:45.167215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":258,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.067833+0000","last_change":"2023-06-10T21:38:22.257007+0000","last_active":"2023-06-10T21:38:47.067833+0000","last_peered":"2023-06-10T21:38:47.067833+0000","last_clean":"2023-06-10T21:38:47.067833+0000","last_became_active":"2023-06-10T21:38:22.256560+0000","last_became_peered":"2023-06-10T21:38:22.256560+0000","last_unstale":"2023-06-10T21:38:47.067833+0000","last_undegraded":"2023-06-10T21:38:47.067833+0000","last_fullsized":"2023-06-10T21:38:47.067833+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.9","version":"340'7090","reported_seq":7596,"reported_epoch":340,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-06-10T21:38:49.876341+0000","last_change":"2023-06-10T21:38:38.600169+0000","last_active":"2023-06-10T21:38:49.876341+0000","last_peered":"2023-06-10T21:38:49.876341+0000","last_clean":"2023-06-10T21:38:21.166872+0000","last_became_active":"2023-06-10T21:38:38.514154+0000","last_became_peered":"2023-06-10T21:38:38.514154+0000","last_unstale":"2023-06-10T21:38:49.876341+0000","last_undegraded":"2023-06-10T21:38:38.485205+0000","last_fullsized":"2023-06-10T21:38:49.876341+0000","mapping_epoch":336,"log_start":"316'1386","ondisk_log_start":"316'1386","created":311,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:36:03.262449+0000","last_clean_scrub_stamp":"2023-06-10T21:36:03.262449+0000","objects_scrubbed":0,"log_size":5704,"log_dups_size":0,"ondisk_log_size":5704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:27:20.107458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259915776,"num_objects":3966,"num_object_clones":0,"num_object_copies":11898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7090,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":224,"num_bytes_recovered":14614528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,0],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":3966}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":70,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.122866+0000","last_change":"2023-06-10T21:38:22.256998+0000","last_active":"2023-06-10T21:38:47.122866+0000","last_peered":"2023-06-10T21:38:47.122866+0000","last_clean":"2023-06-10T21:38:47.122866+0000","last_became_active":"2023-06-10T21:38:22.256614+0000","last_became_peered":"2023-06-10T21:38:22.256614+0000","last_unstale":"2023-06-10T21:38:47.122866+0000","last_undegraded":"2023-06-10T21:38:47.122866+0000","last_fullsized":"2023-06-10T21:38:47.122866+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":182,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.086227+0000","last_change":"2023-06-10T21:37:33.054430+0000","last_active":"2023-06-10T21:38:47.086227+0000","last_peered":"2023-06-10T21:38:47.086227+0000","last_clean":"2023-06-10T21:38:47.086227+0000","last_became_active":"2023-06-10T21:37:33.032336+0000","last_became_peered":"2023-06-10T21:37:33.032336+0000","last_unstale":"2023-06-10T21:38:47.086227+0000","last_undegraded":"2023-06-10T21:38:47.086227+0000","last_fullsized":"2023-06-10T21:38:47.086227+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:45:26.024378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.119500+0000","last_change":"2023-06-10T21:38:37.286377+0000","last_active":"2023-06-10T21:38:47.119500+0000","last_peered":"2023-06-10T21:38:47.119500+0000","last_clean":"2023-06-10T21:38:47.119500+0000","last_became_active":"2023-06-10T21:38:37.286067+0000","last_became_peered":"2023-06-10T21:38:37.286067+0000","last_unstale":"2023-06-10T21:38:47.119500+0000","last_undegraded":"2023-06-10T21:38:47.119500+0000","last_fullsized":"2023-06-10T21:38:47.119500+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":174,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081538+0000","last_change":"2023-06-10T21:37:33.033288+0000","last_active":"2023-06-10T21:38:47.081538+0000","last_peered":"2023-06-10T21:38:47.081538+0000","last_clean":"2023-06-10T21:38:47.081538+0000","last_became_active":"2023-06-10T21:37:33.032719+0000","last_became_peered":"2023-06-10T21:37:33.032719+0000","last_unstale":"2023-06-10T21:38:47.081538+0000","last_undegraded":"2023-06-10T21:38:47.081538+0000","last_fullsized":"2023-06-10T21:38:47.081538+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:01:58.992205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058371+0000","last_change":"2023-06-10T21:38:22.227239+0000","last_active":"2023-06-10T21:38:47.058371+0000","last_peered":"2023-06-10T21:38:47.058371+0000","last_clean":"2023-06-10T21:38:47.058371+0000","last_became_active":"2023-06-10T21:38:22.226792+0000","last_became_peered":"2023-06-10T21:38:22.226792+0000","last_unstale":"2023-06-10T21:38:47.058371+0000","last_undegraded":"2023-06-10T21:38:47.058371+0000","last_fullsized":"2023-06-10T21:38:47.058371+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:33:31.283226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":18,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116919+0000","last_change":"2023-06-10T21:38:37.384689+0000","last_active":"2023-06-10T21:38:47.116919+0000","last_peered":"2023-06-10T21:38:47.116919+0000","last_clean":"2023-06-10T21:38:47.116919+0000","last_became_active":"2023-06-10T21:38:37.273055+0000","last_became_peered":"2023-06-10T21:38:37.273055+0000","last_unstale":"2023-06-10T21:38:47.116919+0000","last_undegraded":"2023-06-10T21:38:47.116919+0000","last_fullsized":"2023-06-10T21:38:47.116919+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":230,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.272769+0000","last_change":"2023-06-10T21:37:33.028850+0000","last_active":"2023-06-10T21:38:38.272769+0000","last_peered":"2023-06-10T21:38:38.272769+0000","last_clean":"2023-06-10T21:38:38.272769+0000","last_became_active":"2023-06-10T21:37:33.028586+0000","last_became_peered":"2023-06-10T21:37:33.028586+0000","last_unstale":"2023-06-10T21:38:38.272769+0000","last_undegraded":"2023-06-10T21:38:38.272769+0000","last_fullsized":"2023-06-10T21:38:38.272769+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:34.329136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":18,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.116962+0000","last_change":"2023-06-10T21:38:37.480996+0000","last_active":"2023-06-10T21:38:47.116962+0000","last_peered":"2023-06-10T21:38:47.116962+0000","last_clean":"2023-06-10T21:38:47.116962+0000","last_became_active":"2023-06-10T21:38:37.275336+0000","last_became_peered":"2023-06-10T21:38:37.275336+0000","last_unstale":"2023-06-10T21:38:47.116962+0000","last_undegraded":"2023-06-10T21:38:47.116962+0000","last_fullsized":"2023-06-10T21:38:47.116962+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":286,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-06-10T21:38:38.274495+0000","last_change":"2023-06-10T21:37:33.035518+0000","last_active":"2023-06-10T21:38:38.274495+0000","last_peered":"2023-06-10T21:38:38.274495+0000","last_clean":"2023-06-10T21:38:38.274495+0000","last_became_active":"2023-06-10T21:37:33.034622+0000","last_became_peered":"2023-06-10T21:37:33.034622+0000","last_unstale":"2023-06-10T21:38:38.274495+0000","last_undegraded":"2023-06-10T21:38:38.274495+0000","last_fullsized":"2023-06-10T21:38:38.274495+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":325,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:45:19.232112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":243,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.058259+0000","last_change":"2023-06-10T21:38:22.223038+0000","last_active":"2023-06-10T21:38:47.058259+0000","last_peered":"2023-06-10T21:38:47.058259+0000","last_clean":"2023-06-10T21:38:47.058259+0000","last_became_active":"2023-06-10T21:38:22.222117+0000","last_became_peered":"2023-06-10T21:38:22.222117+0000","last_unstale":"2023-06-10T21:38:47.058259+0000","last_undegraded":"2023-06-10T21:38:47.058259+0000","last_fullsized":"2023-06-10T21:38:47.058259+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:52.732657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":293,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.060445+0000","last_change":"2023-06-10T21:38:37.269462+0000","last_active":"2023-06-10T21:38:47.060445+0000","last_peered":"2023-06-10T21:38:47.060445+0000","last_clean":"2023-06-10T21:38:47.060445+0000","last_became_active":"2023-06-10T21:38:37.269015+0000","last_became_peered":"2023-06-10T21:38:37.269015+0000","last_unstale":"2023-06-10T21:38:47.060445+0000","last_undegraded":"2023-06-10T21:38:47.060445+0000","last_fullsized":"2023-06-10T21:38:47.060445+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:20:50.758485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":275,"reported_epoch":339,"state":"active+clean","last_fresh":"2023-06-10T21:38:47.081440+0000","last_change":"2023-06-10T21:38:37.377112+0000","last_active":"2023-06-10T21:38:47.081440+0000","last_peered":"2023-06-10T21:38:47.081440+0000","last_clean":"2023-06-10T21:38:47.081440+0000","last_became_active":"2023-06-10T21:38:37.266999+0000","last_became_peered":"2023-06-10T21:38:37.266999+0000","last_unstale":"2023-06-10T21:38:47.081440+0000","last_undegraded":"2023-06-10T21:38:47.081440+0000","last_fullsized":"2023-06-10T21:38:47.081440+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:28:44.083948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":340,"state":"active+clean","last_fresh":"2023-06-10T21:38:48.080624+0000","last_change":"2023-06-10T21:38:22.256742+0000","last_active":"2023-06-10T21:38:48.080624+0000","last_peered":"2023-06-10T21:38:48.080624+0000","last_clean":"2023-06-10T21:38:48.080624+0000","last_became_active":"2023-06-10T21:38:22.256164+0000","last_became_peered":"2023-06-10T21:38:22.256164+0000","last_unstale":"2023-06-10T21:38:48.080624+0000","last_undegraded":"2023-06-10T21:38:48.080624+0000","last_fullsized":"2023-06-10T21:38:48.080624+0000","mapping_epoch":330,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":331,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:06:53.849417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4133486616,"num_objects":63073,"num_object_clones":0,"num_object_copies":189219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21575,"num_objects_misplaced":25917,"num_objects_unfound":0,"num_objects_dirty":63073,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":113023,"num_write_kb":5635009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9152,"num_bytes_recovered":599261184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13325185024,"data_stored":13325172808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84658,"ondisk_log_size":84658,"up":48,"acting":46,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":64,"num_bytes_recovered":12922,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430701,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3452836,"kb_used_data":2775260,"kb_used_omap":0,"kb_used_meta":677568,"kb_avail":90292316,"statfs":{"total":95995035648,"available":92459331584,"internally_reserved":0,"allocated":2841866240,"data_stored":2841272485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":693829632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:38:45 2023","interfaces":[{"interface":"back","average":{"1min":4.8559999999999999,"5min":2.3730000000000002,"15min":5.9169999999999998},"min":{"1min":0.26000000000000001,"5min":0.23200000000000001,"15min":0.224},"max":{"1min":56.639000000000003,"5min":56.639000000000003,"15min":184.83699999999999},"last":7.0869999999999997},{"interface":"front","average":{"1min":5.54,"5min":2.919,"15min":6.2030000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.223},"max":{"1min":56.628,"5min":56.628,"15min":185.58000000000001},"last":7.2169999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:38:18 2023","interfaces":[{"interface":"back","average":{"1min":1.7330000000000001,"5min":3.1080000000000001,"15min":2.8570000000000002},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.21299999999999999},"max":{"1min":7.6299999999999999,"5min":60.863,"15min":93.093999999999994},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.6739999999999999,"5min":3.1840000000000002,"15min":2.6589999999999998},"min":{"1min":0.29599999999999999,"5min":0.27300000000000002,"15min":0.27100000000000002},"max":{"1min":8.0310000000000006,"5min":61.148000000000003,"15min":93.119},"last":0.314}]},{"osd":2,"last update":"Sat Jun 10 21:38:34 2023","interfaces":[{"interface":"back","average":{"1min":7.0049999999999999,"5min":6.0490000000000004,"15min":5.3920000000000003},"min":{"1min":0.29599999999999999,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":106.19499999999999,"5min":137.64099999999999,"15min":172.33600000000001},"last":13.996},{"interface":"front","average":{"1min":6.9699999999999998,"5min":6.335,"15min":5.4290000000000003},"min":{"1min":0.44500000000000001,"5min":0.376,"15min":0.29499999999999998},"max":{"1min":108.70099999999999,"5min":136.96600000000001,"15min":172.185},"last":4.9219999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:38:18 2023","interfaces":[{"interface":"back","average":{"1min":11.970000000000001,"5min":7.2830000000000004,"15min":4.2160000000000002},"min":{"1min":0.49199999999999999,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":102.25700000000001,"5min":147.00700000000001,"15min":147.00700000000001},"last":23.689},{"interface":"front","average":{"1min":12.648,"5min":7.218,"15min":3.8490000000000002},"min":{"1min":0.375,"5min":0.308,"15min":0.308},"max":{"1min":103.401,"5min":146.93899999999999,"15min":146.93899999999999},"last":3.0270000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.44,"15min":0.66800000000000004},"min":{"1min":0.253,"5min":0.253,"15min":0.249},"max":{"1min":3.3900000000000001,"5min":3.3900000000000001,"15min":8.1419999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.48699999999999999,"15min":0.91700000000000004},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.7400000000000002,"5min":3.0310000000000001,"15min":14.657999999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.47999999999999998,"15min":1.7030000000000001},"min":{"1min":0.28699999999999998,"5min":0.28100000000000003,"15min":0.27400000000000002},"max":{"1min":1.1699999999999999,"5min":3.1829999999999998,"15min":211.99799999999999},"last":0.436},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.498,"15min":1.55},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29299999999999998},"max":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":212.23699999999999},"last":1.1890000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:38:01 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.53600000000000003,"15min":1.008},"min":{"1min":0.32700000000000001,"5min":0.32200000000000001,"15min":0.27800000000000002},"max":{"1min":1.379,"5min":3.2559999999999998,"15min":13.073},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.52800000000000002,"15min":0.876},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.3599999999999999,"5min":3.3599999999999999,"15min":10.874000000000001},"last":0.39200000000000002}]}]},{"osd":6,"up_from":0,"seq":180388626929,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":105,"seq":450971566371,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2627624,"kb_used_data":1862368,"kb_used_omap":4,"kb_used_meta":765179,"kb_avail":91117528,"statfs":{"total":95995035648,"available":93304348672,"internally_reserved":0,"allocated":1907064832,"data_stored":1905986309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":783544206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:37:51 2023","interfaces":[{"interface":"back","average":{"1min":3.6749999999999998,"5min":3.5350000000000001,"15min":8.7780000000000005},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":11.568,"5min":70.254000000000005,"15min":154.31399999999999},"last":5.4539999999999997},{"interface":"front","average":{"1min":2.0800000000000001,"5min":3.0310000000000001,"15min":8.4459999999999997},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":6.3529999999999998,"5min":70.100999999999999,"15min":154.27199999999999},"last":5.4939999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:37:47 2023","interfaces":[{"interface":"back","average":{"1min":6.1559999999999997,"5min":7.3780000000000001,"15min":5.9710000000000001},"min":{"1min":0.222,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":75.103999999999999,"5min":191.101,"15min":191.101},"last":5.8399999999999999},{"interface":"front","average":{"1min":6.1909999999999998,"5min":6.2039999999999997,"15min":5.4139999999999997},"min":{"1min":0.308,"5min":0.251,"15min":0.222},"max":{"1min":68.587000000000003,"5min":190.87899999999999,"15min":190.87899999999999},"last":0.434}]},{"osd":3,"last update":"Sat Jun 10 21:38:17 2023","interfaces":[{"interface":"back","average":{"1min":9.0709999999999997,"5min":6.1909999999999998,"15min":3.9540000000000002},"min":{"1min":0.61399999999999999,"5min":0.28699999999999998,"15min":0.23799999999999999},"max":{"1min":63.779000000000003,"5min":83.810000000000002,"15min":83.810000000000002},"last":0.498},{"interface":"front","average":{"1min":12.467000000000001,"5min":7.0640000000000001,"15min":5.0229999999999997},"min":{"1min":3.0760000000000001,"5min":0.26400000000000001,"15min":0.23999999999999999},"max":{"1min":64.280000000000001,"5min":92.757000000000005,"15min":92.757000000000005},"last":2.9609999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:38:17 2023","interfaces":[{"interface":"back","average":{"1min":3.093,"5min":1.6180000000000001,"15min":1.6459999999999999},"min":{"1min":0.33100000000000002,"5min":0.28699999999999998,"15min":0.27700000000000002},"max":{"1min":18.478999999999999,"5min":18.478999999999999,"15min":18.478999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.1680000000000001,"5min":1.6299999999999999,"15min":1.5409999999999999},"min":{"1min":0.39500000000000002,"5min":0.29099999999999998,"15min":0.27400000000000002},"max":{"1min":9.3840000000000003,"5min":9.6790000000000003,"15min":14.460000000000001},"last":2.919}]},{"osd":5,"last update":"Sat Jun 10 21:38:17 2023","interfaces":[{"interface":"back","average":{"1min":3.0339999999999998,"5min":2.375,"15min":4.3799999999999999},"min":{"1min":0.34100000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":9.9440000000000008,"5min":11.959,"15min":383.83100000000002},"last":5.7430000000000003},{"interface":"front","average":{"1min":3.3540000000000001,"5min":1.7130000000000001,"15min":3.8580000000000001},"min":{"1min":0.36899999999999999,"5min":0.35299999999999998,"15min":0.29699999999999999},"max":{"1min":18.422000000000001,"5min":18.422000000000001,"15min":383.84800000000001},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:38:17 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.444,"15min":2.3279999999999998},"min":{"1min":0.39300000000000002,"5min":0.316,"15min":0.316},"max":{"1min":9.9809999999999999,"5min":11.923,"15min":14.858000000000001},"last":5.7889999999999997},{"interface":"front","average":{"1min":3.3999999999999999,"5min":1.8240000000000001,"15min":1.8460000000000001},"min":{"1min":0.44700000000000001,"5min":0.40000000000000002,"15min":0.32900000000000001},"max":{"1min":18.376999999999999,"5min":18.376999999999999,"15min":18.376999999999999},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:38:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3580000000000001,"5min":2.9750000000000001,"15min":2.9569999999999999},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.34599999999999997},"max":{"1min":18.452000000000002,"5min":63.249000000000002,"15min":162.399},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.206,"5min":2.9729999999999999,"15min":2.7690000000000001},"min":{"1min":0.434,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.4700000000000006,"5min":62.975999999999999,"15min":162.27199999999999},"last":2.9900000000000002}]}]},{"osd":0,"up_from":244,"seq":1047972020348,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888272,"kb_used_data":1534088,"kb_used_omap":10,"kb_used_meta":354165,"kb_avail":91856880,"statfs":{"total":95995035648,"available":94061445120,"internally_reserved":0,"allocated":1570906112,"data_stored":1569739736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":362665012},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.492,"15min":4.0579999999999998},"min":{"1min":0.20100000000000001,"5min":0.189,"15min":0.189},"max":{"1min":8.1080000000000005,"5min":22.202000000000002,"15min":139.68000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":3.8210000000000002,"5min":3.335,"15min":4.9770000000000003},"min":{"1min":0.21099999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":16.378,"5min":19.751000000000001,"15min":141.79499999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":17.155999999999999,"5min":10.002000000000001,"15min":7.306},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.216},"max":{"1min":118.146,"5min":132.245,"15min":205.10900000000001},"last":44.890999999999998},{"interface":"front","average":{"1min":15.507999999999999,"5min":9.4429999999999996,"15min":6.8280000000000003},"min":{"1min":0.309,"5min":0.248,"15min":0.248},"max":{"1min":115.68000000000001,"5min":132.26599999999999,"15min":208.34299999999999},"last":44.728000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:38:14 2023","interfaces":[{"interface":"back","average":{"1min":7.944,"5min":7.7089999999999996,"15min":6.3390000000000004},"min":{"1min":0.33200000000000002,"5min":0.23999999999999999,"15min":0.23699999999999999},"max":{"1min":93.677000000000007,"5min":211.864,"15min":211.864},"last":0.30299999999999999},{"interface":"front","average":{"1min":8.3239999999999998,"5min":7.6470000000000002,"15min":6.9980000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":88.513999999999996,"5min":207.517,"15min":207.517},"last":0.372}]},{"osd":4,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":7.5060000000000002,"5min":3.5529999999999999,"15min":3.9140000000000001},"min":{"1min":0.46200000000000002,"5min":0.32200000000000001,"15min":0.27100000000000002},"max":{"1min":87.054000000000002,"5min":87.054000000000002,"15min":87.054000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":7.4509999999999996,"5min":3.52,"15min":3.5619999999999998},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":93.649000000000001,"5min":93.649000000000001,"15min":93.649000000000001},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":7.4450000000000003,"5min":3.343,"15min":3.5579999999999998},"min":{"1min":0.37,"5min":0.34399999999999997,"15min":0.34100000000000003},"max":{"1min":93.533000000000001,"5min":93.533000000000001,"15min":93.533000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.226,"5min":2.0009999999999999,"15min":2.762},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.399,"5min":8.0079999999999991,"15min":19.846},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":7.5270000000000001,"5min":3.4340000000000002,"15min":3.992},"min":{"1min":0.432,"5min":0.36299999999999999,"15min":0.35099999999999998},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":7.4660000000000002,"5min":3.4649999999999999,"15min":3.6219999999999999},"min":{"1min":0.42099999999999999,"5min":0.373,"15min":0.373},"max":{"1min":93.614999999999995,"5min":93.614999999999995,"15min":93.614999999999995},"last":0.499}]},{"osd":7,"last update":"Sat Jun 10 21:37:59 2023","interfaces":[{"interface":"back","average":{"1min":7.1200000000000001,"5min":3.339,"15min":3.4319999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.32400000000000001},"max":{"1min":88.436999999999998,"5min":88.436999999999998,"15min":88.436999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.6859999999999999,"5min":2.7669999999999999,"15min":3.77},"min":{"1min":0.36299999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4279999999999999,"5min":17.792999999999999,"15min":29.77},"last":0.51300000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075760,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3166136,"kb_used_data":2524976,"kb_used_omap":7,"kb_used_meta":641144,"kb_avail":90579016,"statfs":{"total":95995035648,"available":92752912384,"internally_reserved":0,"allocated":2585575424,"data_stored":2584496990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":656531537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:38:48 2023","interfaces":[{"interface":"back","average":{"1min":15.759,"5min":6.4589999999999996,"15min":5.056},"min":{"1min":0.26600000000000001,"5min":0.19,"15min":0.19},"max":{"1min":171.33699999999999,"5min":171.33699999999999,"15min":171.33699999999999},"last":1.171},{"interface":"front","average":{"1min":14.292999999999999,"5min":6.2370000000000001,"15min":5.3369999999999997},"min":{"1min":0.252,"5min":0.215,"15min":0.215},"max":{"1min":171.56200000000001,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.93200000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:38:42 2023","interfaces":[{"interface":"back","average":{"1min":20.475000000000001,"5min":6.5700000000000003,"15min":4.6310000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.19},"max":{"1min":171.286,"5min":171.286,"15min":171.286},"last":1.254},{"interface":"front","average":{"1min":4.3209999999999997,"5min":3.5459999999999998,"15min":3.113},"min":{"1min":0.22800000000000001,"5min":0.215,"15min":0.214},"max":{"1min":24.353999999999999,"5min":24.353999999999999,"15min":147.81299999999999},"last":1.508}]},{"osd":3,"last update":"Sat Jun 10 21:38:14 2023","interfaces":[{"interface":"back","average":{"1min":2.3639999999999999,"5min":4.5700000000000003,"15min":4.9020000000000001},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.29499999999999998},"max":{"1min":14.887,"5min":97.296000000000006,"15min":138.03299999999999},"last":47.688000000000002},{"interface":"front","average":{"1min":3.3380000000000001,"5min":4.8929999999999998,"15min":5.5439999999999996},"min":{"1min":0.34100000000000003,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":14.619,"5min":97.308999999999997,"15min":138.095},"last":50.493000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:38:42 2023","interfaces":[{"interface":"back","average":{"1min":11.746,"5min":3.8119999999999998,"15min":3.7149999999999999},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":171.48699999999999,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":15.371,"5min":4.8460000000000001,"15min":4.3710000000000004},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":171.18100000000001,"5min":171.18100000000001,"15min":223.28399999999999},"last":1.1919999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:38:42 2023","interfaces":[{"interface":"back","average":{"1min":11.848000000000001,"5min":3.7080000000000002,"15min":2.2309999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29699999999999999},"max":{"1min":171.303,"5min":171.303,"15min":171.303},"last":1.3420000000000001},{"interface":"front","average":{"1min":11.731999999999999,"5min":3.7759999999999998,"15min":2.5209999999999999},"min":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.32900000000000001},"max":{"1min":171.203,"5min":171.203,"15min":171.203},"last":0.314}]},{"osd":6,"last update":"Sat Jun 10 21:38:42 2023","interfaces":[{"interface":"back","average":{"1min":11.779,"5min":3.8119999999999998,"15min":2.6150000000000002},"min":{"1min":0.40899999999999997,"5min":0.36099999999999999,"15min":0.34000000000000002},"max":{"1min":171.441,"5min":171.441,"15min":171.441},"last":0.48699999999999999},{"interface":"front","average":{"1min":15.388999999999999,"5min":4.8860000000000001,"15min":3.3340000000000001},"min":{"1min":0.34399999999999997,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":171.24600000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":1.224}]},{"osd":7,"last update":"Sat Jun 10 21:38:42 2023","interfaces":[{"interface":"back","average":{"1min":11.869,"5min":7.3440000000000003,"15min":4.3209999999999997},"min":{"1min":0.39200000000000002,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":171.26499999999999,"5min":219.131,"15min":219.131},"last":1.2110000000000001},{"interface":"front","average":{"1min":11.786,"5min":7.3760000000000003,"15min":4.6210000000000004},"min":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":171.52000000000001,"5min":219.19,"15min":219.19},"last":0.51600000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824119,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2798012,"kb_used_data":2401716,"kb_used_omap":7,"kb_used_meta":396280,"kb_avail":90947140,"statfs":{"total":95995035648,"available":93129871360,"internally_reserved":0,"allocated":2459357184,"data_stored":2458200472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":405790739},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,1],"upper_bound":1024},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":4.3659999999999997,"15min":4.7060000000000004},"min":{"1min":0.23400000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":25.512,"5min":105.09,"15min":105.09},"last":0.57499999999999996},{"interface":"front","average":{"1min":5.2249999999999996,"5min":4.5650000000000004,"15min":4.5960000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.216},"max":{"1min":25.472999999999999,"5min":99.852999999999994,"15min":99.852999999999994},"last":0.317}]},{"osd":1,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":4.2240000000000002,"15min":3.8799999999999999},"min":{"1min":0.27800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.8070000000000004,"5min":94.766000000000005,"15min":94.766000000000005},"last":0.87},{"interface":"front","average":{"1min":5.992,"5min":4.7469999999999999,"15min":3.7530000000000001},"min":{"1min":0.30499999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":41.18,"5min":94.691000000000003,"15min":94.691000000000003},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":3.23,"5min":3.8849999999999998,"15min":7.8070000000000004},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":27.911000000000001,"5min":100.676,"15min":100.676},"last":1.006},{"interface":"front","average":{"1min":5.9240000000000004,"5min":5.3659999999999997,"15min":9.3650000000000002},"min":{"1min":0.33100000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":28.09,"5min":103.032,"15min":103.032},"last":4.8259999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.3560000000000001,"15min":1.391},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.6070000000000002,"5min":13.417,"15min":13.417},"last":0.377},{"interface":"front","average":{"1min":1.714,"5min":2.1190000000000002,"15min":2.5379999999999998},"min":{"1min":0.35599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.2309999999999999,"5min":10.616,"15min":19.728000000000002},"last":0.93400000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7440000000000002,"5min":1.8140000000000001,"15min":1.4690000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.32400000000000001},"max":{"1min":17.298999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.5629999999999999,"5min":1.548,"15min":1.48},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33100000000000002},"max":{"1min":5.0380000000000003,"5min":13.470000000000001,"15min":13.470000000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.1989999999999998,"15min":2.581},"min":{"1min":0.39500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.1980000000000004,"5min":10.537000000000001,"15min":19.805},"last":0.96399999999999997},{"interface":"front","average":{"1min":2.3900000000000001,"5min":1.764,"15min":1.4910000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":17.327000000000002,"5min":17.327000000000002,"15min":17.327000000000002},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:38:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.8250000000000002,"15min":2.798},"min":{"1min":0.379,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.1559999999999997,"5min":79.984999999999999,"15min":79.984999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":2.7829999999999999,"5min":2.5590000000000002,"15min":1.7390000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":17.251000000000001,"5min":80.272999999999996,"15min":80.272999999999996},"last":0.61499999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019385,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823157,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":853380,"kb_used_data":296700,"kb_used_omap":0,"kb_used_meta":556672,"kb_avail":92891772,"statfs":{"total":95995035648,"available":95121174528,"internally_reserved":0,"allocated":303820800,"data_stored":303254622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":570032128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:37:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":2.5459999999999998,"15min":3.3330000000000002},"min":{"1min":0.30199999999999999,"5min":0.23799999999999999,"15min":0.22500000000000001},"max":{"1min":18.562000000000001,"5min":65.697000000000003,"15min":134.71899999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.3220000000000001,"5min":2.6389999999999998,"15min":3.4060000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":19.227,"5min":65.855000000000004,"15min":135.386},"last":0.308}]},{"osd":1,"last update":"Sat Jun 10 21:38:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":3.0910000000000002,"15min":4.4980000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":8.5730000000000004,"5min":71.349000000000004,"15min":149.786},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.8650000000000002,"5min":2.8119999999999998,"15min":4.4489999999999998},"min":{"1min":0.40200000000000002,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":8.7560000000000002,"5min":71.331999999999994,"15min":149.52500000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:38:37 2023","interfaces":[{"interface":"back","average":{"1min":3.7309999999999999,"5min":4.3159999999999998,"15min":5.3440000000000003},"min":{"1min":0.35099999999999998,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":19.224,"5min":111.929,"15min":186.96700000000001},"last":324.21600000000001},{"interface":"front","average":{"1min":2.2450000000000001,"5min":4.0620000000000003,"15min":5.1639999999999997},"min":{"1min":0.437,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.2430000000000003,"5min":111.896,"15min":186.84899999999999},"last":324.16300000000001}]},{"osd":3,"last update":"Sat Jun 10 21:38:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8839999999999999,"5min":3.1440000000000001,"15min":4.173},"min":{"1min":0.40300000000000002,"5min":0.31900000000000001,"15min":0.29899999999999999},"max":{"1min":17.677,"5min":79.034999999999997,"15min":128.089},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.883,"5min":3.4430000000000001,"15min":4.4189999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":17.321999999999999,"5min":77.804000000000002,"15min":128.24299999999999},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.39800000000000002,"15min":1.2709999999999999},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":0.52000000000000002,"5min":1.3480000000000001,"15min":165.73699999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.48999999999999999,"15min":1.3140000000000001},"min":{"1min":0.28999999999999998,"5min":0.25,"15min":0.25},"max":{"1min":4.0940000000000003,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.47299999999999998,"15min":0.99199999999999999},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.22},"max":{"1min":2.1909999999999998,"5min":2.1909999999999998,"15min":29.760000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.48699999999999999,"15min":0.94799999999999995},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.29199999999999998},"max":{"1min":2.194,"5min":2.194,"15min":28.995999999999999},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":2.0840000000000001,"15min":1.276},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.28199999999999997},"max":{"1min":0.51100000000000001,"5min":136.904,"15min":136.904},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":2.0859999999999999,"15min":1.4870000000000001},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.26900000000000002},"max":{"1min":0.65200000000000002,"5min":137.661,"15min":137.661},"last":0.53800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1560150016,"data_stored":1560150016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1877413888,"data_stored":1877409816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2576027648,"data_stored":2576023576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2449997824,"data_stored":2449997824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1723662336,"data_stored":1723662336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":305594368,"data_stored":305594368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2832338944,"data_stored":2832334872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:38:54.282 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:38:54.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_7 pg_num 26 2023-06-10T21:38:54.392 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:54.394 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:54.407 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:54.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:54 smithi152 ceph-mon[156819]: pgmap v1266: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 239 op/s; 21472/187971 objects degraded (11.423%); 24704/187971 objects misplaced (13.142%); 27 MiB/s, 430 objects/s recovering 2023-06-10T21:38:54.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:54.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:54.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:54.802 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:54.962 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:54.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:54.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:55.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:38:55.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:55.408 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:38:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:55.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:38:55.526 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:55.673 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:55.836 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:55.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:55.941 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:38:55.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:38:55.973 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:56.021 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:38:56.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:56.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:38:56.083 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:56.083 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:56.126 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:38:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:56 smithi152 ceph-mon[156819]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 323 op/s; 21195/184716 objects degraded (11.474%); 24275/184716 objects misplaced (13.142%); 26 MiB/s, 411 objects/s recovering 2023-06-10T21:38:56.153 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21472/187971 objects degraded (11.423%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:56.156 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:56.307 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:38:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:56.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:38:56.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:56 smithi064 ceph-mon[180297]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 323 op/s; 21195/184716 objects degraded (11.474%); 24275/184716 objects misplaced (13.142%); 26 MiB/s, 411 objects/s recovering 2023-06-10T21:38:56.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21472/187971 objects degraded (11.423%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:56.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:56 smithi064 ceph-mon[186706]: pgmap v1267: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 323 op/s; 21195/184716 objects degraded (11.474%); 24275/184716 objects misplaced (13.142%); 26 MiB/s, 411 objects/s recovering 2023-06-10T21:38:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21472/187971 objects degraded (11.423%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:38:56.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:38:56.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:56.652 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:38:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:56.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:38:56.722 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:56.767 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:38:56.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:56.892 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:38:56.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:38:57.281 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:38:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:57.382 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:38:57.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:38:57.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:57.579 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:57.935 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:38:57.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:38:57.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:38:58.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:38:58.110 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:38:58.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:38:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/6570254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:58.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:58 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/6570254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:58.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:58 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:58.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/6570254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:58.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:58 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:38:59.071 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:38:59.071 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:38:59.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 21195/184452 objects degraded (11.491%); 24113/184452 objects misplaced (13.073%); 24 MiB/s, 385 objects/s recovering 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: osdmap e341: 8 total, 8 up, 6 in 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:38:59 smithi152 ceph-mon[156819]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 21195/184452 objects degraded (11.491%); 24113/184452 objects misplaced (13.073%); 24 MiB/s, 385 objects/s recovering 2023-06-10T21:38:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:59.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: osdmap e341: 8 total, 8 up, 6 in 2023-06-10T21:38:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[186706]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: pgmap v1268: 89 pgs: 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 298 op/s; 21195/184452 objects degraded (11.491%); 24113/184452 objects misplaced (13.073%); 24 MiB/s, 385 objects/s recovering 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: osdmap e341: 8 total, 8 up, 6 in 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:38:59 smithi064 ceph-mon[180297]: failed to encode map e341 with expected crc 2023-06-10T21:38:59.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:39:00.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:00 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:39:00.085 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:00 smithi064 ceph-mon[180297]: pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/180129 objects degraded (11.546%); 22682/180129 objects misplaced (12.592%); 19 MiB/s, 310 objects/s recovering 2023-06-10T21:39:00.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:00 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:39:00.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:00 smithi152 ceph-mon[156819]: pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/180129 objects degraded (11.546%); 22682/180129 objects misplaced (12.592%); 19 MiB/s, 310 objects/s recovering 2023-06-10T21:39:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:00 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:39:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:00 smithi064 ceph-mon[186706]: pgmap v1270: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/180129 objects degraded (11.546%); 22682/180129 objects misplaced (12.592%); 19 MiB/s, 310 objects/s recovering 2023-06-10T21:39:00.408 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:39:01.097 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:01.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:01.213 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:01.245 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:01.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20797/180129 objects degraded (11.546%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:01.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20797/180129 objects degraded (11.546%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20797/180129 objects degraded (11.546%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:02.081 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:02.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:02.185 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:02.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:02 smithi064 ceph-mon[180297]: pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/179865 objects degraded (11.563%); 22529/179865 objects misplaced (12.526%); 19 MiB/s, 306 objects/s recovering 2023-06-10T21:39:02.219 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:02 smithi064 ceph-mon[186706]: pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/179865 objects degraded (11.563%); 22529/179865 objects misplaced (12.526%); 19 MiB/s, 306 objects/s recovering 2023-06-10T21:39:02.313 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:02.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:02 smithi152 ceph-mon[156819]: pgmap v1271: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 305 op/s; 20797/179865 objects degraded (11.563%); 22529/179865 objects misplaced (12.526%); 19 MiB/s, 306 objects/s recovering 2023-06-10T21:39:02.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:02.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:02.628 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:02.668 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:39:02.747 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:39:02.761 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:02.791 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:39:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:39:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:39:02.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:39:02.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:02.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:02.986 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:39:03.036 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:39:03.054 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:03.080 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:39:03.111 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:39:03.133 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 12 pg_num to 26 2023-06-10T21:39:03.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/854462452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:03.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:39:03.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:02] "GET /metrics HTTP/1.1" 200 224253 "" "Prometheus/2.33.4" 2023-06-10T21:39:03.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/854462452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:03.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:39:03.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.161 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.162 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:39:03.202 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:03.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/854462452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:03.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:39:03.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:39:04.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:39:04.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:04.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:04.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:39:04.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: osdmap e342: 8 total, 8 up, 6 in 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 20677/178626 objects degraded (11.576%); 21675/178626 objects misplaced (12.134%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:39:04.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:39:04.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:39:04.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: osdmap e342: 8 total, 8 up, 6 in 2023-06-10T21:39:04.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.419 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:04.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 20677/178626 objects degraded (11.576%); 21675/178626 objects misplaced (12.134%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:39:04.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:39:04.475 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1523033235' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: osdmap e342: 8 total, 8 up, 6 in 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: failed to encode map e342 with expected crc 2023-06-10T21:39:04.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: pgmap v1273: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 78 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 253 op/s; 20677/178626 objects degraded (11.576%); 21675/178626 objects misplaced (12.134%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:39:04.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:39:04.648 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:04.686 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:39:05.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:39:05.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:05 smithi152 ceph-mon[156819]: osdmap e343: 8 total, 8 up, 6 in 2023-06-10T21:39:05.245 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:05 smithi152 ceph-mon[156819]: failed to encode map e343 with expected crc 2023-06-10T21:39:05.245 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:05 smithi152 ceph-mon[156819]: failed to encode map e343 with expected crc 2023-06-10T21:39:05.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:39:05.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[180297]: osdmap e343: 8 total, 8 up, 6 in 2023-06-10T21:39:05.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[180297]: failed to encode map e343 with expected crc 2023-06-10T21:39:05.404 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[180297]: failed to encode map e343 with expected crc 2023-06-10T21:39:05.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:39:05.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[186706]: osdmap e343: 8 total, 8 up, 6 in 2023-06-10T21:39:05.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[186706]: failed to encode map e343 with expected crc 2023-06-10T21:39:05.405 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:05 smithi064 ceph-mon[186706]: failed to encode map e343 with expected crc 2023-06-10T21:39:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: failed to encode map e343 with expected crc 2023-06-10T21:39:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: osdmap e344: 8 total, 8 up, 6 in 2023-06-10T21:39:06.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: 12.2 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: 12.4 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: 12.3 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: 12.8 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: 12.0 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: pgmap v1276: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 13094/175581 objects degraded (7.458%); 17618/175581 objects misplaced (10.034%); 14 MiB/s, 226 objects/s recovering 2023-06-10T21:39:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20677/178626 objects degraded (11.576%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[180297]: osdmap e345: 8 total, 8 up, 6 in 2023-06-10T21:39:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: failed to encode map e343 with expected crc 2023-06-10T21:39:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: osdmap e344: 8 total, 8 up, 6 in 2023-06-10T21:39:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: 12.2 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: 12.4 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: 12.3 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: 12.8 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: 12.0 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: pgmap v1276: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 13094/175581 objects degraded (7.458%); 17618/175581 objects misplaced (10.034%); 14 MiB/s, 226 objects/s recovering 2023-06-10T21:39:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20677/178626 objects degraded (11.576%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:06 smithi064 ceph-mon[186706]: osdmap e345: 8 total, 8 up, 6 in 2023-06-10T21:39:06.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: failed to encode map e343 with expected crc 2023-06-10T21:39:06.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: osdmap e344: 8 total, 8 up, 6 in 2023-06-10T21:39:06.428 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: 12.2 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: 12.4 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: 12.3 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: 12.8 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: 12.0 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:06.429 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: failed to encode map e344 with expected crc 2023-06-10T21:39:06.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: pgmap v1276: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 252 op/s; 13094/175581 objects degraded (7.458%); 17618/175581 objects misplaced (10.034%); 14 MiB/s, 226 objects/s recovering 2023-06-10T21:39:06.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20677/178626 objects degraded (11.576%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:06.430 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:06 smithi152 ceph-mon[156819]: osdmap e345: 8 total, 8 up, 6 in 2023-06-10T21:39:06.630 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:39:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:39:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:39:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:39:06.630 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:39:07.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: 12.9 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:07.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.164 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.165 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.419 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.420 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.420 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:07 smithi152 ceph-mon[156819]: osdmap e346: 8 total, 8 up, 6 in 2023-06-10T21:39:07.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:39:07.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: 12.9 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.609 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[180297]: osdmap e346: 8 total, 8 up, 6 in 2023-06-10T21:39:07.610 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:07.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: 12.9 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:07.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: failed to encode map e345 with expected crc 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:07.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:07 smithi064 ceph-mon[186706]: osdmap e346: 8 total, 8 up, 6 in 2023-06-10T21:39:07.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:07.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:07.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:07.756 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:07.818 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:07.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:07.858 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:07.872 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:39:07.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:08.024 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:08.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.13 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.18 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.12 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.14 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.19 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:08.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: 12.10 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[186706]: pgmap v1279: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 13094/175452 objects degraded (7.463%); 17341/175452 objects misplaced (9.884%); 10 MiB/s, 160 objects/s recovering 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.299 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.13 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.18 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.12 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.14 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.19 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:08.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: 12.10 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:08.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.301 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:08 smithi064 ceph-mon[180297]: pgmap v1279: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 13094/175452 objects degraded (7.463%); 17341/175452 objects misplaced (9.884%); 10 MiB/s, 160 objects/s recovering 2023-06-10T21:39:08.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:08.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:08.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:08.406 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.13 continuing backfill to osd.5 from (317'2085,342'7429] MIN to 342'7429 2023-06-10T21:39:08.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.18 continuing backfill to osd.7 from (316'1766,342'7471] MIN to 342'7471 2023-06-10T21:39:08.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: failed to encode map e346 with expected crc 2023-06-10T21:39:08.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.12 continuing backfill to osd.0 from (319'2337,342'7404] MIN to 342'7404 2023-06-10T21:39:08.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.14 continuing backfill to osd.0 from (319'2329,342'7364] MIN to 342'7364 2023-06-10T21:39:08.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.19 continuing backfill to osd.5 from (316'1686,342'7372] MIN to 342'7372 2023-06-10T21:39:08.408 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: 12.10 continuing backfill to osd.5 from (316'1580,342'7258] MIN to 342'7258 2023-06-10T21:39:08.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:08.409 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:08 smithi152 ceph-mon[156819]: pgmap v1279: 99 pgs: 1 peering, 3 remapped+peering, 10 unknown, 1 active+recovering+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 77 active+clean; 3.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 13094/175452 objects degraded (7.463%); 17341/175452 objects misplaced (9.884%); 10 MiB/s, 160 objects/s recovering 2023-06-10T21:39:08.474 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:08.506 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:09.361 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:09.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:09.466 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:09.501 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:09.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:39:09.814 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:09.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:09.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:09.951 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:10.005 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:10.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:10.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:10.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:10 smithi064 ceph-mon[180297]: pgmap v1280: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:10.218 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:10.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:10 smithi152 ceph-mon[156819]: pgmap v1280: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:10 smithi064 ceph-mon[186706]: pgmap v1280: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 268 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:10.537 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:39:10.538 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:39:10.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:39:10.538 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85133 11.3 11.3e76a33 (undecoded) ondisk+retry+write+known_if_redirected e288)", 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 376.01237272100002, 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995, 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.539 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85133 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520825+0000", 2023-06-10T21:39:10.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520824+0000", 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.792e-06 2023-06-10T21:39:10.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520829+0000", 2023-06-10T21:39:10.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0971999999999999e-05 2023-06-10T21:39:10.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520840+0000", 2023-06-10T21:39:10.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9069999999999998e-06 2023-06-10T21:39:10.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.520845+0000", 2023-06-10T21:39:10.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032454789999999999 2023-06-10T21:39:10.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524090+0000", 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.843e-06 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.524095+0000", 2023-06-10T21:39:10.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98421531600000001 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508311+0000", 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7920000000000004e-06 2023-06-10T21:39:10.548 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508320+0000", 2023-06-10T21:39:10.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98749531999999995 2023-06-10T21:39:10.550 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.550 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.550 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:85152 11.5 11.363bf865 (undecoded) ondisk+write+known_if_redirected e288)", 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 375.991684959, 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995, 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.551 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.552 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.552 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.552 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.552 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 85152 2023-06-10T21:39:10.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.553 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:39:10.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541513+0000", 2023-06-10T21:39:10.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.556 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541512+0000", 2023-06-10T21:39:10.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9899999999999999e-06 2023-06-10T21:39:10.556 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.556 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541516+0000", 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0390000000000005e-06 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541524+0000", 2023-06-10T21:39:10.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2599999999999999e-06 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541528+0000", 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4838000000000001e-05 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541563+0000", 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.95e-06 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:54.541568+0000", 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96671271599999997 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508281+0000", 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1725999999999999e-05 2023-06-10T21:39:10.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:32:55.508293+0000", 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96677972999999995 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.561 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155021 11.3 11:c69a6bdd:::benchmark_data_smithi064_385725_object70748:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 238.44559076199999, 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99854388599999999, 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.562 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155021 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087607+0000", 2023-06-10T21:39:10.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087606+0000", 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0050000000000001e-06 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087612+0000", 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6149999999999997e-06 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087620+0000", 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2990000000000002e-06 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087627+0000", 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6254999999999998e-05 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087663+0000", 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5990000000000002e-06 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.087672+0000", 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95657830200000005 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044250+0000", 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115345 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044365+0000", 2023-06-10T21:39:10.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00053313099999999995 2023-06-10T21:39:10.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.044898+0000", 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017003877000000001 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.061902+0000", 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018346986999999999 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080249+0000", 2023-06-10T21:39:10.572 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0058076330000000004 2023-06-10T21:39:10.573 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086057+0000", 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4727e-05 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086112+0000", 2023-06-10T21:39:10.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9286000000000003e-05 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086151+0000", 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99854388599999999 2023-06-10T21:39:10.575 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155026 11.3 11:cb0acfcd:::benchmark_data_smithi064_385725_object70753:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:39:10.576 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 238.43244716300001, 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98550636899999999, 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155026 2023-06-10T21:39:10.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100751+0000", 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100749+0000", 2023-06-10T21:39:10.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2400000000000004e-06 2023-06-10T21:39:10.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.580 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.580 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100755+0000", 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1558e-05 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100767+0000", 2023-06-10T21:39:10.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8810000000000004e-06 2023-06-10T21:39:10.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.100772+0000", 2023-06-10T21:39:10.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00034783600000000002 2023-06-10T21:39:10.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.101120+0000", 2023-06-10T21:39:10.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6319999999999997e-06 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.101128+0000", 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95303523300000004 2023-06-10T21:39:10.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054164+0000", 2023-06-10T21:39:10.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013795699999999999 2023-06-10T21:39:10.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054302+0000", 2023-06-10T21:39:10.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015307100000000001 2023-06-10T21:39:10.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:39:10.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.054455+0000", 2023-06-10T21:39:10.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0074513979999999997 2023-06-10T21:39:10.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.061906+0000", 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018387883000000001 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080294+0000", 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059022859999999996 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086196+0000", 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5917000000000002e-05 2023-06-10T21:39:10.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086232+0000", 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4827000000000001e-05 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086257+0000", 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98550636899999999 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155031 11.3 11:c81286d5:::benchmark_data_smithi064_385725_object70758:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 238.42664746599999, 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.979807235, 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.593 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155031 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106550+0000", 2023-06-10T21:39:10.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106549+0000", 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.64e-06 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.596 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106555+0000", 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0412e-05 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106566+0000", 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2620000000000004e-06 2023-06-10T21:39:10.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106572+0000", 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9200000000000003e-05 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106621+0000", 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7770000000000002e-06 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.106629+0000", 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95023215400000005 2023-06-10T21:39:10.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.056861+0000", 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.996e-05 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.056941+0000", 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8929000000000001e-05 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:39:10.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.057040+0000", 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.008220148 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065260+0000", 2023-06-10T21:39:10.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0150592 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080319+0000", 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059690949999999998 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086288+0000", 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7549999999999998e-05 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086326+0000", 2023-06-10T21:39:10.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1538000000000002e-05 2023-06-10T21:39:10.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.086358+0000", 2023-06-10T21:39:10.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.979807235 2023-06-10T21:39:10.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.607 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.607 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.36159.0:155045 11.3 11:cf5ff534:::benchmark_data_smithi064_385725_object70772:head [delete] snapc 0=[] ondisk+write+known_if_redirected e305)", 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 238.40538701599999, 2023-06-10T21:39:10.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96627055699999997, 2023-06-10T21:39:10.609 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.609 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.36159", 2023-06-10T21:39:10.609 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155045 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127811+0000", 2023-06-10T21:39:10.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127810+0000", 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.54e-06 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127814+0000", 2023-06-10T21:39:10.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2520000000000002e-06 2023-06-10T21:39:10.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127821+0000", 2023-06-10T21:39:10.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3760000000000001e-06 2023-06-10T21:39:10.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127825+0000", 2023-06-10T21:39:10.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0646999999999997e-05 2023-06-10T21:39:10.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127886+0000", 2023-06-10T21:39:10.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9740000000000008e-06 2023-06-10T21:39:10.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:12.127895+0000", 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93078737700000003 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058682+0000", 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104113 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058786+0000", 2023-06-10T21:39:10.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2851999999999998e-05 2023-06-10T21:39:10.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:39:10.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.058879+0000", 2023-06-10T21:39:10.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063842059999999999 2023-06-10T21:39:10.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.065264+0000", 2023-06-10T21:39:10.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015182453 2023-06-10T21:39:10.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.080446+0000", 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013573333999999999 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094019+0000", 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5673000000000002e-05 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094055+0000", 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6285999999999999e-05 2023-06-10T21:39:10.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:13.094081+0000", 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96627055699999997 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104489 12.1 12:8d377744:::benchmark_data_smithi064_442287_object16439:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.232954+0000", 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300244223999997, 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5637393500000001, 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.626 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104489 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232954+0000", 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232954+0000", 2023-06-10T21:39:10.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232952+0000", 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.369e-06 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232957+0000", 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.85e-06 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232966+0000", 2023-06-10T21:39:10.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8829999999999997e-06 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232971+0000", 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5308000000000001e-05 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233016+0000", 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5732000000000001e-05 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233032+0000", 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97558294499999998 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208615+0000", 2023-06-10T21:39:10.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0725e-05 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208625+0000", 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5499385619999999 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.634 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.758564+0000", 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012209699999999999 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:39:10.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.758686+0000", 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037929609000000003 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.796616+0000", 2023-06-10T21:39:10.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7311999999999994e-05 2023-06-10T21:39:10.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.796693+0000", 2023-06-10T21:39:10.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5637393500000001 2023-06-10T21:39:10.638 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.638 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.638 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.639 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104491 12.1 12:8e62e27b:::benchmark_data_smithi064_442287_object16441:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:39:10.639 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.232982+0000", 2023-06-10T21:39:10.639 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.30021618, 2023-06-10T21:39:10.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5639637620000002, 2023-06-10T21:39:10.640 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.640 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.641 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.641 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.641 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.642 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104491 2023-06-10T21:39:10.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.642 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232982+0000", 2023-06-10T21:39:10.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232982+0000", 2023-06-10T21:39:10.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232981+0000", 2023-06-10T21:39:10.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8250000000000001e-06 2023-06-10T21:39:10.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232984+0000", 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.156e-06 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232990+0000", 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1189999999999999e-06 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.647 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.232994+0000", 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9278999999999999e-05 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233063+0000", 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3310000000000003e-06 2023-06-10T21:39:10.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233069+0000", 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97563514100000004 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208704+0000", 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0234999999999999e-05 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208714+0000", 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5502250230000001 2023-06-10T21:39:10.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.758939+0000", 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000126122 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.759065+0000", 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037825590999999999 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.796891+0000", 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4450000000000002e-05 2023-06-10T21:39:10.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.796945+0000", 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5639637620000002 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104493 12.1 12:8345ea0a:::benchmark_data_smithi064_442287_object16443:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.233012+0000", 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300185988000003, 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5642183539999999, 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.654 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104493 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233012+0000", 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233012+0000", 2023-06-10T21:39:10.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233011+0000", 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.937e-06 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233014+0000", 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9800000000000003e-06 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233020+0000", 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.075e-06 2023-06-10T21:39:10.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233027+0000", 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5887000000000001e-05 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233093+0000", 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5529999999999993e-06 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233102+0000", 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97563759900000002 2023-06-10T21:39:10.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208739+0000", 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.503e-06 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.661 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208747+0000", 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5503507839999999 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.759098+0000", 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5711999999999996e-05 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.759173+0000", 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.037999342999999998 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.797173+0000", 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7459e-05 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.797230+0000", 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5642183539999999 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.664 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104498 12.4 12:2978aaab:::benchmark_data_smithi064_442287_object16448:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.233032+0000", 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300165296999999, 2023-06-10T21:39:10.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1630177229999998, 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104498 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.666 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233032+0000", 2023-06-10T21:39:10.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233032+0000", 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233032+0000", 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5959999999999998e-06 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233035+0000", 2023-06-10T21:39:10.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0491e-05 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233045+0000", 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.799e-06 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233049+0000", 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1235000000000001e-05 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233090+0000", 2023-06-10T21:39:10.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1627000000000001e-05 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233102+0000", 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97518698599999998 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208289+0000", 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.341e-05 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208302+0000", 2023-06-10T21:39:10.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1866334039999999 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.394936+0000", 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001114613 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396050+0000", 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1630177229999998 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104500 12.4 12:2659094c:::benchmark_data_smithi064_442287_object16450:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.233061+0000", 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300136246999998, 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1630887429999999, 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104500 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233061+0000", 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233061+0000", 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233061+0000", 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7510000000000008e-06 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233069+0000", 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3650000000000004e-06 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233077+0000", 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2639999999999999e-06 2023-06-10T21:39:10.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233082+0000", 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000454497 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233537+0000", 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9399999999999999e-06 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233543+0000", 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97478727399999998 2023-06-10T21:39:10.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208330+0000", 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208340+0000", 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1877507389999999 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396091+0000", 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9604000000000001e-05 2023-06-10T21:39:10.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396150+0000", 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1630887429999999 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.685 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104505 12.4 12:275ff69c:::benchmark_data_smithi064_442287_object16455:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.233090+0000", 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300107730999997, 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1869930700000002, 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.686 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104505 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233090+0000", 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233090+0000", 2023-06-10T21:39:10.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233090+0000", 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9330000000000001e-06 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233092+0000", 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1761e-05 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233104+0000", 2023-06-10T21:39:10.690 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0049999999999998e-06 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233108+0000", 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00089145399999999997 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.234000+0000", 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.301e-06 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.234004+0000", 2023-06-10T21:39:10.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97464323799999997 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208647+0000", 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1114999999999999e-05 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208658+0000", 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.187510343 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396169+0000", 2023-06-10T21:39:10.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021275299999999999 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396381+0000", 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022759399999999999 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,3", 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.396609+0000", 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064027889999999999 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.403012+0000", 2023-06-10T21:39:10.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0099599230000000007 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.412972+0000", 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069481990000000004 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.419920+0000", 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7725999999999998e-05 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.419958+0000", 2023-06-10T21:39:10.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0954e-05 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.420009+0000", 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4488999999999998e-05 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.420083+0000", 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1869930700000002 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104507 12.1 12:82da2a89:::benchmark_data_smithi064_442287_object16457:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.233116+0000", 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.300082115999999, 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.568891362, 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.701 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104507 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233116+0000", 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233116+0000", 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233115+0000", 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2180000000000002e-06 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.704 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233118+0000", 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0210000000000002e-06 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233125+0000", 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.429e-06 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.705 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233127+0000", 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1464e-05 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233148+0000", 2023-06-10T21:39:10.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2080000000000002e-06 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.233153+0000", 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97561590200000003 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208769+0000", 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1721e-05 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208780+0000", 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.550428382 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.759209+0000", 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001516997 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.760726+0000", 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026897899999999997 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3", 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.760995+0000", 2023-06-10T21:39:10.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010257429 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.771252+0000", 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059001080000000003 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.777152+0000", 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091217409999999992 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.786274+0000", 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.015544751000000001 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.801819+0000", 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4381999999999994e-05 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.801883+0000", 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012396299999999999 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.802007+0000", 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.568891362 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:104509 12.4 12:21397f75:::benchmark_data_smithi064_442287_object16459:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:39:10.715 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:21.255921+0000", 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 49.277276546000003, 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1780959559999999, 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.716 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 104509 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255921+0000", 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255921+0000", 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255920+0000", 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0189999999999997e-06 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255925+0000", 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1751e-05 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255937+0000", 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1669999999999998e-06 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.720 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255942+0000", 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3832999999999999e-05 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255986+0000", 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5050000000000001e-06 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:21.255993+0000", 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95271593899999996 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208709+0000", 2023-06-10T21:39:10.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7569999999999999e-06 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:22.208719+0000", 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.192187573 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.400907+0000", 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000125568 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.401032+0000", 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018700499999999999 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,3", 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.401219+0000", 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00180002 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.403019+0000", 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0099988960000000002 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.413018+0000", 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016066123000000002 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.429084+0000", 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048021230000000002 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.433886+0000", 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9575e-05 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.433926+0000", 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1112000000000005e-05 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:23.434017+0000", 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1780959559999999 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.729 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:112463 12.1 12:86bcf6ab:::benchmark_data_smithi064_442287_object24413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e339)", 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.119070+0000", 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 23.414127505, 2023-06-10T21:39:10.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.981340765, 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112463 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.731 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119070+0000", 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119070+0000", 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119069+0000", 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.75e-06 2023-06-10T21:39:10.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119074+0000", 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4809999999999995e-06 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.734 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119084+0000", 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2689999999999996e-06 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119090+0000", 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0015495789999999999 2023-06-10T21:39:10.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.120640+0000", 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6759999999999998e-06 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.120644+0000", 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97060504199999997 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.091249+0000", 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118118 2023-06-10T21:39:10.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.091367+0000", 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101579 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:39:10.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.091469+0000", 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025388820000000001 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.094008+0000", 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024156490000000002 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.096424+0000", 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038951860000000001 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100319+0000", 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2800999999999998e-05 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100372+0000", 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9406999999999997e-05 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100411+0000", 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.981340765 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:112465 12.1 12:8535bd47:::benchmark_data_smithi064_442287_object24415:head [delete] snapc 0=[] ondisk+write+known_if_redirected e339)", 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:38:47.119623+0000", 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 23.41357519, 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98114232199999996, 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.743 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 112465 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119623+0000", 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119623+0000", 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:39:10.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119622+0000", 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8799999999999996e-06 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119627+0000", 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1085e-05 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119639+0000", 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5250000000000002e-06 2023-06-10T21:39:10.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.119645+0000", 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001009511 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.120655+0000", 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6189999999999998e-06 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:47.120657+0000", 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97337845599999995 2023-06-10T21:39:10.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.094036+0000", 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6539999999999995e-05 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.094122+0000", 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9480000000000005e-05 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.094202+0000", 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0063910909999999998 2023-06-10T21:39:10.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100593+0000", 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1396000000000006e-05 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100654+0000", 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.169e-05 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100666+0000", 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7500999999999997e-05 2023-06-10T21:39:10.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100723+0000", 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1560999999999997e-05 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:38:48.100765+0000", 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98114232199999996 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.755 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:117678 12.2 12:45599f53:::benchmark_data_smithi064_442287_object29628:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:04.217037+0000", 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.3161609030000001, 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0404769250000001, 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.756 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 117678 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217037+0000", 2023-06-10T21:39:10.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217037+0000", 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999962 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217033+0000", 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8161000000000001e-05 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217051+0000", 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1321000000000001e-05 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217072+0000", 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0654e-05 2023-06-10T21:39:10.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217083+0000", 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.14028025299999999 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.357363+0000", 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5160000000000001e-05 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.357379+0000", 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90002707400000004 2023-06-10T21:39:10.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:05.257406+0000", 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108156 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:05.257514+0000", 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0404769250000001 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:117680 12.12 12:4d0e264b:::benchmark_data_smithi064_442287_object29630:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:04.217098+0000", 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.3161001250000002, 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9250720710000002, 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.765 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 117680 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217098+0000", 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217098+0000", 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217097+0000", 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2540000000000001e-06 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217100+0000", 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6169999999999998e-06 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217106+0000", 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.771e-06 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.217110+0000", 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.132404992 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349515+0000", 2023-06-10T21:39:10.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9189999999999999e-06 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349523+0000", 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.72844511 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.077968+0000", 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8185999999999998e-05 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.077986+0000", 2023-06-10T21:39:10.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0641121 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.142098+0000", 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1644999999999996e-05 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.142170+0000", 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9250720710000002 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.774 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:117688 12.12 12:4e792fff:::benchmark_data_smithi064_442287_object29638:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:04.234881+0000", 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.2983168679999997, 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9271254899999999, 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.775 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 117688 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234881+0000", 2023-06-10T21:39:10.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234881+0000", 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.777 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234879+0000", 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5100000000000001e-06 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234886+0000", 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2768e-05 2023-06-10T21:39:10.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234899+0000", 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0142e-05 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.234909+0000", 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11463300899999999 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349542+0000", 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5979999999999999e-06 2023-06-10T21:39:10.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349547+0000", 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7284651929999999 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.078012+0000", 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9119999999999998e-06 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.078020+0000", 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.064179585 2023-06-10T21:39:10.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.142199+0000", 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100888 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.142300+0000", 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000217309 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,3", 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.142518+0000", 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001845298 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.144363+0000", 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.016764274999999999 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161127+0000", 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1879999999999997e-05 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161189+0000", 2023-06-10T21:39:10.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2069000000000001e-05 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161221+0000", 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2738999999999997e-05 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161284+0000", 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00072251399999999995 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.162006+0000", 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9271254899999999 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:117689 12.12 12:4c4f18ed:::benchmark_data_smithi064_442287_object29639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:39:10.789 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:04.292006+0000", 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.2411915860000002, 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8707593839999999, 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:39:10.790 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 117689 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292006+0000", 2023-06-10T21:39:10.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292006+0000", 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999962 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292002+0000", 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7657000000000001e-05 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292020+0000", 2023-06-10T21:39:10.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6427999999999997e-05 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292056+0000", 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.295e-05 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.292069+0000", 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.057495196999999998 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349565+0000", 2023-06-10T21:39:10.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0270000000000003e-06 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:04.349570+0000", 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.728469773 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.078039+0000", 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2010000000000002e-06 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:06.078047+0000", 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0654078490000001 2023-06-10T21:39:10.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.143454+0000", 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6240000000000001e-05 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:39:10.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.143541+0000", 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000165551 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,1,3", 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.143706+0000", 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017376366000000001 2023-06-10T21:39:10.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161083+0000", 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8378000000000001e-05 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.161161+0000", 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00088831999999999995 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.162049+0000", 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1121000000000002e-05 2023-06-10T21:39:10.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.162080+0000", 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5769000000000001e-05 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:39:10.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.162136+0000", 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00062934899999999997 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:07.162766+0000", 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8707593839999999 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.803 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.804 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.804 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:39:10.804 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:39:10.804 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:39:10.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:10.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:10.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:10.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:10.975 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:11.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20068/172425 objects degraded (11.639%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:11.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:11.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:11.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:11.350 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:11.378 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:11.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20068/172425 objects degraded (11.639%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:11.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20068/172425 objects degraded (11.639%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:11.497 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:11.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:11.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:11.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:11.822 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:39:11.909 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:11.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:12.026 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:12.059 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:12.107 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:12.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:12.214 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:12.348 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:12.524 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:12.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:12.630 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:12.648 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:39:12.649 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:39:12.649 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-06-10T21:39:12.649 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:39:12.650 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:39:12.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-06-10T21:39:12.733 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:12.975 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:12] "GET /metrics HTTP/1.1" 200 224250 "" "Prometheus/2.33.4" 2023-06-10T21:39:12.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:12.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:12.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[180297]: pgmap v1281: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 7.8 MiB/s, 124 objects/s recovering 2023-06-10T21:39:12.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4162325308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:12.976 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:13.050 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:39:13.074 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:39:13.294 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:13.294 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:13.295 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:12 smithi152 ceph-mon[156819]: pgmap v1281: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 7.8 MiB/s, 124 objects/s recovering 2023-06-10T21:39:13.295 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4162325308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:13.295 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[186706]: pgmap v1281: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 20068/172425 objects degraded (11.639%); 21616/172425 objects misplaced (12.536%); 7.8 MiB/s, 124 objects/s recovering 2023-06-10T21:39:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4162325308' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:13.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:14.366 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:39:14.366 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:14.367 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: osdmap e347: 8 total, 8 up, 6 in 2023-06-10T21:39:14.367 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.367 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.367 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:13 smithi152 ceph-mon[156819]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:39:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: osdmap e347: 8 total, 8 up, 6 in 2023-06-10T21:39:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[180297]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: osdmap e347: 8 total, 8 up, 6 in 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:13 smithi064 ceph-mon[186706]: failed to encode map e347 with expected crc 2023-06-10T21:39:14.837 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:39:14.838 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:39:14.838 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:39:15.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:14 smithi064 ceph-mon[186706]: pgmap v1283: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 20068/170127 objects degraded (11.796%); 20948/170127 objects misplaced (12.313%); 9.5 MiB/s, 152 objects/s recovering 2023-06-10T21:39:15.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:14 smithi064 ceph-mon[180297]: pgmap v1283: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 20068/170127 objects degraded (11.796%); 20948/170127 objects misplaced (12.313%); 9.5 MiB/s, 152 objects/s recovering 2023-06-10T21:39:15.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:15.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:15.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:14 smithi152 ceph-mon[156819]: pgmap v1283: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 20068/170127 objects degraded (11.796%); 20948/170127 objects misplaced (12.313%); 9.5 MiB/s, 152 objects/s recovering 2023-06-10T21:39:15.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:15.608 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:16.005 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:16.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:16.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:16.149 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:16.200 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:16.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:16.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:16.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:16.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:16 smithi064 ceph-mon[180297]: pgmap v1284: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 19469/165210 objects degraded (11.784%); 20509/165210 objects misplaced (12.414%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:39:16.382 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20068/170127 objects degraded (11.796%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:16.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:16.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:16 smithi064 ceph-mon[186706]: pgmap v1284: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 19469/165210 objects degraded (11.784%); 20509/165210 objects misplaced (12.414%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:39:16.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20068/170127 objects degraded (11.796%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:16 smithi152 ceph-mon[156819]: pgmap v1284: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 380 op/s; 19469/165210 objects degraded (11.784%); 20509/165210 objects misplaced (12.414%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:39:16.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20068/170127 objects degraded (11.796%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:16.915 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:16.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:17.036 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:17.089 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:17.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:17.858 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:39:17.861 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:17.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:17.924 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:39:17.960 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/444626907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:17.960 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/444626907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:17.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:17.999 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:39:18.001 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:18.046 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:39:18.050 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:18.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:18.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:18.082 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:39:18.126 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:39:18.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:18.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:39:18.188 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:39:18.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/444626907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:18.206 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.207 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:39:18.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:18.927 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:39:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[180297]: pgmap v1285: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 19469/164784 objects degraded (11.815%); 20307/164784 objects misplaced (12.323%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:39:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:19.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:19.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[186706]: pgmap v1285: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 19469/164784 objects degraded (11.815%); 20307/164784 objects misplaced (12.323%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:39:19.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:19.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:18 smithi152 ceph-mon[156819]: pgmap v1285: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 355 op/s; 19469/164784 objects degraded (11.815%); 20307/164784 objects misplaced (12.323%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:39:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:20.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:20 smithi064 ceph-mon[186706]: pgmap v1286: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 19325/160815 objects degraded (12.017%); 19567/160815 objects misplaced (12.167%); 9.2 MiB/s, 146 objects/s recovering 2023-06-10T21:39:20.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:20 smithi064 ceph-mon[180297]: pgmap v1286: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 19325/160815 objects degraded (12.017%); 19567/160815 objects misplaced (12.167%); 9.2 MiB/s, 146 objects/s recovering 2023-06-10T21:39:20.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:20 smithi152 ceph-mon[156819]: pgmap v1286: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.3 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 19325/160815 objects degraded (12.017%); 19567/160815 objects misplaced (12.167%); 9.2 MiB/s, 146 objects/s recovering 2023-06-10T21:39:20.699 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 0, 2, 5] out_osds: [6, 4] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:39:20.699 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:39:20.699 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 3, 7, 0, 2, 5] 2023-06-10T21:39:20.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 0 2023-06-10T21:39:20.768 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:20.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:20.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:20.929 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:21.145 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 19325/160815 objects degraded (12.017%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:21.145 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 19325/160815 objects degraded (12.017%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:21.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:21.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:21.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 19325/160815 objects degraded (12.017%), 9 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:21.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:21.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:21.598 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:21.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:21.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:21.823 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:22.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:22.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:22.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:22.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:22.406 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:22.407 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:22.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:22 smithi064 ceph-mon[180297]: pgmap v1287: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 18884/158262 objects degraded (11.932%); 19429/158262 objects misplaced (12.276%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:22.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:22 smithi064 ceph-mon[186706]: pgmap v1287: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 18884/158262 objects degraded (11.932%); 19429/158262 objects misplaced (12.276%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:22.420 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:22.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:22 smithi152 ceph-mon[156819]: pgmap v1287: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 18884/158262 objects degraded (11.932%); 19429/158262 objects misplaced (12.276%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:39:22.453 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:22.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:22.559 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:22.569 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:39:22.569 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:39:22.650 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:22.885 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:22] "GET /metrics HTTP/1.1" 200 224292 "" "Prometheus/2.33.4" 2023-06-10T21:39:23.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:23.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:23.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:39:23.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:23.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:23.557 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:23.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:23.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:23 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:23 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.664 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:23 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:23.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:24 smithi064 ceph-mon[180297]: pgmap v1288: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 18884/155988 objects degraded (12.106%); 18789/155988 objects misplaced (12.045%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:39:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:24 smithi064 ceph-mon[186706]: pgmap v1288: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 18884/155988 objects degraded (12.106%); 18789/155988 objects misplaced (12.045%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:39:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:24 smithi152 ceph-mon[156819]: pgmap v1288: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 18884/155988 objects degraded (12.106%); 18789/155988 objects misplaced (12.045%); 12 MiB/s, 186 objects/s recovering 2023-06-10T21:39:25.020 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:25.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:25.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:25.124 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:25.152 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:25.200 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:25.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:25.305 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:25.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:39:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:39:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:39:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:39:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:39:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:39:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:26.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:39:26.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:26 smithi064 ceph-mon[186706]: pgmap v1289: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 18292/151260 objects degraded (12.093%); 18348/151260 objects misplaced (12.130%); 13 MiB/s, 207 objects/s recovering 2023-06-10T21:39:26.367 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18884/155988 objects degraded (12.106%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:26.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:26 smithi064 ceph-mon[180297]: pgmap v1289: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 18292/151260 objects degraded (12.093%); 18348/151260 objects misplaced (12.130%); 13 MiB/s, 207 objects/s recovering 2023-06-10T21:39:26.375 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18884/155988 objects degraded (12.106%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:26.513 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:26.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:26.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:26 smithi152 ceph-mon[156819]: pgmap v1289: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 18292/151260 objects degraded (12.093%); 18348/151260 objects misplaced (12.130%); 13 MiB/s, 207 objects/s recovering 2023-06-10T21:39:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18884/155988 objects degraded (12.106%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:26.715 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:27.509 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:27.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:27.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:27.621 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:27.625 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:27.650 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:27.702 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:27.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:27.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:27.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:27.943 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:28.152 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:28 smithi064 ceph-mon[180297]: pgmap v1290: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 18292/150834 objects degraded (12.127%); 18167/150834 objects misplaced (12.044%); 11 MiB/s, 172 objects/s recovering 2023-06-10T21:39:28.152 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:28 smithi064 ceph-mon[186706]: pgmap v1290: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 18292/150834 objects degraded (12.127%); 18167/150834 objects misplaced (12.044%); 11 MiB/s, 172 objects/s recovering 2023-06-10T21:39:28.177 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:28.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:28.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:28.409 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:28.437 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:28 smithi152 ceph-mon[156819]: pgmap v1290: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 398 op/s; 18292/150834 objects degraded (12.127%); 18167/150834 objects misplaced (12.044%); 11 MiB/s, 172 objects/s recovering 2023-06-10T21:39:28.439 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:28.490 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:28.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:28.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:28.715 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:29.221 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:39:29.227 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.0. 2023-06-10T21:39:29.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:29.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:29.350 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:29.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/912362513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3634494063' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[186706]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:39:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 conmon[180272]: 2023-06-10T21:39:29.102+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e347 definitely_dead 0 2023-06-10T21:39:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/912362513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3634494063' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:29 smithi064 ceph-mon[180297]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:39:29.419 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:29.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/912362513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:39:29.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3634494063' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-06-10T21:39:29.425 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:29 smithi152 ceph-mon[156819]: Client client.admin marked osd.0 out, while it was still marked up 2023-06-10T21:39:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:29.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:29.527 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:29.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:29.595 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:29.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:29.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:29.703 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:29.729 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:29.769 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:29.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:29.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:29.906 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:29.948 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:29.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:29.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:30.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:30.080 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:30.122 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:30.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:30.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:30.270 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:30.320 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:30.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:30.425 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:39:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:39:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: osdmap e348: 8 total, 8 up, 5 in 2023-06-10T21:39:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:30 smithi152 ceph-mon[156819]: pgmap v1292: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 18165/147276 objects degraded (12.334%); 17579/147276 objects misplaced (11.936%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:39:30.453 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: osdmap e348: 8 total, 8 up, 5 in 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[186706]: pgmap v1292: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 18165/147276 objects degraded (12.334%); 17579/147276 objects misplaced (11.936%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: osdmap e348: 8 total, 8 up, 5 in 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.455 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: failed to encode map e348 with expected crc 2023-06-10T21:39:30.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:30 smithi064 ceph-mon[180297]: pgmap v1292: 99 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 450 op/s; 18165/147276 objects degraded (12.334%); 17579/147276 objects misplaced (11.936%); 11 MiB/s, 183 objects/s recovering 2023-06-10T21:39:30.458 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:31.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e348 with expected crc 2023-06-10T21:39:31.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: osdmap e349: 8 total, 8 up, 5 in 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.18 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7778 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.12 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7718 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.4 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7693 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.14 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7674 2023-06-10T21:39:31.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.c continuing backfill to osd.5 from (320'3473,347'8200] MIN to 347'8200 2023-06-10T21:39:31.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.18 continuing backfill to osd.7 from (316'1766,347'7778] MIN to 347'7778 2023-06-10T21:39:31.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: 12.2 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7720 2023-06-10T21:39:31.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18165/147276 objects degraded (12.334%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:31.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e348 with expected crc 2023-06-10T21:39:31.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: osdmap e349: 8 total, 8 up, 5 in 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.18 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7778 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.12 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7718 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.4 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7693 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.14 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7674 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.c continuing backfill to osd.5 from (320'3473,347'8200] MIN to 347'8200 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.18 continuing backfill to osd.7 from (316'1766,347'7778] MIN to 347'7778 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: 12.2 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7720 2023-06-10T21:39:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18165/147276 objects degraded (12.334%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e348 with expected crc 2023-06-10T21:39:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: osdmap e349: 8 total, 8 up, 5 in 2023-06-10T21:39:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.18 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7778 2023-06-10T21:39:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: failed to encode map e349 with expected crc 2023-06-10T21:39:31.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.12 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7718 2023-06-10T21:39:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.4 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7693 2023-06-10T21:39:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.14 starting backfill to osd.2 from (0'0,0'0] MAX to 347'7674 2023-06-10T21:39:31.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.c continuing backfill to osd.5 from (320'3473,347'8200] MIN to 347'8200 2023-06-10T21:39:31.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.18 continuing backfill to osd.7 from (316'1766,347'7778] MIN to 347'7778 2023-06-10T21:39:31.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: 12.2 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7720 2023-06-10T21:39:31.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18165/147276 objects degraded (12.334%), 9 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-10T21:39:32.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:32.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: osdmap e350: 8 total, 8 up, 5 in 2023-06-10T21:39:32.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.a starting backfill to osd.7 from (0'0,0'0] MAX to 347'8085 2023-06-10T21:39:32.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.e starting backfill to osd.1 from (316'1845,334'6847] MIN to 347'8002 2023-06-10T21:39:32.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.3 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7772 2023-06-10T21:39:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.8 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7769 2023-06-10T21:39:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.3 continuing backfill to osd.5 from (317'2085,347'7772] MIN to 347'7772 2023-06-10T21:39:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.13 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7756 2023-06-10T21:39:32.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.e continuing backfill to osd.5 from (320'3283,347'8002] MIN to 347'8002 2023-06-10T21:39:32.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.8 continuing backfill to osd.7 from (316'1766,347'7769] MIN to 347'7769 2023-06-10T21:39:32.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.13 continuing backfill to osd.5 from (317'2085,347'7756] MIN to 347'7756 2023-06-10T21:39:32.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7530 2023-06-10T21:39:32.481 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7574 2023-06-10T21:39:32.481 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[186706]: pgmap v1295: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 12264/145236 objects degraded (8.444%); 14388/145236 objects misplaced (9.907%); 8.3 MiB/s, 133 objects/s recovering 2023-06-10T21:39:32.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:32.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:32.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: osdmap e350: 8 total, 8 up, 5 in 2023-06-10T21:39:32.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.a starting backfill to osd.7 from (0'0,0'0] MAX to 347'8085 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.e starting backfill to osd.1 from (316'1845,334'6847] MIN to 347'8002 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.3 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7772 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.8 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7769 2023-06-10T21:39:32.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.3 continuing backfill to osd.5 from (317'2085,347'7772] MIN to 347'7772 2023-06-10T21:39:32.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.13 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7756 2023-06-10T21:39:32.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.e continuing backfill to osd.5 from (320'3283,347'8002] MIN to 347'8002 2023-06-10T21:39:32.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.8 continuing backfill to osd.7 from (316'1766,347'7769] MIN to 347'7769 2023-06-10T21:39:32.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.13 continuing backfill to osd.5 from (317'2085,347'7756] MIN to 347'7756 2023-06-10T21:39:32.485 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7530 2023-06-10T21:39:32.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7574 2023-06-10T21:39:32.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:32 smithi064 ceph-mon[180297]: pgmap v1295: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 12264/145236 objects degraded (8.444%); 14388/145236 objects misplaced (9.907%); 8.3 MiB/s, 133 objects/s recovering 2023-06-10T21:39:32.583 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:32.656 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: osdmap e350: 8 total, 8 up, 5 in 2023-06-10T21:39:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.a starting backfill to osd.7 from (0'0,0'0] MAX to 347'8085 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: failed to encode map e350 with expected crc 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.e starting backfill to osd.1 from (316'1845,334'6847] MIN to 347'8002 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.3 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7772 2023-06-10T21:39:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.8 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7769 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.3 continuing backfill to osd.5 from (317'2085,347'7772] MIN to 347'7772 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.13 starting backfill to osd.1 from (316'1985,334'6922] MIN to 347'7756 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.e continuing backfill to osd.5 from (320'3283,347'8002] MIN to 347'8002 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.8 continuing backfill to osd.7 from (316'1766,347'7769] MIN to 347'7769 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.13 continuing backfill to osd.5 from (317'2085,347'7756] MIN to 347'7756 2023-06-10T21:39:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.16 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7530 2023-06-10T21:39:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: 12.6 starting backfill to osd.5 from (0'0,0'0] MAX to 347'7574 2023-06-10T21:39:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:32 smithi152 ceph-mon[156819]: pgmap v1295: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 334 op/s; 12264/145236 objects degraded (8.444%); 14388/145236 objects misplaced (9.907%); 8.3 MiB/s, 133 objects/s recovering 2023-06-10T21:39:33.170 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:32] "GET /metrics HTTP/1.1" 200 224292 "" "Prometheus/2.33.4" 2023-06-10T21:39:33.447 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:33.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:33.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:33.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:39:33.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:33.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:33.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:39:33.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:33.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:33.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:33 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:39:33.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:33.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:33.582 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:33.633 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:33.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:33.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:33.806 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:34.046 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:34.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:34.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:34.163 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:34.189 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:34.231 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:34.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:34.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:34.339 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:34.366 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:34 smithi064 ceph-mon[180297]: pgmap v1296: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 12264/144714 objects degraded (8.475%); 13982/144714 objects misplaced (9.662%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:39:34.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:34 smithi064 ceph-mon[186706]: pgmap v1296: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 12264/144714 objects degraded (8.475%); 13982/144714 objects misplaced (9.662%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:39:34.423 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:34.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:34.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:34 smithi152 ceph-mon[156819]: pgmap v1296: 99 pgs: 3 remapped+peering, 10 peering, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 72 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 12264/144714 objects degraded (8.475%); 13982/144714 objects misplaced (9.662%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:39:34.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:34.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:35.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:35.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:35.111 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:35.246 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:35.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:39:35.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:35.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:35.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:35.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:39:35.754 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:39:35.755 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:39:35.755 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:39:35.755 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:39:35.755 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:39:35.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:39:35.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:35.834 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:39:35.881 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:39:35.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1974305457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1974305457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:35.915 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:39:35.957 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:39:35.995 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:39:36.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:39:36.063 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:36.064 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.064 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.064 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.064 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.064 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.065 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.065 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.065 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:39:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:36.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:36.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:36.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1974305457' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:37.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:36 smithi064 ceph-mon[180297]: pgmap v1297: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 311 op/s; 16274/140859 objects degraded (11.553%); 32748/140859 objects misplaced (23.249%); 12 MiB/s, 1 keys/s, 201 objects/s recovering 2023-06-10T21:39:37.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 12264/144714 objects degraded (8.475%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:37.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:36 smithi064 ceph-mon[186706]: pgmap v1297: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 311 op/s; 16274/140859 objects degraded (11.553%); 32748/140859 objects misplaced (23.249%); 12 MiB/s, 1 keys/s, 201 objects/s recovering 2023-06-10T21:39:37.158 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 12264/144714 objects degraded (8.475%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:37.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:36 smithi152 ceph-mon[156819]: pgmap v1297: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 311 op/s; 16274/140859 objects degraded (11.553%); 32748/140859 objects misplaced (23.249%); 12 MiB/s, 1 keys/s, 201 objects/s recovering 2023-06-10T21:39:37.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 12264/144714 objects degraded (8.475%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:38.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:37 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:39:38.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:37 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:39:38.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:37 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:39:38.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:38.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:38.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:38.343 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:38.371 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:38.423 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:38.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:38.529 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:38.559 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:38.768 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:38.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:39:38.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:38.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:38.816 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:38 smithi064 ceph-mon[186706]: pgmap v1298: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 284 op/s; 16274/140448 objects degraded (11.587%); 32557/140448 objects misplaced (23.181%); 12 MiB/s, 1 keys/s, 196 objects/s recovering 2023-06-10T21:39:38.822 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:38 smithi064 ceph-mon[180297]: pgmap v1298: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 284 op/s; 16274/140448 objects degraded (11.587%); 32557/140448 objects misplaced (23.181%); 12 MiB/s, 1 keys/s, 196 objects/s recovering 2023-06-10T21:39:38.880 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:38.909 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:38.958 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:38.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:39.022 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:39.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:39.170 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:39.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:38 smithi152 ceph-mon[156819]: pgmap v1298: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 284 op/s; 16274/140448 objects degraded (11.587%); 32557/140448 objects misplaced (23.181%); 12 MiB/s, 1 keys/s, 196 objects/s recovering 2023-06-10T21:39:39.394 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:39.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:39.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:39.530 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:39.580 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:39.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:39.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:39.688 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:39.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:39.947 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:39.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:39.983 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 7, 2, 5] out_osds: [6, 4, 0] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:39:39.983 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:39:39.983 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 3, 7, 2, 5] 2023-06-10T21:39:39.983 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 1 2023-06-10T21:39:40.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:40.137 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:40.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:40.337 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:40 smithi064 ceph-mon[180297]: pgmap v1299: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 16011/136740 objects degraded (11.709%); 31737/136740 objects misplaced (23.210%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2023-06-10T21:39:40.338 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:40 smithi064 ceph-mon[186706]: pgmap v1299: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 16011/136740 objects degraded (11.709%); 31737/136740 objects misplaced (23.210%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2023-06-10T21:39:40.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:40.440 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:40 smithi152 ceph-mon[156819]: pgmap v1299: 99 pgs: 2 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.8 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 287 op/s; 16011/136740 objects degraded (11.709%); 31737/136740 objects misplaced (23.210%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2023-06-10T21:39:40.470 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:40.519 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:40.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:40.624 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:40.652 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:40.702 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:40.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:40.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:40.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:39:40.837 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:40.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:40.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:40.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:41.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:41.029 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:41.078 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:41.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:41.160 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:41.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:41.211 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:41.259 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:41.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:41.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:41.364 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:41.431 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:41.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 16011/136740 objects degraded (11.709%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:41.482 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 16011/136740 objects degraded (11.709%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:41.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 16011/136740 objects degraded (11.709%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:41.484 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:41.800 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:41.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:41.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:41.915 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:42.044 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:42.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:42 smithi064 ceph-mon[186706]: pgmap v1300: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 15506/134139 objects degraded (11.560%); 31066/134139 objects misplaced (23.160%); 12 MiB/s, 0 keys/s, 192 objects/s recovering 2023-06-10T21:39:42.415 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:42 smithi064 ceph-mon[180297]: pgmap v1300: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 15506/134139 objects degraded (11.560%); 31066/134139 objects misplaced (23.160%); 12 MiB/s, 0 keys/s, 192 objects/s recovering 2023-06-10T21:39:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:42 smithi152 ceph-mon[156819]: pgmap v1300: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 340 op/s; 15506/134139 objects degraded (11.560%); 31066/134139 objects misplaced (23.160%); 12 MiB/s, 0 keys/s, 192 objects/s recovering 2023-06-10T21:39:42.747 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:42] "GET /metrics HTTP/1.1" 200 224290 "" "Prometheus/2.33.4" 2023-06-10T21:39:43.246 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:39:43.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:43.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:43.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2848757123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:43 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.539 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2848757123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:43 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:43.663 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2848757123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.664 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:43 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:39:43.665 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:43.710 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:43.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:43.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:43.846 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:43.895 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:43.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:44.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:44.079 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:44.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: osdmap e351: 8 total, 8 up, 5 in 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[186706]: pgmap v1302: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 15506/133212 objects degraded (11.640%); 30725/133212 objects misplaced (23.065%); 12 MiB/s, 0 keys/s, 195 objects/s recovering 2023-06-10T21:39:44.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:44.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:44.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: osdmap e351: 8 total, 8 up, 5 in 2023-06-10T21:39:44.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:44 smithi064 ceph-mon[180297]: pgmap v1302: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 15506/133212 objects degraded (11.640%); 30725/133212 objects misplaced (23.065%); 12 MiB/s, 0 keys/s, 195 objects/s recovering 2023-06-10T21:39:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: failed to encode map e351 with expected crc 2023-06-10T21:39:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:39:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: osdmap e351: 8 total, 8 up, 5 in 2023-06-10T21:39:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:44 smithi152 ceph-mon[156819]: pgmap v1302: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 383 op/s; 15506/133212 objects degraded (11.640%); 30725/133212 objects misplaced (23.065%); 12 MiB/s, 0 keys/s, 195 objects/s recovering 2023-06-10T21:39:45.911 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:45.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:46.017 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:46.047 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:46.097 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:46.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:46.206 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:46.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:46 smithi064 ceph-mon[180297]: pgmap v1303: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 470 op/s; 14637/126720 objects degraded (11.551%); 29264/126720 objects misplaced (23.093%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:39:46.286 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15506/133212 objects degraded (11.640%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:46.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:46 smithi064 ceph-mon[186706]: pgmap v1303: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 470 op/s; 14637/126720 objects degraded (11.551%); 29264/126720 objects misplaced (23.093%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:39:46.287 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15506/133212 objects degraded (11.640%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:46.288 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:46.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:46 smithi152 ceph-mon[156819]: pgmap v1303: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.9 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 470 op/s; 14637/126720 objects degraded (11.551%); 29264/126720 objects misplaced (23.093%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:39:46.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15506/133212 objects degraded (11.640%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:47.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:47.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:47.260 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:47.342 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:47.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:47.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:47.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:39:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:47.954 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:39:47.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:47.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:39:47.966 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85152 11.5 11:a61fdc6c:::benchmark_data_smithi064_385725_object879:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:47.967 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.44584521500002, 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.711236145, 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:47.968 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85152 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509232+0000", 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509230+0000", 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509235+0000", 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4939999999999997e-06 2023-06-10T21:39:47.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509247+0000", 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2203e-05 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509256+0000", 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0589999999999992e-06 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509326+0000", 2023-06-10T21:39:47.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9915999999999999e-05 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509339+0000", 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527999999999999e-05 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.195228+0000", 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6858897430000002 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.196887+0000", 2023-06-10T21:39:47.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016585160000000001 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.197074+0000", 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018745799999999999 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205769+0000", 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086945730000000006 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205810+0000", 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1607000000000003e-05 2023-06-10T21:39:47.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217677+0000", 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011866642 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217739+0000", 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2154000000000002e-05 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220468+0000", 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027286659999999998 2023-06-10T21:39:47.979 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85191 11.5 11:ad36ddde:::benchmark_data_smithi064_385725_object918:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.44579064700002, 2023-06-10T21:39:47.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7144825799999999, 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85191 2023-06-10T21:39:47.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509286+0000", 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:47.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509285+0000", 2023-06-10T21:39:47.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:39:47.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509290+0000", 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.203e-06 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509304+0000", 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3966e-05 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:47.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509314+0000", 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5650000000000007e-06 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509379+0000", 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5468000000000004e-05 2023-06-10T21:39:47.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509401+0000", 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1651e-05 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:47.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202573+0000", 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.693172154 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202760+0000", 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018716300000000001 2023-06-10T21:39:47.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.202937+0000", 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017642100000000001 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:47.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.217663+0000", 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014726461999999999 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220485+0000", 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028217699999999999 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220695+0000", 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000209547 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:47.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220756+0000", 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0946999999999998e-05 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223769+0000", 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030133080000000001 2023-06-10T21:39:47.993 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85250 11.5 11:ad3b70c3:::benchmark_data_smithi064_385725_object977:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.44575662699998, 2023-06-10T21:39:47.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.718711994, 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85250 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.995 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509320+0000", 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:47.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509323+0000", 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.83e-06 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509330+0000", 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.951e-06 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509336+0000", 2023-06-10T21:39:47.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1190000000000002e-06 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509513+0000", 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017721800000000001 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509522+0000", 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.969e-06 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.205836+0000", 2023-06-10T21:39:48.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6963141390000001 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206003+0000", 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016704 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.206123+0000", 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011994600000000001 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220498+0000", 2023-06-10T21:39:48.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014375188000000001 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223827+0000", 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033290490000000002 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223968+0000", 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014019100000000001 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.224024+0000", 2023-06-10T21:39:48.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6718000000000001e-05 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228032+0000", 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040080999999999997 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.006 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85263 11.5 11:a183d882:::benchmark_data_smithi064_385725_object990:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.445609623, 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7214953290000001, 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.007 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85263 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509467+0000", 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:39:48.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509471+0000", 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6689999999999997e-06 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509482+0000", 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0746e-05 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509490+0000", 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4659999999999999e-06 2023-06-10T21:39:48.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509846+0000", 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035595000000000001 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509855+0000", 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9090000000000006e-06 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209100+0000", 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6992444710000001 2023-06-10T21:39:48.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209286+0000", 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018623000000000001 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.209449+0000", 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162953 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220508+0000", 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011058678000000001 2023-06-10T21:39:48.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.223930+0000", 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003422671 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228127+0000", 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041968999999999999 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228171+0000", 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3764999999999999e-05 2023-06-10T21:39:48.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.230963+0000", 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002791689 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.019 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85264 11.5 11:a2a1b725:::benchmark_data_smithi064_385725_object991:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.44531413200002, 2023-06-10T21:39:48.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.724233897, 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85264 2023-06-10T21:39:48.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509763+0000", 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509762+0000", 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:39:48.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509767+0000", 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.135e-06 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509778+0000", 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.058e-05 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509787+0000", 2023-06-10T21:39:48.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8769999999999999e-06 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509887+0000", 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010014900000000001 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509893+0000", 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3409999999999999e-06 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214185+0000", 2023-06-10T21:39:48.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.704292031 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214314+0000", 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012892599999999999 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.214446+0000", 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013181 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.220634+0000", 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061883290000000002 2023-06-10T21:39:48.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.228085+0000", 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074512379999999998 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231052+0000", 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029661269999999998 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.231110+0000", 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8733000000000002e-05 2023-06-10T21:39:48.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.233997+0000", 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028864709999999998 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:85269 11.5 11:a8f773e1:::benchmark_data_smithi064_385725_object996:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e289)", 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 412.44528461800002, 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7386240339999999, 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.033 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 85269 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509792+0000", 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.036 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509795+0000", 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0850000000000001e-06 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509802+0000", 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9120000000000001e-06 2023-06-10T21:39:48.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.509810+0000", 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3529999999999995e-06 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510067+0000", 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000256896 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:55.510076+0000", 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0270000000000001e-06 2023-06-10T21:39:48.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234066+0000", 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7239896319999999 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234235+0000", 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016923200000000001 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.234406+0000", 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017128799999999999 2023-06-10T21:39:48.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.240469+0000", 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006062217 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245113+0000", 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046439960000000001 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245151+0000", 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8617e-05 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.245210+0000", 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9166999999999998e-05 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:32:57.248416+0000", 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032060209999999999 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104502 12.3 12:c60ebc2b:::benchmark_data_smithi064_442287_object16452:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:39:48.045 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 86.757067726000002, 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2634145729999999, 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.046 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104502 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198012+0000", 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7499999999999999e-06 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198017+0000", 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8730000000000004e-06 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198022+0000", 2023-06-10T21:39:48.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.899e-06 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.200726+0000", 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027034200000000002 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.200875+0000", 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014875899999999999 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.201009+0000", 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013431900000000001 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207565+0000", 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065557020000000001 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.208794+0000", 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012298840000000001 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229308+0000", 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020513547 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229322+0000", 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3995999999999999e-05 2023-06-10T21:39:48.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203125+0000", 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97380280500000005 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203136+0000", 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1531e-05 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461328+0000", 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2581918000000001 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461424+0000", 2023-06-10T21:39:48.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.569e-05 2023-06-10T21:39:48.058 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.059 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.059 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.060 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104504 12.3 12:c95ca6d2:::benchmark_data_smithi064_442287_object16454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:39:48.060 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:39:48.060 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 86.752814130000004, 2023-06-10T21:39:48.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.25957418, 2023-06-10T21:39:48.061 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104504 2023-06-10T21:39:48.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202261+0000", 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202268+0000", 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1489999999999997e-06 2023-06-10T21:39:48.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202282+0000", 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3679000000000001e-05 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202290+0000", 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5410000000000003e-06 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.203820+0000", 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001530954 2023-06-10T21:39:48.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.203940+0000", 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119145 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:39:48.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.204054+0000", 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114466 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.208801+0000", 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047466110000000004 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.210128+0000", 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013276329999999999 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229351+0000", 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019222966000000001 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229368+0000", 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6806000000000001e-05 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203250+0000", 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97388200400000002 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203261+0000", 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981e-05 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461757+0000", 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.258495736 2023-06-10T21:39:48.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461837+0000", 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0174999999999997e-05 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:110588 12.c 12:32a597f0:::benchmark_data_smithi064_442287_object22538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 66.846401287000006, 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32362693999999997, 2023-06-10T21:39:48.075 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 110588 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108675+0000", 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108678+0000", 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0230000000000001e-06 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108687+0000", 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2900000000000002e-06 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108692+0000", 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0270000000000003e-06 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108740+0000", 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7923000000000002e-05 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108896+0000", 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015683099999999999 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5,7", 2023-06-10T21:39:48.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.109018+0000", 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122058 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.120560+0000", 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011541332 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.129815+0000", 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092551660000000004 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.129838+0000", 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3451999999999998e-05 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432143+0000", 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30230479700000001 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432195+0000", 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1430000000000001e-05 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432303+0000", 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108051 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111271 12.e 12:71b77781:::benchmark_data_smithi064_442287_object23221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.087 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.786196794000006, 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34391442799999999, 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.088 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111271 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168883+0000", 2023-06-10T21:39:48.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2780000000000002e-06 2023-06-10T21:39:48.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168889+0000", 2023-06-10T21:39:48.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8139999999999997e-06 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168894+0000", 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.626e-06 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.169018+0000", 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124146 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.181820+0000", 2023-06-10T21:39:48.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012801554999999999 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476308+0000", 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29448750000000001 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476432+0000", 2023-06-10T21:39:48.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012422299999999999 2023-06-10T21:39:48.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476567+0000", 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013533600000000001 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.489518+0000", 2023-06-10T21:39:48.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012950721 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.489561+0000", 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2778999999999998e-05 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509183+0000", 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019622695999999999 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509231+0000", 2023-06-10T21:39:48.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7969000000000001e-05 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509276+0000", 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5182999999999998e-05 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512795+0000", 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035181790000000002 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111272 12.e 12:7d0d8fe8:::benchmark_data_smithi064_442287_object23222:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.786170833, 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34743270999999998, 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.102 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111272 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:39:48.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168908+0000", 2023-06-10T21:39:48.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.289e-06 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168913+0000", 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2440000000000001e-06 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168918+0000", 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2799999999999997e-06 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.181864+0000", 2023-06-10T21:39:48.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012946077 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182630+0000", 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076583 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486615+0000", 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30398561099999999 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486722+0000", 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106419 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486862+0000", 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140097 2023-06-10T21:39:48.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506259+0000", 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019397116999999998 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509129+0000", 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028700869999999999 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512883+0000", 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037540400000000002 2023-06-10T21:39:48.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512915+0000", 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2101999999999999e-05 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512960+0000", 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4907999999999998e-05 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516339+0000", 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033789279999999998 2023-06-10T21:39:48.114 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111276 12.e 12:70f74f42:::benchmark_data_smithi064_442287_object23226:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.778574864000007, 2023-06-10T21:39:48.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34345777999999999, 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111276 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.116 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176501+0000", 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:48.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176508+0000", 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9410000000000001e-06 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176521+0000", 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3344e-05 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176527+0000", 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.959e-06 2023-06-10T21:39:48.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182668+0000", 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061415300000000001 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:39:48.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.183316+0000", 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064711 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496369+0000", 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31305350599999998 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496497+0000", 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000128102 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496626+0000", 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012860200000000001 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509144+0000", 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012518184999999999 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512849+0000", 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003704948 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516399+0000", 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035501859999999999 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516435+0000", 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6162999999999999e-05 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516493+0000", 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7615000000000003e-05 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.519960+0000", 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034670529999999999 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111279 12.e 12:79bea3d8:::benchmark_data_smithi064_442287_object23229:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.128 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.772631782000005, 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34643435099999997, 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111279 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182444+0000", 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182451+0000", 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3679999999999996e-06 2023-06-10T21:39:48.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182464+0000", 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527999999999999e-05 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182472+0000", 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0909999999999993e-06 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.184408+0000", 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019360619999999999 2023-06-10T21:39:48.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.185139+0000", 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073120499999999996 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506300+0000", 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32116106999999999 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506438+0000", 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013793599999999999 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506589+0000", 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151132 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525267+0000", 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018678178 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525276+0000", 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9209999999999998e-06 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525311+0000", 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4999000000000002e-05 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525348+0000", 2023-06-10T21:39:48.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6294000000000002e-05 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525399+0000", 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1554999999999997e-05 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.528880+0000", 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034804789999999999 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111285 12.3 12:c9e541db:::benchmark_data_smithi064_442287_object23235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 64.761094630000002, 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33020181300000001, 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.142 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111285 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193980+0000", 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193992+0000", 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1773000000000001e-05 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.470758+0000", 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27676603100000002 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.470772+0000", 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3966e-05 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471014+0000", 2023-06-10T21:39:48.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000242725 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471167+0000", 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000152967 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471277+0000", 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109799 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.474344+0000", 2023-06-10T21:39:48.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003067354 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.481669+0000", 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073244490000000002 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.481676+0000", 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7789999999999998e-06 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524025+0000", 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042349554999999997 2023-06-10T21:39:48.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524076+0000", 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0383000000000002e-05 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524184+0000", 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108587 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.153 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117664 12.0 e342/336 12:0001adf0:::benchmark_data_smithi064_442287_object29614:head v 342'7257, mlcod=316'1811)", 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.807877779999998, 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.45376187699999998, 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147203+0000", 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1980000000000003e-06 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147209+0000", 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5380000000000002e-06 2023-06-10T21:39:48.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147213+0000", 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7040000000000002e-06 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.151470+0000", 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042562950000000002 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.151505+0000", 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4841000000000001e-05 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.600961+0000", 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44945654200000001 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117672 12.0 e342/336 12:08d95f39:::benchmark_data_smithi064_442287_object29622:head v 342'7258, mlcod=316'1811)", 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.776669658000003, 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.42256061299999997, 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:39:48.161 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.163 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.163 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:39:48.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.164 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178412+0000", 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.118e-06 2023-06-10T21:39:48.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178421+0000", 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.755e-06 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178427+0000", 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1700000000000002e-06 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.182568+0000", 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041405349999999999 2023-06-10T21:39:48.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.182601+0000", 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3775000000000001e-05 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.600968+0000", 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.418366511 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.169 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117681 12.13 12:cf7cb710:::benchmark_data_smithi064_442287_object29631:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.743754244000002, 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.889619234, 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.170 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117681 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:39:48.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211320+0000", 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211331+0000", 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0591000000000001e-05 2023-06-10T21:39:48.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211352+0000", 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0720000000000002e-05 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211370+0000", 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8448e-05 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424732+0000", 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213361354 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:48.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424740+0000", 2023-06-10T21:39:48.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6810000000000004e-06 2023-06-10T21:39:48.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.078401+0000", 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.653661219 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.078414+0000", 2023-06-10T21:39:48.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2357999999999999e-05 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.100864+0000", 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022449875 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.100942+0000", 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8227999999999997e-05 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117686 12.13 12:cc0e1ca3:::benchmark_data_smithi064_442287_object29636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.743696894000003, 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9062216680000001, 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.181 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117686 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211383+0000", 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6179999999999999e-06 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211391+0000", 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5780000000000001e-06 2023-06-10T21:39:48.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211397+0000", 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0630000000000002e-06 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424757+0000", 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21336037299999999 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424762+0000", 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1919999999999996e-06 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.083725+0000", 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.658962869 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.083737+0000", 2023-06-10T21:39:48.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1284000000000001e-05 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101248+0000", 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.017511311 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101387+0000", 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013950599999999999 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101678+0000", 2023-06-10T21:39:48.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029038499999999999 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.103061+0000", 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001383553 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.113231+0000", 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010170102 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.114525+0000", 2023-06-10T21:39:48.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012936619999999999 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117436+0000", 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029114499999999999 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117492+0000", 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5167000000000002e-05 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117602+0000", 2023-06-10T21:39:48.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110105 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117687 12.a e343/331 12:50ee7d02:::benchmark_data_smithi064_442287_object29637:head v 343'7458, mlcod=316'1040)", 2023-06-10T21:39:48.195 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.721978407000002, 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32658561000000003, 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233097+0000", 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233111+0000", 2023-06-10T21:39:48.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4912000000000001e-05 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233134+0000", 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2920000000000001e-05 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233149+0000", 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4758e-05 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.235997+0000", 2023-06-10T21:39:48.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028478689999999998 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.236030+0000", 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3021000000000001e-05 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.559594+0000", 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32356415199999999 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.559684+0000", 2023-06-10T21:39:48.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9957999999999995e-05 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117694 12.16 12:6ae122fd:::benchmark_data_smithi064_442287_object29644:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:39:48.203 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 43.281980765, 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.56191629600000004, 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:39:48.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117694 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:39:48.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673094+0000", 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673106+0000", 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2442e-05 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673128+0000", 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2512000000000001e-05 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:39:48.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673142+0000", 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3696000000000001e-05 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673202+0000", 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9373999999999998e-05 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673218+0000", 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6918999999999998e-05 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.217815+0000", 2023-06-10T21:39:48.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.54459649300000001 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.217922+0000", 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010657100000000001 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.218094+0000", 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017210200000000001 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.220622+0000", 2023-06-10T21:39:48.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025278940000000001 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.220645+0000", 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3152000000000001e-05 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.234923+0000", 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014277817999999999 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.234971+0000", 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8021e-05 2023-06-10T21:39:48.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.235012+0000", 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1964e-05 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:39:48.216 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:39:48.216 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:39:48.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:48 smithi064 ceph-mon[180297]: pgmap v1304: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 472 op/s; 14637/126249 objects degraded (11.594%); 29037/126249 objects misplaced (23.000%); 15 MiB/s, 232 objects/s recovering 2023-06-10T21:39:48.323 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:48 smithi064 ceph-mon[186706]: pgmap v1304: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 472 op/s; 14637/126249 objects degraded (11.594%); 29037/126249 objects misplaced (23.000%); 15 MiB/s, 232 objects/s recovering 2023-06-10T21:39:48.336 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:48 smithi152 ceph-mon[156819]: pgmap v1304: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.6 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 472 op/s; 14637/126249 objects degraded (11.594%); 29037/126249 objects misplaced (23.000%); 15 MiB/s, 232 objects/s recovering 2023-06-10T21:39:48.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:48 smithi064 conmon[180272]: 2023-06-10T21:39:48.317+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e351 definitely_dead 0 2023-06-10T21:39:49.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:39:49.236 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.1. 2023-06-10T21:39:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4227113149' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:49 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:49 smithi152 ceph-mon[156819]: Client client.admin marked osd.1 out, while it was still marked up 2023-06-10T21:39:49.469 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:49.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:49.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4227113149' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.472 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[186706]: Client client.admin marked osd.1 out, while it was still marked up 2023-06-10T21:39:49.472 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:39:49.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:39:49.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4227113149' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-10T21:39:49.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:49 smithi064 ceph-mon[180297]: Client client.admin marked osd.1 out, while it was still marked up 2023-06-10T21:39:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:49.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:49.575 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:49.601 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:49.643 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:49.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:49.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:49.782 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-10T21:39:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: osdmap e352: 8 total, 8 up, 4 in 2023-06-10T21:39:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[186706]: pgmap v1306: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.5 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 505 op/s; 14316/121995 objects degraded (11.735%); 28091/121995 objects misplaced (23.026%); 14 MiB/s, 222 objects/s recovering 2023-06-10T21:39:50.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-10T21:39:50.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: osdmap e352: 8 total, 8 up, 4 in 2023-06-10T21:39:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:50 smithi064 ceph-mon[180297]: pgmap v1306: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.5 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 505 op/s; 14316/121995 objects degraded (11.735%); 28091/121995 objects misplaced (23.026%); 14 MiB/s, 222 objects/s recovering 2023-06-10T21:39:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-10T21:39:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: osdmap e352: 8 total, 8 up, 4 in 2023-06-10T21:39:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: failed to encode map e352 with expected crc 2023-06-10T21:39:50.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:50 smithi152 ceph-mon[156819]: pgmap v1306: 99 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 79 active+clean; 2.5 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 505 op/s; 14316/121995 objects degraded (11.735%); 28091/121995 objects misplaced (23.026%); 14 MiB/s, 222 objects/s recovering 2023-06-10T21:39:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: osdmap e353: 8 total, 8 up, 4 in 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.12 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7983 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.19 continuing backfill to osd.5 from (316'1686,351'7958] MIN to 351'7958 2023-06-10T21:39:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.9 continuing backfill to osd.5 from (316'1686,351'8001] MIN to 351'8001 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.19 starting backfill to osd.7 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.9 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8001 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.4 continuing backfill to osd.2 from (319'2329,351'7958] MIN to 351'7958 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.14 continuing backfill to osd.2 from (319'2329,351'7939] MIN to 351'7939 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.4 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7939 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.13 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8023 2023-06-10T21:39:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.3 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8060 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.e starting backfill to osd.2 from (0'0,0'0] MAX to 351'8535 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.12 continuing backfill to osd.5 from (319'2337,351'7983] MIN to 351'7983 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.13 continuing backfill to osd.5 from (317'2085,351'8023] MIN to 351'8023 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.3 continuing backfill to osd.5 from (317'2085,351'8060] MIN to 351'8060 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.e continuing backfill to osd.5 from (322'4555,351'8535] MIN to 351'8535 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.2 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7992 2023-06-10T21:39:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.2 continuing backfill to osd.5 from (319'2337,351'7992] MIN to 351'7992 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.6 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7820 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.16 continuing backfill to osd.5 from (316'1901,351'7802] MIN to 351'7802 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: 12.6 continuing backfill to osd.5 from (316'1901,351'7820] MIN to 351'7820 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14316/121995 objects degraded (11.735%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:51.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: osdmap e353: 8 total, 8 up, 4 in 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.12 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7983 2023-06-10T21:39:51.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.19 continuing backfill to osd.5 from (316'1686,351'7958] MIN to 351'7958 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.9 continuing backfill to osd.5 from (316'1686,351'8001] MIN to 351'8001 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.19 starting backfill to osd.7 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.9 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8001 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.4 continuing backfill to osd.2 from (319'2329,351'7958] MIN to 351'7958 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.14 continuing backfill to osd.2 from (319'2329,351'7939] MIN to 351'7939 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.4 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7939 2023-06-10T21:39:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.13 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8023 2023-06-10T21:39:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.3 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8060 2023-06-10T21:39:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.e starting backfill to osd.2 from (0'0,0'0] MAX to 351'8535 2023-06-10T21:39:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.12 continuing backfill to osd.5 from (319'2337,351'7983] MIN to 351'7983 2023-06-10T21:39:51.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.13 continuing backfill to osd.5 from (317'2085,351'8023] MIN to 351'8023 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.3 continuing backfill to osd.5 from (317'2085,351'8060] MIN to 351'8060 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.e continuing backfill to osd.5 from (322'4555,351'8535] MIN to 351'8535 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.2 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7992 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.2 continuing backfill to osd.5 from (319'2337,351'7992] MIN to 351'7992 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.6 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7820 2023-06-10T21:39:51.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:51.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.16 continuing backfill to osd.5 from (316'1901,351'7802] MIN to 351'7802 2023-06-10T21:39:51.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: 12.6 continuing backfill to osd.5 from (316'1901,351'7820] MIN to 351'7820 2023-06-10T21:39:51.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14316/121995 objects degraded (11.735%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: osdmap e353: 8 total, 8 up, 4 in 2023-06-10T21:39:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: failed to encode map e353 with expected crc 2023-06-10T21:39:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.12 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7983 2023-06-10T21:39:51.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.19 continuing backfill to osd.5 from (316'1686,351'7958] MIN to 351'7958 2023-06-10T21:39:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.9 continuing backfill to osd.5 from (316'1686,351'8001] MIN to 351'8001 2023-06-10T21:39:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.19 starting backfill to osd.7 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.9 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8001 2023-06-10T21:39:51.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.4 continuing backfill to osd.2 from (319'2329,351'7958] MIN to 351'7958 2023-06-10T21:39:51.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.14 continuing backfill to osd.2 from (319'2329,351'7939] MIN to 351'7939 2023-06-10T21:39:51.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.4 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7958 2023-06-10T21:39:51.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.14 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7939 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.13 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8023 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.3 starting backfill to osd.2 from (0'0,0'0] MAX to 351'8060 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.e starting backfill to osd.2 from (0'0,0'0] MAX to 351'8535 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.12 continuing backfill to osd.5 from (319'2337,351'7983] MIN to 351'7983 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.13 continuing backfill to osd.5 from (317'2085,351'8023] MIN to 351'8023 2023-06-10T21:39:51.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.3 continuing backfill to osd.5 from (317'2085,351'8060] MIN to 351'8060 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.e continuing backfill to osd.5 from (322'4555,351'8535] MIN to 351'8535 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.2 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7992 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.2 continuing backfill to osd.5 from (319'2337,351'7992] MIN to 351'7992 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.6 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7820 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.16 starting backfill to osd.2 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.16 continuing backfill to osd.5 from (316'1901,351'7802] MIN to 351'7802 2023-06-10T21:39:51.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: 12.6 continuing backfill to osd.5 from (316'1901,351'7820] MIN to 351'7820 2023-06-10T21:39:51.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14316/121995 objects degraded (11.735%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-10T21:39:52.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:39:52.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:52.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:52.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:52.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:52.409 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:52.462 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:52.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:52.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.b starting backfill to osd.5 from (0'0,0'0] MAX to 351'8514 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: osdmap e354: 8 total, 8 up, 4 in 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.1 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7790 2023-06-10T21:39:52.563 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.17 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.15 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8036 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 351'8390 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7807 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.5 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8007 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.11 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7781 2023-06-10T21:39:52.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.a starting backfill to osd.5 from (0'0,0'0] MAX to 351'8604 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: 12.a starting backfill to osd.7 from (321'4300,351'8603] MIN to 351'8604 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[180297]: pgmap v1309: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 397 op/s; 8873/119556 objects degraded (7.422%); 22243/119556 objects misplaced (18.605%); 15 MiB/s, 240 objects/s recovering 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.565 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.b starting backfill to osd.5 from (0'0,0'0] MAX to 351'8514 2023-06-10T21:39:52.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: osdmap e354: 8 total, 8 up, 4 in 2023-06-10T21:39:52.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.1 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7790 2023-06-10T21:39:52.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.17 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:52.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.15 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8036 2023-06-10T21:39:52.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 351'8390 2023-06-10T21:39:52.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7807 2023-06-10T21:39:52.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.5 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8007 2023-06-10T21:39:52.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.11 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7781 2023-06-10T21:39:52.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.a starting backfill to osd.5 from (0'0,0'0] MAX to 351'8604 2023-06-10T21:39:52.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: 12.a starting backfill to osd.7 from (321'4300,351'8603] MIN to 351'8604 2023-06-10T21:39:52.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:52 smithi064 ceph-mon[186706]: pgmap v1309: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 397 op/s; 8873/119556 objects degraded (7.422%); 22243/119556 objects misplaced (18.605%); 15 MiB/s, 240 objects/s recovering 2023-06-10T21:39:52.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:52.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.b starting backfill to osd.5 from (0'0,0'0] MAX to 351'8514 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: osdmap e354: 8 total, 8 up, 4 in 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.1 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7790 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.17 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7802 2023-06-10T21:39:52.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.15 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8036 2023-06-10T21:39:52.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.d starting backfill to osd.3 from (0'0,0'0] MAX to 351'8390 2023-06-10T21:39:52.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.7 starting backfill to osd.5 from (0'0,0'0] MAX to 351'7807 2023-06-10T21:39:52.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.5 starting backfill to osd.7 from (0'0,0'0] MAX to 351'8007 2023-06-10T21:39:52.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.11 starting backfill to osd.2 from (319'2066,334'6709] MIN to 351'7781 2023-06-10T21:39:52.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.a starting backfill to osd.5 from (0'0,0'0] MAX to 351'8604 2023-06-10T21:39:52.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: 12.a starting backfill to osd.7 from (321'4300,351'8603] MIN to 351'8604 2023-06-10T21:39:52.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: failed to encode map e354 with expected crc 2023-06-10T21:39:52.601 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:52 smithi152 ceph-mon[156819]: pgmap v1309: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 397 op/s; 8873/119556 objects degraded (7.422%); 22243/119556 objects misplaced (18.605%); 15 MiB/s, 240 objects/s recovering 2023-06-10T21:39:52.601 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:52.657 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:52.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:52.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:52.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:52.824 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:52.833 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:52.966 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:52.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:52.997 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:39:53.032 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:39:53.065 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:39:53.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:53.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:39:53.149 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:39:53.170 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:39:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:39:52] "GET /metrics HTTP/1.1" 200 224281 "" "Prometheus/2.33.4" 2023-06-10T21:39:53.187 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:39:53.192 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:53.221 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:39:53.253 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:39:53.271 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.271 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.271 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.271 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.271 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.272 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:39:53.338 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:53.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:53.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:53.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2324625802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:53.526 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2324625802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:53.529 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2324625802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:39:54.220 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:54.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:54.330 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:39:54.361 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:54.422 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:39:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:54.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:39:54.530 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:54.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:54 smithi152 ceph-mon[156819]: pgmap v1310: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 401 op/s; 9918/119022 objects degraded (8.333%); 22304/119022 objects misplaced (18.739%); 14 MiB/s, 221 objects/s recovering 2023-06-10T21:39:54.561 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:54.618 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:54.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:54.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:54 smithi064 ceph-mon[180297]: pgmap v1310: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 401 op/s; 9918/119022 objects degraded (8.333%); 22304/119022 objects misplaced (18.739%); 14 MiB/s, 221 objects/s recovering 2023-06-10T21:39:54.666 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:54 smithi064 ceph-mon[186706]: pgmap v1310: 99 pgs: 3 remapped+peering, 16 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 63 active+clean; 2.4 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 401 op/s; 9918/119022 objects degraded (8.333%); 22304/119022 objects misplaced (18.739%); 14 MiB/s, 221 objects/s recovering 2023-06-10T21:39:54.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:54.755 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:54.811 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:54.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:54.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:39:54.952 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:55.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:39:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:55.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:39:55.155 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:55.219 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:55.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:55.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:55.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:55.411 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:55.479 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:56 smithi064 ceph-mon[186706]: pgmap v1311: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 14313/114924 objects degraded (12.454%); 39502/114924 objects misplaced (34.372%); 18 MiB/s, 1 keys/s, 304 objects/s recovering 2023-06-10T21:39:56.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9918/119022 objects degraded (8.333%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:39:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:56 smithi064 ceph-mon[180297]: pgmap v1311: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 14313/114924 objects degraded (12.454%); 39502/114924 objects misplaced (34.372%); 18 MiB/s, 1 keys/s, 304 objects/s recovering 2023-06-10T21:39:56.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9918/119022 objects degraded (8.333%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:39:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:56 smithi152 ceph-mon[156819]: pgmap v1311: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 343 op/s; 14313/114924 objects degraded (12.454%); 39502/114924 objects misplaced (34.372%); 18 MiB/s, 1 keys/s, 304 objects/s recovering 2023-06-10T21:39:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9918/119022 objects degraded (8.333%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:39:56.519 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:56.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:56.545 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:39:56.632 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:56.721 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:57.085 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:57.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:57.208 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:39:57.269 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:39:57.269 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:39:57.270 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-06-10T21:39:57.270 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:39:57.270 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:39:57.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-06-10T21:39:57.291 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:57.662 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:39:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:57.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:39:57.719 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:39:57.773 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:57.814 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:57.866 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:57.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:57.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:57.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:58.013 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:39:58.024 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:58.071 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:58.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:39:58.073 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:58 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:58.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:58.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:39:58.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:58.250 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:39:58.251 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:39:58.252 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:58 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:58.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:39:58.252 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:58 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:39:58.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:58.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:39:58.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:58.385 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:58.438 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:58.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:58.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:39:58.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:58.580 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:58.634 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:58.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:58.739 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:39:58.774 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:58.836 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:39:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:39:58.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:39:58.970 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:59.035 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:39:59.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:59.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:39:59.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:39:59.178 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:39:59.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:59 smithi152 ceph-mon[156819]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:59 smithi152 ceph-mon[156819]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:39:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:59 smithi152 ceph-mon[156819]: osdmap e355: 8 total, 8 up, 4 in 2023-06-10T21:39:59.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:39:59 smithi152 ceph-mon[156819]: pgmap v1313: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 219 op/s; 14313/114390 objects degraded (12.512%); 39324/114390 objects misplaced (34.377%); 8.6 MiB/s, 1 keys/s, 151 objects/s recovering 2023-06-10T21:39:59.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[180297]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[180297]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:39:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[180297]: osdmap e355: 8 total, 8 up, 4 in 2023-06-10T21:39:59.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[180297]: pgmap v1313: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 219 op/s; 14313/114390 objects degraded (12.512%); 39324/114390 objects misplaced (34.377%); 8.6 MiB/s, 1 keys/s, 151 objects/s recovering 2023-06-10T21:39:59.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[186706]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[186706]: failed to encode map e355 with expected crc 2023-06-10T21:39:59.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3917279333' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:39:59.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[186706]: osdmap e355: 8 total, 8 up, 4 in 2023-06-10T21:39:59.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:39:59 smithi064 ceph-mon[186706]: pgmap v1313: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.3 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 219 op/s; 14313/114390 objects degraded (12.512%); 39324/114390 objects misplaced (34.377%); 8.6 MiB/s, 1 keys/s, 151 objects/s recovering 2023-06-10T21:39:59.840 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:39:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:39:59.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:39:59.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:00.059 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-06-10T21:40:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: [WRN] PG_DEGRADED: Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-06-10T21:40:00.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.2 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.9 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.a is active+recovery_wait+degraded+remapped, acting [2,0,1] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.b is active+recovery_wait+degraded+remapped, acting [3,2,1] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.e is active+recovery_wait+degraded+remapped, acting [7,0,3] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.f is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.10 is active+recovery_wait+degraded+remapped, acting [2,7,0] 2023-06-10T21:40:00.159 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.12 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[180297]: pg 12.19 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-06-10T21:40:00.160 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: [WRN] PG_DEGRADED: Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.2 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.9 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.a is active+recovery_wait+degraded+remapped, acting [2,0,1] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.b is active+recovery_wait+degraded+remapped, acting [3,2,1] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.e is active+recovery_wait+degraded+remapped, acting [7,0,3] 2023-06-10T21:40:00.161 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.f is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-06-10T21:40:00.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.10 is active+recovery_wait+degraded+remapped, acting [2,7,0] 2023-06-10T21:40:00.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.12 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.162 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:00 smithi064 ceph-mon[186706]: pg 12.19 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.233 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:00.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:00.349 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:00.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:00.419 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: [WRN] PG_DEGRADED: Degraded data redundancy: 14313/114390 objects degraded (12.512%), 10 pgs degraded 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,2] 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.2 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.9 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.a is active+recovery_wait+degraded+remapped, acting [2,0,1] 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.b is active+recovery_wait+degraded+remapped, acting [3,2,1] 2023-06-10T21:40:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.e is active+recovery_wait+degraded+remapped, acting [7,0,3] 2023-06-10T21:40:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.f is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-06-10T21:40:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.10 is active+recovery_wait+degraded+remapped, acting [2,7,0] 2023-06-10T21:40:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.12 is active+recovery_wait+degraded+remapped, acting [7,1,3] 2023-06-10T21:40:00.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:00 smithi152 ceph-mon[156819]: pg 12.19 is active+recovery_wait+degraded+remapped, acting [2,1,0] 2023-06-10T21:40:00.445 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:00.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:00.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:00.538 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:40:00.538 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:40:00.549 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:00.580 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:00.865 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:00.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:00.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:01.055 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:01.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:01.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:01.298 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:01.363 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:01 smithi064 ceph-mon[186706]: pgmap v1314: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 357 op/s; 14019/110049 objects degraded (12.739%); 37532/110049 objects misplaced (34.105%); 9.7 MiB/s, 1 keys/s, 167 objects/s recovering 2023-06-10T21:40:01.365 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:01 smithi064 ceph-mon[180297]: pgmap v1314: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 357 op/s; 14019/110049 objects degraded (12.739%); 37532/110049 objects misplaced (34.105%); 9.7 MiB/s, 1 keys/s, 167 objects/s recovering 2023-06-10T21:40:01.369 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:01.399 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:01 smithi152 ceph-mon[156819]: pgmap v1314: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 357 op/s; 14019/110049 objects degraded (12.739%); 37532/110049 objects misplaced (34.105%); 9.7 MiB/s, 1 keys/s, 167 objects/s recovering 2023-06-10T21:40:01.481 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:01.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:01.595 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:01.653 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:01.679 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:40:01.679 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:40:01.679 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:40:01.725 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:01.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:01.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:01.923 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:01.940 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:40:01.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:40:02.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:02 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 14019/110049 objects degraded (12.739%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:02.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:02 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 14019/110049 objects degraded (12.739%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:02.237 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:02.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:02.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:02.396 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:02.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:02 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 14019/110049 objects degraded (12.739%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:02.913 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:02.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:03.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:03.062 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:03.116 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:03.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:03.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:03.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:03 smithi064 ceph-mon[186706]: pgmap v1315: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 13466/107280 objects degraded (12.552%); 36764/107280 objects misplaced (34.269%); 12 MiB/s, 0 keys/s, 234 objects/s recovering 2023-06-10T21:40:03.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:02] "GET /metrics HTTP/1.1" 200 224281 "" "Prometheus/2.33.4" 2023-06-10T21:40:03.160 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:03 smithi064 ceph-mon[180297]: pgmap v1315: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 13466/107280 objects degraded (12.552%); 36764/107280 objects misplaced (34.269%); 12 MiB/s, 0 keys/s, 234 objects/s recovering 2023-06-10T21:40:03.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:03.244 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:03.254 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:03.268 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:03.310 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:03.316 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:03 smithi152 ceph-mon[156819]: pgmap v1315: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 13466/107280 objects degraded (12.552%); 36764/107280 objects misplaced (34.269%); 12 MiB/s, 0 keys/s, 234 objects/s recovering 2023-06-10T21:40:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:03.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:03.418 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:03.521 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:03.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:40:03.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:03.849 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:40:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:03.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:03.889 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:40:03.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:03.954 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:40:04.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:40:04.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:40:04.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:04.084 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:40:04.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:40:04.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.154 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.155 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:40:04.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:04.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3254700337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:04.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:04.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3254700337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:04.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:04.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3254700337' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:05.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:05 smithi064 ceph-mon[180297]: pgmap v1316: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 423 op/s; 13466/106296 objects degraded (12.668%); 36441/106296 objects misplaced (34.283%); 11 MiB/s, 0 keys/s, 222 objects/s recovering 2023-06-10T21:40:05.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:05 smithi064 ceph-mon[186706]: pgmap v1316: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 423 op/s; 13466/106296 objects degraded (12.668%); 36441/106296 objects misplaced (34.283%); 11 MiB/s, 0 keys/s, 222 objects/s recovering 2023-06-10T21:40:05.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:05 smithi152 ceph-mon[156819]: pgmap v1316: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 9 active+recovery_wait+degraded+remapped, 72 active+clean; 2.2 GiB data, 7.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 423 op/s; 13466/106296 objects degraded (12.668%); 36441/106296 objects misplaced (34.283%); 11 MiB/s, 0 keys/s, 222 objects/s recovering 2023-06-10T21:40:05.797 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:05.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:05.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:05.948 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:06.017 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:06.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:06.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:06.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 13466/106296 objects degraded (12.668%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 13466/106296 objects degraded (12.668%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 13466/106296 objects degraded (12.668%), 10 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:06.778 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:06.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:06.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:06.893 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:06.927 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:06.989 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:06.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:07.096 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:07.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:07 smithi152 ceph-mon[156819]: pgmap v1317: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 11591/99867 objects degraded (11.606%); 35977/99867 objects misplaced (36.025%); 10 MiB/s, 223 objects/s recovering 2023-06-10T21:40:07.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:07.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:07 smithi064 ceph-mon[180297]: pgmap v1317: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 11591/99867 objects degraded (11.606%); 35977/99867 objects misplaced (36.025%); 10 MiB/s, 223 objects/s recovering 2023-06-10T21:40:07.407 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:07.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:07 smithi064 ceph-mon[186706]: pgmap v1317: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 11591/99867 objects degraded (11.606%); 35977/99867 objects misplaced (36.025%); 10 MiB/s, 223 objects/s recovering 2023-06-10T21:40:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:07.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:07.433 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:40:07.434 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:40:07.434 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:40:07.434 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:40:07.434 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:40:07.518 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:07.554 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:07.575 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:40:07.575 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:40:07.576 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-06-10T21:40:07.576 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:40:07.576 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:40:07.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-06-10T21:40:07.620 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:07.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:07.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:07.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:07.768 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:07.831 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:07.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:07.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:07.970 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:08.027 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:08.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:08.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:08.166 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:08.227 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:08.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:08.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:08.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:08.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:40:08.934 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:08.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:08.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:40:09.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:09.071 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:09.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:09 smithi064 ceph-mon[180297]: pgmap v1318: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 495 op/s; 11591/99399 objects degraded (11.661%); 35821/99399 objects misplaced (36.038%); 10 MiB/s, 222 objects/s recovering 2023-06-10T21:40:09.161 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:09.183 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:09.185 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:40:09.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:09 smithi152 ceph-mon[156819]: pgmap v1318: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 495 op/s; 11591/99399 objects degraded (11.661%); 35821/99399 objects misplaced (36.038%); 10 MiB/s, 222 objects/s recovering 2023-06-10T21:40:09.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:09.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:09 smithi064 ceph-mon[186706]: pgmap v1318: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 2.0 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 495 op/s; 11591/99399 objects degraded (11.661%); 35821/99399 objects misplaced (36.038%); 10 MiB/s, 222 objects/s recovering 2023-06-10T21:40:09.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:09.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:09.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:09.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:09.710 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:09.769 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:09.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:09.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:09.880 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:09.987 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:40:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: osdmap e356: 8 total, 8 up, 4 in 2023-06-10T21:40:10.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[186706]: pgmap v1320: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 513 op/s; 11302/94623 objects degraded (11.944%); 33811/94623 objects misplaced (35.732%); 11 MiB/s, 230 objects/s recovering 2023-06-10T21:40:10.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: osdmap e356: 8 total, 8 up, 4 in 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:10 smithi064 ceph-mon[180297]: pgmap v1320: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 513 op/s; 11302/94623 objects degraded (11.944%); 33811/94623 objects misplaced (35.732%); 11 MiB/s, 230 objects/s recovering 2023-06-10T21:40:10.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:40:10.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1600718612' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:10.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: osdmap e356: 8 total, 8 up, 4 in 2023-06-10T21:40:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: failed to encode map e356 with expected crc 2023-06-10T21:40:10.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:10 smithi152 ceph-mon[156819]: pgmap v1320: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 513 op/s; 11302/94623 objects degraded (11.944%); 33811/94623 objects misplaced (35.732%); 11 MiB/s, 230 objects/s recovering 2023-06-10T21:40:10.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:10.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:10.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:10.675 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:11.064 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:11.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:11.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:11.196 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:11.239 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:11.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:11.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:11.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:11.377 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:11.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 11302/94623 objects degraded (11.944%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:11.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 11302/94623 objects degraded (11.944%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:11.439 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:11.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 11302/94623 objects degraded (11.944%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:11.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:11.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:11.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:11.587 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:11.648 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:11.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:11.722 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:11.759 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:11.796 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:11.805 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:11.856 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:11.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:11.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:12.048 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:12.190 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:12 smithi152 ceph-mon[156819]: pgmap v1321: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 519 op/s; 10774/91695 objects degraded (11.750%); 32896/91695 objects misplaced (35.875%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:12.235 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:12.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:12.345 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:12.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:12.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:12 smithi064 ceph-mon[186706]: pgmap v1321: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 519 op/s; 10774/91695 objects degraded (11.750%); 32896/91695 objects misplaced (35.875%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:12.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:12 smithi064 ceph-mon[180297]: pgmap v1321: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.9 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 519 op/s; 10774/91695 objects degraded (11.750%); 32896/91695 objects misplaced (35.875%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:12.436 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:12.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:12.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:12.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:40:12.545 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:12.625 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:12.782 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:12.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:12.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:12.924 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:12.972 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:12.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:12.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:12.984 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:40:12.984 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:40:12.984 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:40:13.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:13.130 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:12] "GET /metrics HTTP/1.1" 200 224259 "" "Prometheus/2.33.4" 2023-06-10T21:40:13.134 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:14 smithi152 ceph-mon[156819]: pgmap v1322: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 10774/90597 objects degraded (11.892%); 32531/90597 objects misplaced (35.907%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:14.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:14 smithi064 ceph-mon[186706]: pgmap v1322: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 10774/90597 objects degraded (11.892%); 32531/90597 objects misplaced (35.907%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:14.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:14 smithi064 ceph-mon[180297]: pgmap v1322: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 10774/90597 objects degraded (11.892%); 32531/90597 objects misplaced (35.907%); 11 MiB/s, 197 objects/s recovering 2023-06-10T21:40:14.952 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:14.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:14.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:15.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:15.100 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:15.154 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:15.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:15.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:15.351 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:15.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:15.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:15.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:15.951 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:15.982 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:16.039 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:16.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:16.144 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:16.226 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:16.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:16 smithi064 ceph-mon[180297]: pgmap v1323: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 9935/83379 objects degraded (11.915%); 29737/83379 objects misplaced (35.665%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:16.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 10774/90597 objects degraded (11.892%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:16.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:16 smithi064 ceph-mon[186706]: pgmap v1323: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 9935/83379 objects degraded (11.915%); 29737/83379 objects misplaced (35.665%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:16.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 10774/90597 objects degraded (11.892%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:16 smithi152 ceph-mon[156819]: pgmap v1323: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 9935/83379 objects degraded (11.915%); 29737/83379 objects misplaced (35.665%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:16.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 10774/90597 objects degraded (11.892%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:16.482 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:16.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:16.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:16.507 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:16.536 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:40:16.536 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:40:16.536 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99515 11.1 11:8ea0c8ed:::benchmark_data_smithi064_385725_object15242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 414.31801940700001, 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30862035399999999, 2023-06-10T21:40:16.537 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99515 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.538 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.539 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203463+0000", 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203461+0000", 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:40:16.540 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203469+0000", 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2830000000000004e-06 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.541 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203482+0000", 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3318e-05 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203491+0000", 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7749999999999998e-06 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.542 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203533+0000", 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2716999999999998e-05 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.543 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203662+0000", 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012818000000000001 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.203771+0000", 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109725 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.544 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.545 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.545 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.213580+0000", 2023-06-10T21:40:16.545 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098082800000000008 2023-06-10T21:40:16.545 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.545 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.218941+0000", 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053615360000000001 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.546 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.511973+0000", 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29303174399999998 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512034+0000", 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1147000000000003e-05 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.547 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.512083+0000", 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9351000000000002e-05 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.548 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99523 11.1 11:88f160b1:::benchmark_data_smithi064_385725_object15250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 414.30623122600002, 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.311404917, 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.549 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99523 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.550 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215251+0000", 2023-06-10T21:40:16.551 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215250+0000", 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.552 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215255+0000", 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0019999999999997e-06 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215263+0000", 2023-06-10T21:40:16.553 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1470000000000001e-06 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215270+0000", 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4099999999999996e-06 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.554 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215321+0000", 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1413999999999999e-05 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215475+0000", 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000153478 2023-06-10T21:40:16.555 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.215610+0000", 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135716 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.556 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.219102+0000", 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034921179999999998 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.220787+0000", 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001685099 2023-06-10T21:40:16.557 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526574+0000", 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30578659800000002 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.558 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526618+0000", 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4140000000000001e-05 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526656+0000", 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8003999999999997e-05 2023-06-10T21:40:16.559 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:99530 11.1 11:816041a6:::benchmark_data_smithi064_385725_object15257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:40:16.560 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 414.295759428, 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30104990300000001, 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99530 2023-06-10T21:40:16.561 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.562 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225723+0000", 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225722+0000", 2023-06-10T21:40:16.563 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225728+0000", 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9259999999999997e-06 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.564 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225741+0000", 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2894999999999999e-05 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.565 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225749+0000", 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9209999999999992e-06 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225790+0000", 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1196999999999999e-05 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.566 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.225895+0000", 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010484699999999999 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.226005+0000", 2023-06-10T21:40:16.567 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110655 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229259+0000", 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032538469999999998 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.568 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.229476+0000", 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021646299999999999 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.569 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526694+0000", 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29721868499999998 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526739+0000", 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4938999999999998e-05 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.570 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:22.526773+0000", 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3716e-05 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.571 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:101027 11.1 11:84832f10:::benchmark_data_smithi064_385725_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected e296)", 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 411.274565516, 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29481494899999999, 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.572 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 101027 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.573 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.574 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246917+0000", 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.575 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246919+0000", 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.565e-06 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246925+0000", 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7769999999999999e-06 2023-06-10T21:40:16.576 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.246930+0000", 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6949999999999999e-06 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.577 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252174+0000", 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052442779999999998 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252278+0000", 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010410499999999999 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.578 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.252344+0000", 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6168999999999996e-05 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.256536+0000", 2023-06-10T21:40:16.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041921479999999997 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.531254+0000", 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27471757699999999 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541651+0000", 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010397027 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541697+0000", 2023-06-10T21:40:16.581 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5889000000000001e-05 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:33:25.541732+0000", 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5098999999999997e-05 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.582 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126745 11.1 11:8766de5a:::benchmark_data_smithi064_385725_object42472:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 362.09789643300002, 2023-06-10T21:40:16.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33496997699999997, 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126745 2023-06-10T21:40:16.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.585 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423586+0000", 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423584+0000", 2023-06-10T21:40:16.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423595+0000", 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1586000000000001e-05 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423618+0000", 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2609999999999999e-05 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423630+0000", 2023-06-10T21:40:16.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2449000000000001e-05 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423686+0000", 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5452999999999999e-05 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423832+0000", 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146155 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423960+0000", 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012785100000000001 2023-06-10T21:40:16.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.434684+0000", 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01072394 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.591 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625142+0000", 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19045854700000001 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758437+0000", 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13329444800000001 2023-06-10T21:40:16.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758501+0000", 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3959000000000002e-05 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758556+0000", 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5461000000000003e-05 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126747 11.1 11:8706db2c:::benchmark_data_smithi064_385725_object42474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 362.09783739900001, 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33505084800000001, 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.595 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126747 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423645+0000", 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423648+0000", 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1719999999999999e-06 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423654+0000", 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8340000000000004e-06 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423660+0000", 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7059999999999998e-06 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437557+0000", 2023-06-10T21:40:16.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013896382000000001 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437673+0000", 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116501 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437787+0000", 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113354 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625022+0000", 2023-06-10T21:40:16.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.18723535299999999 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625168+0000", 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000146494 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.603 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758602+0000", 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13343400399999999 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758650+0000", 2023-06-10T21:40:16.604 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7052000000000003e-05 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758696+0000", 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6424000000000001e-05 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.605 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126748 11.8 11:1c2ac95f:::benchmark_data_smithi064_385725_object42475:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 362.097799431, 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32430128899999999, 2023-06-10T21:40:16.606 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126748 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.607 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423683+0000", 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423682+0000", 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:40:16.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423688+0000", 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6160000000000001e-06 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423699+0000", 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1623e-05 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423707+0000", 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8830000000000006e-06 2023-06-10T21:40:16.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423759+0000", 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1573000000000002e-05 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423869+0000", 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109951 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.423973+0000", 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000104144 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.437147+0000", 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013174289 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.624707+0000", 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.187559904 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747900+0000", 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12319322200000001 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747947+0000", 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6390000000000001e-05 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.747984+0000", 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7703999999999997e-05 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126751 11.1 11:8ace7e90:::benchmark_data_smithi064_385725_object42478:head [delete] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-06-10T21:40:16.617 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 362.082356715, 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31970830500000003, 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.618 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126751 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.619 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439126+0000", 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.620 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439125+0000", 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439130+0000", 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5249999999999996e-06 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439144+0000", 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3719e-05 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439152+0000", 2023-06-10T21:40:16.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2449999999999998e-06 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625233+0000", 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.186081301 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.623 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625322+0000", 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8226999999999994e-05 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.625417+0000", 2023-06-10T21:40:16.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5429999999999999e-05 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631580+0000", 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061630039999999997 2023-06-10T21:40:16.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631643+0000", 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3408e-05 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758744+0000", 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.127100451 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758794+0000", 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0526999999999999e-05 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.758834+0000", 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9748e-05 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.36159.0:126756 11.1 11:81432d08:::benchmark_data_smithi064_385725_object42483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e300)", 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 362.08231880099999, 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.328752612, 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.36159", 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/1749142030", 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 126756 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439164+0000", 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439163+0000", 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439166+0000", 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1990000000000002e-06 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439174+0000", 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3300000000000001e-06 2023-06-10T21:40:16.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.439180+0000", 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8000000000000004e-06 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627884+0000", 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.188704802 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.627970+0000", 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5963000000000003e-05 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2", 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.628045+0000", 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4239000000000005e-05 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631687+0000", 2023-06-10T21:40:16.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036425799999999999 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.631697+0000", 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0166999999999999e-05 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767840+0000", 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.13614217200000001 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767882+0000", 2023-06-10T21:40:16.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2262000000000003e-05 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:34:14.767916+0000", 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4548000000000002e-05 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.639 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:93726 12.3 e328/311 12:c23b8a83:::benchmark_data_smithi064_442287_object5676:head v 328'5928, mlcod=328'5925)", 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:56.343434+0000", 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 140.17804813199999, 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30815320699999998, 2023-06-10T21:40:16.640 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343434+0000", 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343434+0000", 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343434+0000", 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343438+0000", 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.33e-06 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343446+0000", 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2250000000000003e-06 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343451+0000", 2023-06-10T21:40:16.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6049999999999997e-06 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.350457+0000", 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070057360000000003 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.350474+0000", 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7102999999999999e-05 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.651553+0000", 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.301078913 2023-06-10T21:40:16.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.651588+0000", 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4635999999999998e-05 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.647 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:93733 12.3 e328/311 12:c87c1692:::benchmark_data_smithi064_442287_object5683:head v 328'5929, mlcod=328'5925)", 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:37:56.343488+0000", 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 140.177994531, 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30812189499999998, 2023-06-10T21:40:16.648 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343488+0000", 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343488+0000", 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343488+0000", 2023-06-10T21:40:16.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343492+0000", 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.98e-06 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343497+0000", 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9359999999999996e-06 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.343502+0000", 2023-06-10T21:40:16.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3730000000000003e-06 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.549568+0000", 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20606574699999999 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.549591+0000", 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3125e-05 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.651593+0000", 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.102002331 2023-06-10T21:40:16.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:37:56.651610+0000", 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6852999999999999e-05 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111290 12.b 12:d4fdbcf7:::benchmark_data_smithi064_442287_object23240:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.199021+0000", 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 93.322461877999999, 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.295532674, 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.656 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111290 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199021+0000", 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199021+0000", 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199019+0000", 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199027+0000", 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2090000000000005e-06 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199041+0000", 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.417e-05 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199050+0000", 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2e-06 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199106+0000", 2023-06-10T21:40:16.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5065999999999998e-05 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199248+0000", 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014282300000000001 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.199389+0000", 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014062300000000001 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.206216+0000", 2023-06-10T21:40:16.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068269200000000002 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.211270+0000", 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050537510000000004 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.491417+0000", 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28014776800000002 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.491481+0000", 2023-06-10T21:40:16.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3570000000000003e-05 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.494553+0000", 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00307231 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.666 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:112448 12.b 12:d1683130:::benchmark_data_smithi064_442287_object24398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e338)", 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:46.695771+0000", 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 89.825711157000001, 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29641580299999998, 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.667 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 112448 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695771+0000", 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695771+0000", 2023-06-10T21:40:16.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695769+0000", 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695778+0000", 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5750000000000001e-06 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695795+0000", 2023-06-10T21:40:16.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7376000000000001e-05 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.695804+0000", 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0499999999999997e-06 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.701501+0000", 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005697059 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.701647+0000", 2023-06-10T21:40:16.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014558299999999999 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.701729+0000", 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1879000000000007e-05 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.705170+0000", 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034408759999999998 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.713156+0000", 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079859779999999995 2023-06-10T21:40:16.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.989192+0000", 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27603572500000001 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.989254+0000", 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1999000000000001e-05 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.992187+0000", 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029336240000000001 2023-06-10T21:40:16.677 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136398 12.b 12:d51f51df:::benchmark_data_smithi064_442287_object48348:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 27.327496754999999, 2023-06-10T21:40:16.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1895044019999998, 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136398 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.679 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193984+0000", 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:40:16.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193990+0000", 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7289999999999997e-06 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194000+0000", 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0328e-05 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194007+0000", 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5259999999999999e-06 2023-06-10T21:40:16.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194082+0000", 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5372999999999998e-05 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194250+0000", 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168278 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194419+0000", 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016897799999999999 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.200597+0000", 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061777100000000003 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.206684+0000", 2023-06-10T21:40:16.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060865779999999996 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.219819+0000", 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013135413 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.219832+0000", 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3101e-05 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.216190+0000", 2023-06-10T21:40:16.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99635775400000004 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.216208+0000", 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8501000000000001e-05 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.383417+0000", 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.16720892 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.383490+0000", 2023-06-10T21:40:16.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2949999999999998e-05 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136421 12.b 12:d4a6a5b7:::benchmark_data_smithi064_442287_object48371:head [delete] snapc 0=[] ondisk+write+known_if_redirected e353)", 2023-06-10T21:40:16.691 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:50.572596+0000", 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 25.948886583, 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89362444699999999, 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.692 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136421 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572596+0000", 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572596+0000", 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572594+0000", 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572604+0000", 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0556e-05 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572627+0000", 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3067000000000001e-05 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572638+0000", 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0234999999999999e-05 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572703+0000", 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5679000000000006e-05 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.572719+0000", 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.605e-05 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.384475+0000", 2023-06-10T21:40:16.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81175530500000004 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.384639+0000", 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000164289 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.384789+0000", 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000150209 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.389196+0000", 2023-06-10T21:40:16.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044070209999999997 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.389305+0000", 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108594 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.390433+0000", 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011288159999999999 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.464951+0000", 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074517652000000004 2023-06-10T21:40:16.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.465020+0000", 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9139000000000003e-05 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.466220+0000", 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001200104 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.704 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136642 12.b 12:d6d9984d:::benchmark_data_smithi064_442287_object48592:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.847153+0000", 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.674329190999998, 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.54441097299999996, 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.705 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136642 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847153+0000", 2023-06-10T21:40:16.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847153+0000", 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847149+0000", 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999962 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847166+0000", 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6410999999999999e-05 2023-06-10T21:40:16.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847200+0000", 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4343e-05 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847215+0000", 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5418999999999999e-05 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847296+0000", 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0703000000000003e-05 2023-06-10T21:40:16.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847534+0000", 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000238273 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.847733+0000", 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000198736 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.856253+0000", 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085193690000000006 2023-06-10T21:40:16.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.857372+0000", 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011189559999999999 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.869799+0000", 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012427891999999999 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.390387+0000", 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.52058727000000005 2023-06-10T21:40:16.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.390442+0000", 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5640999999999998e-05 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391564+0000", 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011219660000000001 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.716 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136653 12.b 12:d5043e89:::benchmark_data_smithi064_442287_object48603:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.904393+0000", 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.617089173, 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49691988399999998, 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.717 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136653 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904393+0000", 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904393+0000", 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904391+0000", 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904402+0000", 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0389e-05 2023-06-10T21:40:16.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904428+0000", 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6029000000000001e-05 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904438+0000", 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0825000000000001e-05 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904594+0000", 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155632 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904741+0000", 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014660399999999999 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904885+0000", 2023-06-10T21:40:16.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014465600000000001 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.917797+0000", 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012912085 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918670+0000", 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00087279700000000003 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.932798+0000", 2023-06-10T21:40:16.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014127783999999999 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.400255+0000", 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.467457187 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.400314+0000", 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8638999999999999e-05 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.401313+0000", 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00099942 2023-06-10T21:40:16.727 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136658 12.b 12:d9355ce3:::benchmark_data_smithi064_442287_object48608:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.904453+0000", 2023-06-10T21:40:16.728 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.617029131999999, 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49771093, 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136658 2023-06-10T21:40:16.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904453+0000", 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904453+0000", 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904453+0000", 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904456+0000", 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2090000000000001e-06 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904462+0000", 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.993e-06 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904467+0000", 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1320000000000002e-06 2023-06-10T21:40:16.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.908924+0000", 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044561959999999999 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.909086+0000", 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016282100000000001 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.909219+0000", 2023-06-10T21:40:16.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013282600000000001 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.917805+0000", 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085853709999999996 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918711+0000", 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00090667799999999995 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.932830+0000", 2023-06-10T21:40:16.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014118726 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.401348+0000", 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.468517723 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.401404+0000", 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6644999999999997e-05 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.402164+0000", 2023-06-10T21:40:16.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00075998899999999998 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136659 12.b 12:dcd34507:::benchmark_data_smithi064_442287_object48609:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.904472+0000", 2023-06-10T21:40:16.740 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.617010797999999, 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49840248799999998, 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.741 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136659 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904472+0000", 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904472+0000", 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904471+0000", 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904473+0000", 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.074e-06 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904479+0000", 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.057e-06 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.904483+0000", 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6020000000000002e-06 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913455+0000", 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089719650000000001 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913610+0000", 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015472199999999999 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913746+0000", 2023-06-10T21:40:16.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013577700000000001 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918680+0000", 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049340959999999998 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918740+0000", 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0652999999999997e-05 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.932859+0000", 2023-06-10T21:40:16.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014119069999999999 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.402200+0000", 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46934068699999998 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.402227+0000", 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7050999999999999e-05 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.402874+0000", 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064703000000000004 2023-06-10T21:40:16.751 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136687 12.b 12:dcf05046:::benchmark_data_smithi064_442287_object48637:head [delete] snapc 0=[] ondisk+write+known_if_redirected e354)", 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:52.121860+0000", 2023-06-10T21:40:16.752 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 24.399622206, 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29689060499999997, 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136687 2023-06-10T21:40:16.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121860+0000", 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121860+0000", 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121857+0000", 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-06-10T21:40:16.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121871+0000", 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3339999999999999e-05 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121895+0000", 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3949999999999999e-05 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121905+0000", 2023-06-10T21:40:16.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0774000000000001e-05 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.121961+0000", 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6165000000000002e-05 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122111+0000", 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014905699999999999 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122257+0000", 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014658600000000001 2023-06-10T21:40:16.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.126591+0000", 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043341289999999999 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.126713+0000", 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122144 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.127684+0000", 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00097018699999999996 2023-06-10T21:40:16.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.417642+0000", 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.28995854900000001 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:40:16.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.417680+0000", 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7700999999999997e-05 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.418751+0000", 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010710730000000001 2023-06-10T21:40:16.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.764 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.764 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.764 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:40:16.764 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:40:16.764 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:40:16.764 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:16.805 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:16.860 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:16.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:16.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:16.884 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:40:16.927 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:40:16.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:16.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:40:17.023 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:40:17.051 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:17.077 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:40:17.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:40:17.143 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:40:17.189 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:40:17.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.196 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.197 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:40:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3291410449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:17.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3291410449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:17.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3291410449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:18.195 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:18.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:18.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:18.335 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:18.397 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:18.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:18.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:18 smithi152 ceph-mon[156819]: pgmap v1324: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 9935/82869 objects degraded (11.989%); 29567/82869 objects misplaced (35.679%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:18.467 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:18 smithi064 ceph-mon[186706]: pgmap v1324: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 9935/82869 objects degraded (11.989%); 29567/82869 objects misplaced (35.679%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:18.467 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:18 smithi064 ceph-mon[180297]: pgmap v1324: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 9935/82869 objects degraded (11.989%); 29567/82869 objects misplaced (35.679%); 11 MiB/s, 182 objects/s recovering 2023-06-10T21:40:18.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:18.515 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:40:18.530 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:18.588 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:18.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:18.694 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:18.779 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:19.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:19.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:19.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:19.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:19.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:19.615 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:19.615 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:19.617 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:19.673 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:19.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:19.782 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:19.813 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:19.889 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:19.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:19.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:20.025 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:20.066 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:20.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:20.173 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:20.220 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:20.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:20.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:20 smithi064 ceph-mon[180297]: pgmap v1325: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 499 op/s; 9670/78219 objects degraded (12.363%); 27633/78219 objects misplaced (35.328%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:20.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:20.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:20 smithi064 ceph-mon[186706]: pgmap v1325: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 499 op/s; 9670/78219 objects degraded (12.363%); 27633/78219 objects misplaced (35.328%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:20.672 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:20 smithi152 ceph-mon[156819]: pgmap v1325: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 499 op/s; 9670/78219 objects degraded (12.363%); 27633/78219 objects misplaced (35.328%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:20.710 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:20.741 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:20.798 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:20.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:20.888 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:40:20.888 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:40:20.904 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:21.057 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:21.485 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:40:21.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9670/78219 objects degraded (12.363%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:21.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9670/78219 objects degraded (12.363%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9670/78219 objects degraded (12.363%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:21.931 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:21.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:21.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:22.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:22.101 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:22.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/221586804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[180297]: pgmap v1326: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 544 op/s; 9033/74991 objects degraded (12.045%); 26645/74991 objects misplaced (35.531%); 13 MiB/s, 202 objects/s recovering 2023-06-10T21:40:22.236 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:40:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/221586804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:22 smithi064 ceph-mon[186706]: pgmap v1326: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 544 op/s; 9033/74991 objects degraded (12.045%); 26645/74991 objects misplaced (35.531%); 13 MiB/s, 202 objects/s recovering 2023-06-10T21:40:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/221586804' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:40:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:22 smithi152 ceph-mon[156819]: pgmap v1326: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 544 op/s; 9033/74991 objects degraded (12.045%); 26645/74991 objects misplaced (35.531%); 13 MiB/s, 202 objects/s recovering 2023-06-10T21:40:22.887 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:22.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:22.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:22.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:23.036 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:23.098 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:23.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:23.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:23.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:22] "GET /metrics HTTP/1.1" 200 224250 "" "Prometheus/2.33.4" 2023-06-10T21:40:23.212 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:23.237 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:23.298 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:23.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:23 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:23.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:23 smithi152 ceph-mon[156819]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:23 smithi152 ceph-mon[156819]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:23 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:40:23.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:23 smithi152 ceph-mon[156819]: osdmap e357: 8 total, 8 up, 4 in 2023-06-10T21:40:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:23.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:23.406 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:23.488 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:23.489 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[186706]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.489 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[186706]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.489 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:40:23.490 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[186706]: osdmap e357: 8 total, 8 up, 4 in 2023-06-10T21:40:23.491 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:23.491 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[180297]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[180297]: failed to encode map e357 with expected crc 2023-06-10T21:40:23.492 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:40:23.493 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:23 smithi064 ceph-mon[180297]: osdmap e357: 8 total, 8 up, 4 in 2023-06-10T21:40:23.497 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:23.831 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:23.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:23.941 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:23.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:23.984 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:24.033 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:24.039 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:24.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:24.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:24.180 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:24.234 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:24.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:24.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:24.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:24.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:24.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:24 smithi064 ceph-mon[180297]: pgmap v1328: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 9033/73986 objects degraded (12.209%); 26313/73986 objects misplaced (35.565%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:40:24.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:24 smithi064 ceph-mon[186706]: pgmap v1328: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 9033/73986 objects degraded (12.209%); 26313/73986 objects misplaced (35.565%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:40:24.635 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:40:24.635 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:40:24.684 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:24.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:24 smithi152 ceph-mon[156819]: pgmap v1328: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 553 op/s; 9033/73986 objects degraded (12.209%); 26313/73986 objects misplaced (35.565%); 12 MiB/s, 190 objects/s recovering 2023-06-10T21:40:24.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:24.869 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:25.344 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:25.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:25.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:25.490 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:40:25.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:40:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:40:25.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:40:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:26.003 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:26.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:26.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:26.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:26.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:26 smithi064 ceph-mon[180297]: pgmap v1329: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 8093/66288 objects degraded (12.209%); 23407/66288 objects misplaced (35.311%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:26.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 9033/73986 objects degraded (12.209%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:26.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:26 smithi064 ceph-mon[186706]: pgmap v1329: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 8093/66288 objects degraded (12.209%); 23407/66288 objects misplaced (35.311%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:26.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 9033/73986 objects degraded (12.209%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:26 smithi152 ceph-mon[156819]: pgmap v1329: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 569 op/s; 8093/66288 objects degraded (12.209%); 23407/66288 objects misplaced (35.311%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 9033/73986 objects degraded (12.209%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:27.049 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:27.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:27.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:27.188 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:27.449 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:40:27.669 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:27.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:27.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:27.785 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:27.937 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:28.211 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:28.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:28.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:28.448 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:28.521 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:28 smithi064 ceph-mon[180297]: pgmap v1330: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 8093/65817 objects degraded (12.296%); 23250/65817 objects misplaced (35.325%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:28.522 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:28 smithi064 ceph-mon[186706]: pgmap v1330: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 8093/65817 objects degraded (12.296%); 23250/65817 objects misplaced (35.325%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:28.539 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:28 smithi152 ceph-mon[156819]: pgmap v1330: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 567 op/s; 8093/65817 objects degraded (12.296%); 23250/65817 objects misplaced (35.325%); 13 MiB/s, 200 objects/s recovering 2023-06-10T21:40:28.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:40:28.808 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:40:28.841 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:40:28.885 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:40:28.888 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:28.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:28.927 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:40:28.971 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:40:29.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:29.006 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:40:29.035 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:29.056 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:40:29.060 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.061 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.062 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:40:29.094 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:29.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:29.200 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:29.236 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:40:29.255 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:29.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/149122280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:29.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4210669755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:29.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/149122280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4210669755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:29.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/149122280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4210669755' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:40:30.006 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:30.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:30.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:30.142 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:30.187 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:30.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:30.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:30.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:40:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:30 smithi152 ceph-mon[156819]: osdmap e358: 8 total, 8 up, 4 in 2023-06-10T21:40:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:30 smithi152 ceph-mon[156819]: failed to encode map e358 with expected crc 2023-06-10T21:40:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:30 smithi152 ceph-mon[156819]: pgmap v1332: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7880/61782 objects degraded (12.755%); 21532/61782 objects misplaced (34.852%); 10 MiB/s, 165 objects/s recovering 2023-06-10T21:40:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:30 smithi152 ceph-mon[156819]: osdmap e359: 8 total, 8 up, 4 in 2023-06-10T21:40:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:40:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[186706]: osdmap e358: 8 total, 8 up, 4 in 2023-06-10T21:40:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[186706]: failed to encode map e358 with expected crc 2023-06-10T21:40:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[186706]: pgmap v1332: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7880/61782 objects degraded (12.755%); 21532/61782 objects misplaced (34.852%); 10 MiB/s, 165 objects/s recovering 2023-06-10T21:40:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[186706]: osdmap e359: 8 total, 8 up, 4 in 2023-06-10T21:40:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:40:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[180297]: osdmap e358: 8 total, 8 up, 4 in 2023-06-10T21:40:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[180297]: failed to encode map e358 with expected crc 2023-06-10T21:40:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[180297]: pgmap v1332: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 8 active+recovery_wait+degraded+remapped, 72 active+clean; 1.3 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7880/61782 objects degraded (12.755%); 21532/61782 objects misplaced (34.852%); 10 MiB/s, 165 objects/s recovering 2023-06-10T21:40:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:30 smithi064 ceph-mon[180297]: osdmap e359: 8 total, 8 up, 4 in 2023-06-10T21:40:31.198 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:31.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:31.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:31.353 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:31.399 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:31.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:31.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:31.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:31.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: failed to encode map e358 with expected crc 2023-06-10T21:40:31.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.532 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.533 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.a starting backfill to osd.3 from (0'0,329'6593] MAX to 357'9860 2023-06-10T21:40:31.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.1 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8486 2023-06-10T21:40:31.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.11 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8390 2023-06-10T21:40:31.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.16 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8442 2023-06-10T21:40:31.534 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8430 2023-06-10T21:40:31.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.a continuing backfill to osd.7 from (333'6613,357'9860] MIN to 357'9860 2023-06-10T21:40:31.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.16 continuing backfill to osd.5 from (320'3396,357'8442] MIN to 357'8442 2023-06-10T21:40:31.535 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: 12.6 continuing backfill to osd.5 from (320'3399,357'8430] MIN to 357'8430 2023-06-10T21:40:31.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.536 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7880/61782 objects degraded (12.755%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:31.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:31 smithi152 ceph-mon[156819]: osdmap e360: 8 total, 8 up, 4 in 2023-06-10T21:40:31.537 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:31.578 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:31.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: failed to encode map e358 with expected crc 2023-06-10T21:40:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.a starting backfill to osd.3 from (0'0,329'6593] MAX to 357'9860 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.1 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8486 2023-06-10T21:40:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.11 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8390 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.16 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8442 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8430 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.a continuing backfill to osd.7 from (333'6613,357'9860] MIN to 357'9860 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.16 continuing backfill to osd.5 from (320'3396,357'8442] MIN to 357'8442 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: 12.6 continuing backfill to osd.5 from (320'3399,357'8430] MIN to 357'8430 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7880/61782 objects degraded (12.755%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:31.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[180297]: osdmap e360: 8 total, 8 up, 4 in 2023-06-10T21:40:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: failed to encode map e358 with expected crc 2023-06-10T21:40:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: failed to encode map e359 with expected crc 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.a starting backfill to osd.3 from (0'0,329'6593] MAX to 357'9860 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.1 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8486 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.11 starting backfill to osd.3 from (319'2066,338'6912] MIN to 357'8390 2023-06-10T21:40:31.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.16 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8442 2023-06-10T21:40:31.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.6 starting backfill to osd.3 from (0'0,0'0] MAX to 357'8430 2023-06-10T21:40:31.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.a continuing backfill to osd.7 from (333'6613,357'9860] MIN to 357'9860 2023-06-10T21:40:31.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.16 continuing backfill to osd.5 from (320'3396,357'8442] MIN to 357'8442 2023-06-10T21:40:31.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: 12.6 continuing backfill to osd.5 from (320'3399,357'8430] MIN to 357'8430 2023-06-10T21:40:31.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:31.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7880/61782 objects degraded (12.755%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:31.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:31 smithi064 ceph-mon[186706]: osdmap e360: 8 total, 8 up, 4 in 2023-06-10T21:40:31.684 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:31.720 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.12 continuing backfill to osd.5 from (320'3447,357'8581] MIN to 357'8581 2023-06-10T21:40:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.14 continuing backfill to osd.2 from (320'3487,357'8550] MIN to 357'8550 2023-06-10T21:40:32.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.4 continuing backfill to osd.2 from (320'3507,357'8562] MIN to 357'8562 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.2 continuing backfill to osd.5 from (320'3559,357'8575] MIN to 357'8575 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.3 continuing backfill to osd.5 from (320'3524,357'8654] MIN to 357'8654 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.e continuing backfill to osd.5 from (333'6610,357'9761] MIN to 357'9761 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.18 continuing backfill to osd.7 from (320'3382,357'8671] MIN to 357'8671 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.8 continuing backfill to osd.7 from (320'3594,357'8670] MIN to 357'8670 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.13 continuing backfill to osd.5 from (320'3775,357'8675] MIN to 357'8675 2023-06-10T21:40:32.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.9 continuing backfill to osd.5 from (320'3441,357'8603] MIN to 357'8603 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: 12.19 starting backfill to osd.5 from (320'3449,357'8598] MIN to 357'8599 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: pgmap v1335: 99 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 344 op/s; 5627/60081 objects degraded (9.366%); 20241/60081 objects misplaced (33.690%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: failed to encode map e361 with expected crc 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[180297]: osdmap e361: 8 total, 8 up, 4 in 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.12 continuing backfill to osd.5 from (320'3447,357'8581] MIN to 357'8581 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.14 continuing backfill to osd.2 from (320'3487,357'8550] MIN to 357'8550 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.4 continuing backfill to osd.2 from (320'3507,357'8562] MIN to 357'8562 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.2 continuing backfill to osd.5 from (320'3559,357'8575] MIN to 357'8575 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.3 continuing backfill to osd.5 from (320'3524,357'8654] MIN to 357'8654 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.e continuing backfill to osd.5 from (333'6610,357'9761] MIN to 357'9761 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.18 continuing backfill to osd.7 from (320'3382,357'8671] MIN to 357'8671 2023-06-10T21:40:32.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.8 continuing backfill to osd.7 from (320'3594,357'8670] MIN to 357'8670 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.13 continuing backfill to osd.5 from (320'3775,357'8675] MIN to 357'8675 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.9 continuing backfill to osd.5 from (320'3441,357'8603] MIN to 357'8603 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: 12.19 starting backfill to osd.5 from (320'3449,357'8598] MIN to 357'8599 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: pgmap v1335: 99 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 344 op/s; 5627/60081 objects degraded (9.366%); 20241/60081 objects misplaced (33.690%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: failed to encode map e361 with expected crc 2023-06-10T21:40:32.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:32 smithi064 ceph-mon[186706]: osdmap e361: 8 total, 8 up, 4 in 2023-06-10T21:40:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: failed to encode map e360 with expected crc 2023-06-10T21:40:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.12 continuing backfill to osd.5 from (320'3447,357'8581] MIN to 357'8581 2023-06-10T21:40:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.14 continuing backfill to osd.2 from (320'3487,357'8550] MIN to 357'8550 2023-06-10T21:40:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.4 continuing backfill to osd.2 from (320'3507,357'8562] MIN to 357'8562 2023-06-10T21:40:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.2 continuing backfill to osd.5 from (320'3559,357'8575] MIN to 357'8575 2023-06-10T21:40:32.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.3 continuing backfill to osd.5 from (320'3524,357'8654] MIN to 357'8654 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.e continuing backfill to osd.5 from (333'6610,357'9761] MIN to 357'9761 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.18 continuing backfill to osd.7 from (320'3382,357'8671] MIN to 357'8671 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.8 continuing backfill to osd.7 from (320'3594,357'8670] MIN to 357'8670 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.13 continuing backfill to osd.5 from (320'3775,357'8675] MIN to 357'8675 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.9 continuing backfill to osd.5 from (320'3441,357'8603] MIN to 357'8603 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: 12.19 starting backfill to osd.5 from (320'3449,357'8598] MIN to 357'8599 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: pgmap v1335: 99 pgs: 1 activating+degraded+remapped, 2 remapped+peering, 2 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 344 op/s; 5627/60081 objects degraded (9.366%); 20241/60081 objects misplaced (33.690%); 10 MiB/s, 164 objects/s recovering 2023-06-10T21:40:32.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: failed to encode map e361 with expected crc 2023-06-10T21:40:32.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:32 smithi152 ceph-mon[156819]: osdmap e361: 8 total, 8 up, 4 in 2023-06-10T21:40:33.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:32] "GET /metrics HTTP/1.1" 200 224250 "" "Prometheus/2.33.4" 2023-06-10T21:40:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[186706]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[186706]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[186706]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[180297]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[180297]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[180297]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:33.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:33 smithi152 ceph-mon[156819]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:33 smithi152 ceph-mon[156819]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:33 smithi152 ceph-mon[156819]: failed to encode map e361 with expected crc 2023-06-10T21:40:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:33.795 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:33.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:33.901 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:33.972 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:34.572 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:34 smithi064 ceph-mon[180297]: pgmap v1337: 99 pgs: 1 active+remapped+backfilling, 1 active+degraded+remapped, 1 activating+degraded+remapped, 2 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 167 op/s; 6209/59328 objects degraded (10.466%); 14763/59328 objects misplaced (24.884%); 23 MiB/s, 362 objects/s recovering 2023-06-10T21:40:34.572 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:34 smithi064 ceph-mon[186706]: pgmap v1337: 99 pgs: 1 active+remapped+backfilling, 1 active+degraded+remapped, 1 activating+degraded+remapped, 2 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 167 op/s; 6209/59328 objects degraded (10.466%); 14763/59328 objects misplaced (24.884%); 23 MiB/s, 362 objects/s recovering 2023-06-10T21:40:34.601 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:34.602 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:40:34.620 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:34.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:34.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:34 smithi152 ceph-mon[156819]: pgmap v1337: 99 pgs: 1 active+remapped+backfilling, 1 active+degraded+remapped, 1 activating+degraded+remapped, 2 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 72 active+clean; 1.2 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 167 op/s; 6209/59328 objects degraded (10.466%); 14763/59328 objects misplaced (24.884%); 23 MiB/s, 362 objects/s recovering 2023-06-10T21:40:34.743 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:34.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:35.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:35.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:35.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:35.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:35.412 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:35.459 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:35.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:35.568 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:35.592 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:35 smithi152 ceph-mon[156819]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.592 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:35 smithi152 ceph-mon[156819]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.592 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:35 smithi152 ceph-mon[156819]: osdmap e362: 8 total, 8 up, 4 in 2023-06-10T21:40:35.592 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:35 smithi152 ceph-mon[156819]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.594 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:35.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[180297]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[180297]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[180297]: osdmap e362: 8 total, 8 up, 4 in 2023-06-10T21:40:35.625 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[180297]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[186706]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[186706]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[186706]: osdmap e362: 8 total, 8 up, 4 in 2023-06-10T21:40:35.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:35 smithi064 ceph-mon[186706]: failed to encode map e362 with expected crc 2023-06-10T21:40:35.635 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:35.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:35.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:35.741 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:35.774 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:36.313 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:40:36.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:40:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: osdmap e363: 8 total, 8 up, 4 in 2023-06-10T21:40:36.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2469119610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: pgmap v1340: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 243 op/s; 8638/56451 objects degraded (15.302%); 14631/56451 objects misplaced (25.918%); 28 MiB/s, 451 objects/s recovering 2023-06-10T21:40:36.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 6209/59328 objects degraded (10.466%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:36.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: osdmap e363: 8 total, 8 up, 4 in 2023-06-10T21:40:36.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2469119610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: pgmap v1340: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 243 op/s; 8638/56451 objects degraded (15.302%); 14631/56451 objects misplaced (25.918%); 28 MiB/s, 451 objects/s recovering 2023-06-10T21:40:36.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 6209/59328 objects degraded (10.466%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: osdmap e363: 8 total, 8 up, 4 in 2023-06-10T21:40:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: failed to encode map e363 with expected crc 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2469119610' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: pgmap v1340: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 243 op/s; 8638/56451 objects degraded (15.302%); 14631/56451 objects misplaced (25.918%); 28 MiB/s, 451 objects/s recovering 2023-06-10T21:40:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 6209/59328 objects degraded (10.466%), 7 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:37.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: osdmap e364: 8 total, 8 up, 4 in 2023-06-10T21:40:37.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[186706]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: osdmap e364: 8 total, 8 up, 4 in 2023-06-10T21:40:37.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:37 smithi064 ceph-mon[180297]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:40:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: osdmap e364: 8 total, 8 up, 4 in 2023-06-10T21:40:37.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:37 smithi152 ceph-mon[156819]: failed to encode map e364 with expected crc 2023-06-10T21:40:37.907 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:37.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:38.022 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:38.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:38.321 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:38 smithi064 ceph-mon[180297]: pgmap v1342: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 299 op/s; 8344/54705 objects degraded (15.253%); 14342/54705 objects misplaced (26.217%); 26 MiB/s, 424 objects/s recovering 2023-06-10T21:40:38.485 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:38 smithi064 ceph-mon[186706]: pgmap v1342: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 299 op/s; 8344/54705 objects degraded (15.253%); 14342/54705 objects misplaced (26.217%); 26 MiB/s, 424 objects/s recovering 2023-06-10T21:40:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:38 smithi152 ceph-mon[156819]: pgmap v1342: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 73 active+clean; 1.1 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 299 op/s; 8344/54705 objects degraded (15.253%); 14342/54705 objects misplaced (26.217%); 26 MiB/s, 424 objects/s recovering 2023-06-10T21:40:38.716 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:38.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:38.823 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:38.936 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:39.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:39.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:39.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:39.321 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:39.341 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:39.392 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:39.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:39.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:39.528 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:39.580 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:40:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:40:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:40:39.583 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:40:39.583 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:40:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:39.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:39.688 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:39.717 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:39.771 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:39.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:39.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:39.911 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:39.960 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:39.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:40.070 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:40.150 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:40.207 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:40 smithi064 ceph-mon[180297]: pgmap v1343: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 441 op/s; 8189/51369 objects degraded (15.942%); 13315/51369 objects misplaced (25.920%); 18 MiB/s, 296 objects/s recovering 2023-06-10T21:40:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:40 smithi152 ceph-mon[156819]: pgmap v1343: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 441 op/s; 8189/51369 objects degraded (15.942%); 13315/51369 objects misplaced (25.920%); 18 MiB/s, 296 objects/s recovering 2023-06-10T21:40:40.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:40 smithi064 ceph-mon[186706]: pgmap v1343: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1.0 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 441 op/s; 8189/51369 objects degraded (15.942%); 13315/51369 objects misplaced (25.920%); 18 MiB/s, 296 objects/s recovering 2023-06-10T21:40:40.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:40:41.459 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:41.466 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 8189/51369 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:41.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:41.468 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 8189/51369 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:41.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:41.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:41.649 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 8189/51369 objects degraded (15.942%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:42.500 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:42.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:42.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:42.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:42 smithi064 ceph-mon[180297]: pgmap v1344: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1017 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 393 op/s; 7426/48846 objects degraded (15.203%); 12887/48846 objects misplaced (26.383%); 13 MiB/s, 205 objects/s recovering 2023-06-10T21:40:42.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:42 smithi064 ceph-mon[186706]: pgmap v1344: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1017 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 393 op/s; 7426/48846 objects degraded (15.203%); 12887/48846 objects misplaced (26.383%); 13 MiB/s, 205 objects/s recovering 2023-06-10T21:40:42.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:42.646 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:42 smithi152 ceph-mon[156819]: pgmap v1344: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 1017 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 393 op/s; 7426/48846 objects degraded (15.203%); 12887/48846 objects misplaced (26.383%); 13 MiB/s, 205 objects/s recovering 2023-06-10T21:40:42.649 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:42.697 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:42.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:42.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:42.835 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:42.885 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:42] "GET /metrics HTTP/1.1" 200 224223 "" "Prometheus/2.33.4" 2023-06-10T21:40:42.889 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:42.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:42.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:43.027 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:43.082 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:43.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:43.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:43.216 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:43.273 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:43.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:43.382 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:43.413 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:43.468 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:43.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:43.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:43.600 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:43.641 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:43.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:43.746 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:43.779 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:43.821 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:43.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:43.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:43.962 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:44.007 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:44.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:44.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:44.030 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:40:44.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:44.143 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:44.196 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:44.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:44.296 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:44 smithi064 ceph-mon[180297]: pgmap v1345: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 948 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 474 op/s; 7003/45543 objects degraded (15.377%); 12426/45543 objects misplaced (27.284%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:44.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:44 smithi064 ceph-mon[186706]: pgmap v1345: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 948 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 474 op/s; 7003/45543 objects degraded (15.377%); 12426/45543 objects misplaced (27.284%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:44.302 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:44.329 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:44.354 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:44.377 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:44.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:44.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:44.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:44 smithi152 ceph-mon[156819]: pgmap v1345: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 948 MiB data, 5.2 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 474 op/s; 7003/45543 objects degraded (15.377%); 12426/45543 objects misplaced (27.284%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:44.512 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:44.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:44.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:44.675 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:44.704 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:44.756 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:44.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:44.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:44.862 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:44.968 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:45.280 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:40:45.280 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/474187515' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:45.281 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:45.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/474187515' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:45.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/474187515' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:40:46.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:40:46.633 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:40:46.633 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:40:46.653 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:40:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:40:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[180297]: osdmap e365: 8 total, 8 up, 4 in 2023-06-10T21:40:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[180297]: pgmap v1347: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 813 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 590 op/s; 6005/39075 objects degraded (15.368%); 10944/39075 objects misplaced (28.008%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:40:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 7003/45543 objects degraded (15.377%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:40:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[186706]: osdmap e365: 8 total, 8 up, 4 in 2023-06-10T21:40:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[186706]: pgmap v1347: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 813 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 590 op/s; 6005/39075 objects degraded (15.368%); 10944/39075 objects misplaced (28.008%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:40:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 7003/45543 objects degraded (15.377%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:40:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:46 smithi152 ceph-mon[156819]: osdmap e365: 8 total, 8 up, 4 in 2023-06-10T21:40:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:46 smithi152 ceph-mon[156819]: pgmap v1347: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 813 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 590 op/s; 6005/39075 objects degraded (15.368%); 10944/39075 objects misplaced (28.008%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:40:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 7003/45543 objects degraded (15.377%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:46.703 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:40:46.735 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:40:46.782 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:40:46.821 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:40:46.822 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:40:46.823 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:40:47.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[180297]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[180297]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/992086549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:47.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[186706]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[186706]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/992086549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:47 smithi152 ceph-mon[156819]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:47 smithi152 ceph-mon[156819]: failed to encode map e365 with expected crc 2023-06-10T21:40:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/992086549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:40:48.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:48 smithi064 ceph-mon[186706]: pgmap v1348: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 762 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 5601/36609 objects degraded (15.300%); 10526/36609 objects misplaced (28.752%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:48.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:48.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:48 smithi064 ceph-mon[180297]: pgmap v1348: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 762 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 5601/36609 objects degraded (15.300%); 10526/36609 objects misplaced (28.752%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:48.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:48.615 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:48.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:48 smithi152 ceph-mon[156819]: pgmap v1348: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 762 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 5601/36609 objects degraded (15.300%); 10526/36609 objects misplaced (28.752%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:40:48.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:48.787 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:48.846 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:48.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:48.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:49.038 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:49.297 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:49.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:49.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:49.462 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:49.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:40:50.029 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:50.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:50.135 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:50.243 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:50.340 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:50.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:50 smithi064 ceph-mon[180297]: pgmap v1349: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 675 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 5396/32463 objects degraded (16.622%); 9417/32463 objects misplaced (29.008%); 14 MiB/s, 224 objects/s recovering 2023-06-10T21:40:50.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:50 smithi064 ceph-mon[186706]: pgmap v1349: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 675 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 5396/32463 objects degraded (16.622%); 9417/32463 objects misplaced (29.008%); 14 MiB/s, 224 objects/s recovering 2023-06-10T21:40:50.563 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:50.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:50.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:50 smithi152 ceph-mon[156819]: pgmap v1349: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 675 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 5396/32463 objects degraded (16.622%); 9417/32463 objects misplaced (29.008%); 14 MiB/s, 224 objects/s recovering 2023-06-10T21:40:50.762 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:51.187 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:51.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:51.270 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:40:51.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:51.328 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:51.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:51.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:51.486 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:51.514 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3539557647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:51 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:51 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.515 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 5396/32463 objects degraded (16.622%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:51.519 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:51.576 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:51.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:51.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:51.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3539557647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 5396/32463 objects degraded (16.622%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:51.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3539557647' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:40:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:51.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 5396/32463 objects degraded (16.622%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:51.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:51.718 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:51.774 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:51.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:40:51.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:51.928 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:40:51.952 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[180297]: failed to encode map e366 with expected crc 2023-06-10T21:40:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:40:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[180297]: osdmap e366: 8 total, 8 up, 4 in 2023-06-10T21:40:52.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[180297]: pgmap v1351: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 611 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 673 op/s; 4521/29379 objects degraded (15.389%); 9000/29379 objects misplaced (30.634%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[186706]: failed to encode map e366 with expected crc 2023-06-10T21:40:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:40:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[186706]: osdmap e366: 8 total, 8 up, 4 in 2023-06-10T21:40:52.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:52 smithi064 ceph-mon[186706]: pgmap v1351: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 611 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 673 op/s; 4521/29379 objects degraded (15.389%); 9000/29379 objects misplaced (30.634%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:52 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:40:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:52 smithi152 ceph-mon[156819]: failed to encode map e366 with expected crc 2023-06-10T21:40:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:40:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:52 smithi152 ceph-mon[156819]: osdmap e366: 8 total, 8 up, 4 in 2023-06-10T21:40:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:52 smithi152 ceph-mon[156819]: pgmap v1351: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 611 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 673 op/s; 4521/29379 objects degraded (15.389%); 9000/29379 objects misplaced (30.634%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:40:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:40:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:40:52] "GET /metrics HTTP/1.1" 200 224271 "" "Prometheus/2.33.4" 2023-06-10T21:40:53.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[180297]: failed to encode map e366 with expected crc 2023-06-10T21:40:53.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:40:53.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:40:53.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[186706]: failed to encode map e366 with expected crc 2023-06-10T21:40:53.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:40:53.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:40:53.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:53 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:53 smithi152 ceph-mon[156819]: failed to encode map e366 with expected crc 2023-06-10T21:40:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:40:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:40:53.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:53 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:40:53.750 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:53.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:53.855 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:53.889 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:53.955 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:53.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:53.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:54.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:40:54.092 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:54.147 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:40:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:54.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:40:54.219 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:54.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:54.361 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:54.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:54 smithi152 ceph-mon[156819]: pgmap v1352: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 555 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 4180/26679 objects degraded (15.668%); 8638/26679 objects misplaced (32.378%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:54.523 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:54.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:54 smithi064 ceph-mon[180297]: pgmap v1352: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 555 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 4180/26679 objects degraded (15.668%); 8638/26679 objects misplaced (32.378%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:54.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:54 smithi064 ceph-mon[186706]: pgmap v1352: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 555 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 4180/26679 objects degraded (15.668%); 8638/26679 objects misplaced (32.378%); 14 MiB/s, 230 objects/s recovering 2023-06-10T21:40:54.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:54.575 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:40:54.637 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:40:54.722 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:54.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:40:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:54.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:40:55.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:40:55.090 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:55.199 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:40:55.199 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:40:55.398 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:40:55.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:55.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:40:55.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:55.623 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:56.271 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:56.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:56.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:56.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:40:56.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:40:56.405 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:56 smithi152 ceph-mon[156819]: pgmap v1353: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 425 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 620 op/s; 3203/20433 objects degraded (15.676%); 7180/20433 objects misplaced (35.139%); 14 MiB/s, 231 objects/s recovering 2023-06-10T21:40:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 4180/26679 objects degraded (15.668%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:56.463 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:40:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:56.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:40:56.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:40:56.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:56 smithi064 ceph-mon[180297]: pgmap v1353: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 425 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 620 op/s; 3203/20433 objects degraded (15.676%); 7180/20433 objects misplaced (35.139%); 14 MiB/s, 231 objects/s recovering 2023-06-10T21:40:56.644 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 4180/26679 objects degraded (15.668%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:56.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:56 smithi064 ceph-mon[186706]: pgmap v1353: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 425 MiB data, 4.2 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 620 op/s; 3203/20433 objects degraded (15.676%); 7180/20433 objects misplaced (35.139%); 14 MiB/s, 231 objects/s recovering 2023-06-10T21:40:56.645 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 4180/26679 objects degraded (15.668%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:40:56.654 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:58.618 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:40:58 smithi064 ceph-mon[180297]: pgmap v1354: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 372 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 622 op/s; 2765/17913 objects degraded (15.436%); 6778/17913 objects misplaced (37.838%); 15 MiB/s, 235 objects/s recovering 2023-06-10T21:40:58.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:40:58 smithi064 ceph-mon[186706]: pgmap v1354: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 372 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 622 op/s; 2765/17913 objects degraded (15.436%); 6778/17913 objects misplaced (37.838%); 15 MiB/s, 235 objects/s recovering 2023-06-10T21:40:58.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:40:58.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:40:58 smithi152 ceph-mon[156819]: pgmap v1354: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 372 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 622 op/s; 2765/17913 objects degraded (15.436%); 6778/17913 objects misplaced (37.838%); 15 MiB/s, 235 objects/s recovering 2023-06-10T21:40:58.949 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:40:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:58.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:40:59.061 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:59.094 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:59.137 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:59.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:59.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:40:59.272 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:59.328 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:40:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:59.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:40:59.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:40:59.514 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:59.723 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:40:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:59.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:40:59.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:40:59.863 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:40:59.921 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:40:59.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:40:59.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:00.029 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:00.097 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:00.200 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:00.221 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:00 smithi064 ceph-mon[180297]: pgmap v1355: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 284 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 625 op/s; 2563/13683 objects degraded (18.731%); 5681/13683 objects misplaced (41.519%); 15 MiB/s, 234 objects/s recovering 2023-06-10T21:41:00.221 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:00 smithi064 ceph-mon[186706]: pgmap v1355: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 284 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 625 op/s; 2563/13683 objects degraded (18.731%); 5681/13683 objects misplaced (41.519%); 15 MiB/s, 234 objects/s recovering 2023-06-10T21:41:00.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:00.415 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:00.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:00.528 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:00.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:00 smithi152 ceph-mon[156819]: pgmap v1355: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 284 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 625 op/s; 2563/13683 objects degraded (18.731%); 5681/13683 objects misplaced (41.519%); 15 MiB/s, 234 objects/s recovering 2023-06-10T21:41:00.557 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:00.557 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:00.606 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:00.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:00.615 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:00.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:00.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:00.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:00.774 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:00.808 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:00.838 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:00.868 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:00.911 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:00.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.921 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:41:00.922 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:41:01.252 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:41:01.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/951920806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:01.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2197860302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 2563/13683 objects degraded (18.731%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/951920806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2197860302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 2563/13683 objects degraded (18.731%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:01.579 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:01.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:01.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/951920806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2197860302' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:41:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 2563/13683 objects degraded (18.731%), 9 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:01.716 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:01.778 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:01.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:01.886 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:01.974 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:02.247 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[180297]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:41:02.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[180297]: osdmap e367: 8 total, 8 up, 4 in 2023-06-10T21:41:02.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[180297]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.248 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[180297]: pgmap v1357: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 220 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1929/10623 objects degraded (18.159%); 5448/10623 objects misplaced (51.285%); 13 MiB/s, 235 objects/s recovering 2023-06-10T21:41:02.465 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:02.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:02.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:02.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:02.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[186706]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.643 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:41:02.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[186706]: osdmap e367: 8 total, 8 up, 4 in 2023-06-10T21:41:02.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[186706]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:02 smithi064 ceph-mon[186706]: pgmap v1357: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 220 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1929/10623 objects degraded (18.159%); 5448/10623 objects misplaced (51.285%); 13 MiB/s, 235 objects/s recovering 2023-06-10T21:41:02.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:02 smithi152 ceph-mon[156819]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:02 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:41:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:02 smithi152 ceph-mon[156819]: osdmap e367: 8 total, 8 up, 4 in 2023-06-10T21:41:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:02 smithi152 ceph-mon[156819]: failed to encode map e367 with expected crc 2023-06-10T21:41:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:02 smithi152 ceph-mon[156819]: pgmap v1357: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 220 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 624 op/s; 1929/10623 objects degraded (18.159%); 5448/10623 objects misplaced (51.285%); 13 MiB/s, 235 objects/s recovering 2023-06-10T21:41:02.785 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:02.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:02.896 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:02.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:02] "GET /metrics HTTP/1.1" 200 224271 "" "Prometheus/2.33.4" 2023-06-10T21:41:02.927 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:02.986 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:02.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:03.093 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:03.174 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:03.308 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:03.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:03.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:03.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:03.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:03.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:03.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:03.798 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 88047 objects 2023-06-10T21:41:03.798 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :199.992 2023-06-10T21:41:04.118 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:04.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:04.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:04.250 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:04.294 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:04.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:04.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:04.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:04.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:04 smithi064 ceph-mon[186706]: pgmap v1358: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 148 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 650 op/s; 1522/7164 objects degraded (21.245%); 4916/7164 objects misplaced (68.621%); 12 MiB/s, 212 objects/s recovering 2023-06-10T21:41:04.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:04 smithi064 ceph-mon[180297]: pgmap v1358: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 148 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 650 op/s; 1522/7164 objects degraded (21.245%); 4916/7164 objects misplaced (68.621%); 12 MiB/s, 212 objects/s recovering 2023-06-10T21:41:04.427 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:04.469 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:04.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:04.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:04.604 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:04 smithi152 ceph-mon[156819]: pgmap v1358: 99 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 148 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 650 op/s; 1522/7164 objects degraded (21.245%); 4916/7164 objects misplaced (68.621%); 12 MiB/s, 212 objects/s recovering 2023-06-10T21:41:04.955 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:04.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:05.059 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:05.089 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:05.135 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:05.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:05.240 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:05.268 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:06 smithi152 ceph-mon[156819]: pgmap v1359: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 26 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 651 op/s; 829/1311 objects degraded (63.234%); 3897/1311 objects misplaced (297.254%); 8.0 MiB/s, 213 objects/s recovering 2023-06-10T21:41:06.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 1522/7164 objects degraded (21.245%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:06.501 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:06 smithi064 ceph-mon[186706]: pgmap v1359: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 26 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 651 op/s; 829/1311 objects degraded (63.234%); 3897/1311 objects misplaced (297.254%); 8.0 MiB/s, 213 objects/s recovering 2023-06-10T21:41:06.502 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 1522/7164 objects degraded (21.245%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:06.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:06 smithi064 ceph-mon[180297]: pgmap v1359: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 74 active+clean; 26 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 651 op/s; 829/1311 objects degraded (63.234%); 3897/1311 objects misplaced (297.254%); 8.0 MiB/s, 213 objects/s recovering 2023-06-10T21:41:06.503 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 1522/7164 objects degraded (21.245%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:06.503 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:06.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:06.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:06.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 5 1 2023-06-10T21:41:06.607 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:06.639 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:06.693 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:06.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:06.785 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-06-10T21:41:06.786 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-06-10T21:41:06.796 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:06.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:07.142 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:07.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:07.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:07.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:07.329 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:07.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:07.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:07.354 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:41:07.355 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:41:07.355 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:41:07.355 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:41:07.362 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:07.414 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:07.435 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:07.446 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:41:07.464 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:07.509 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:07.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:07.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:07.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3776066870' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.648 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:07.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3776066870' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:07 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:07.650 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3776066870' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:07 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:07 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:08.660 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: osdmap e368: 8 total, 8 up, 4 in 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[186706]: pgmap v1361: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 566 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 5.1 MiB/s, 187 objects/s recovering 2023-06-10T21:41:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: osdmap e368: 8 total, 8 up, 4 in 2023-06-10T21:41:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:08 smithi064 ceph-mon[180297]: pgmap v1361: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 566 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 5.1 MiB/s, 187 objects/s recovering 2023-06-10T21:41:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: osdmap e368: 8 total, 8 up, 4 in 2023-06-10T21:41:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: failed to encode map e368 with expected crc 2023-06-10T21:41:08.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:08 smithi152 ceph-mon[156819]: pgmap v1361: 99 pgs: 3 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 566 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 5.1 MiB/s, 187 objects/s recovering 2023-06-10T21:41:09.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:09.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:09.650 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:09.684 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:09.733 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:09.739 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:09 smithi152 ceph-mon[156819]: failed to encode map e368 with expected crc 2023-06-10T21:41:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:09.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:09.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:09.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:09 smithi064 ceph-mon[180297]: failed to encode map e368 with expected crc 2023-06-10T21:41:09.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:09 smithi064 ceph-mon[186706]: failed to encode map e368 with expected crc 2023-06-10T21:41:09.870 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:10.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:41:10.202 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:10.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:10.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:10.340 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:10.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:10 smithi064 ceph-mon[186706]: pgmap v1362: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 395 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 2.0 MiB/s, 101 objects/s recovering 2023-06-10T21:41:10.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:10 smithi064 ceph-mon[180297]: pgmap v1362: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 395 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 2.0 MiB/s, 101 objects/s recovering 2023-06-10T21:41:10.680 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:10.700 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:10.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:10.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:10.805 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:10.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:10 smithi152 ceph-mon[156819]: pgmap v1362: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovering+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 14 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 395 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 2.0 MiB/s, 101 objects/s recovering 2023-06-10T21:41:10.838 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:10.886 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:10.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:10.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:10.993 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:11.021 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:11.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:11.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:11.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:11.172 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:11.199 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:11.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:11.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:11.348 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:11.382 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:11.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 698/543 objects degraded (128.545%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 698/543 objects degraded (128.545%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:11.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 698/543 objects degraded (128.545%), 5 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:12.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:41:12.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:12.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:12.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:12.789 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:12 smithi152 ceph-mon[156819]: pgmap v1363: 99 pgs: 5 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 351 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 1.7 MiB/s, 258 objects/s recovering 2023-06-10T21:41:12.791 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:12.802 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:12 smithi064 ceph-mon[180297]: pgmap v1363: 99 pgs: 5 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 351 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 1.7 MiB/s, 258 objects/s recovering 2023-06-10T21:41:12.803 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:12 smithi064 ceph-mon[186706]: pgmap v1363: 99 pgs: 5 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 15 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 351 op/s; 698/543 objects degraded (128.545%); 3772/543 objects misplaced (694.659%); 1.7 MiB/s, 258 objects/s recovering 2023-06-10T21:41:12.838 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:12.838 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:12.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:12.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:12.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:12.971 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:13.012 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:13.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:13.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:13.108 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:12] "GET /metrics HTTP/1.1" 200 224244 "" "Prometheus/2.33.4" 2023-06-10T21:41:13.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:13.143 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:13.184 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:13.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:13.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:13.291 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:13.327 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:13.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1416505682' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:13.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:14.001 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-06-10T21:41:14.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1416505682' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:14.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:14.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1416505682' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:14.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:41:14.787 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:14.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:14.892 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:14.920 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:14.963 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:14.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: failed to encode map e369 with expected crc 2023-06-10T21:41:14.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:41:14.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: osdmap e369: 8 total, 8 up, 4 in 2023-06-10T21:41:14.967 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: failed to encode map e369 with expected crc 2023-06-10T21:41:14.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: pgmap v1365: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 34 op/s; 691/543 objects degraded (127.256%); 3779/543 objects misplaced (695.948%); 0 B/s, 298 objects/s recovering 2023-06-10T21:41:14.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: failed to encode map e369 with expected crc 2023-06-10T21:41:14.968 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:14 smithi152 ceph-mon[156819]: failed to encode map e369 with expected crc 2023-06-10T21:41:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:14.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:15.070 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:15.097 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:15.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.107 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: osdmap e369: 8 total, 8 up, 4 in 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: pgmap v1365: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 34 op/s; 691/543 objects degraded (127.256%); 3779/543 objects misplaced (695.948%); 0 B/s, 298 objects/s recovering 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[186706]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.108 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: osdmap e369: 8 total, 8 up, 4 in 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: pgmap v1365: 99 pgs: 4 active+recovery_wait+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 17 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 10 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 34 op/s; 691/543 objects degraded (127.256%); 3779/543 objects misplaced (695.948%); 0 B/s, 298 objects/s recovering 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.109 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:14 smithi064 ceph-mon[180297]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.141 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:15.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:15.247 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:15.274 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:15.319 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:15.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:15.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:15.451 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:15.498 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:15.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:15.604 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:15.636 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:15.814 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_7' removed 2023-06-10T21:41:15.894 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: failed to encode map e370 with expected crc 2023-06-10T21:41:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: failed to encode map e370 with expected crc 2023-06-10T21:41:15.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: osdmap e370: 8 total, 8 up, 4 in 2023-06-10T21:41:15.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3671200048' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:15.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: failed to encode map e369 with expected crc 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: failed to encode map e370 with expected crc 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: failed to encode map e370 with expected crc 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: osdmap e370: 8 total, 8 up, 4 in 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3671200048' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:15.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:15 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:15.935 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:41:15.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:41:16.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:16.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:16.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:16.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: failed to encode map e369 with expected crc 2023-06-10T21:41:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: failed to encode map e370 with expected crc 2023-06-10T21:41:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: failed to encode map e370 with expected crc 2023-06-10T21:41:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: osdmap e370: 8 total, 8 up, 4 in 2023-06-10T21:41:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3671200048' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:16.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:15 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:41:16.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:16.268 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[186706]: failed to encode map e371 with expected crc 2023-06-10T21:41:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:41:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[186706]: osdmap e371: 8 total, 8 up, 4 in 2023-06-10T21:41:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[186706]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[180297]: failed to encode map e371 with expected crc 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[180297]: osdmap e371: 8 total, 8 up, 4 in 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[180297]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:16.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:17.134 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:17.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:17.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:16 smithi152 ceph-mon[156819]: failed to encode map e371 with expected crc 2023-06-10T21:41:17.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:41:17.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:16 smithi152 ceph-mon[156819]: osdmap e371: 8 total, 8 up, 4 in 2023-06-10T21:41:17.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:16 smithi152 ceph-mon[156819]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:17.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded (PG_DEGRADED) 2023-06-10T21:41:17.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:17.271 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:17.320 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:17.325 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:41:17.326 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:41:17.326 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-06-10T21:41:17.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_8 16 2023-06-10T21:41:17.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:17.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:17.430 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:17.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:17.631 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:17.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:17.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:17.767 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:17.817 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:17.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[180297]: failed to encode map e371 with expected crc 2023-06-10T21:41:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded) 2023-06-10T21:41:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:41:17.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[186706]: failed to encode map e371 with expected crc 2023-06-10T21:41:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded) 2023-06-10T21:41:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:17 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:41:17.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:17.957 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:18.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:18.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:18.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:18.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.146 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.147 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:41:18.157 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:18.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:18.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:17 smithi152 ceph-mon[156819]: failed to encode map e371 with expected crc 2023-06-10T21:41:18.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:17 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 691/543 objects degraded (127.256%), 3 pgs degraded) 2023-06-10T21:41:18.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:17 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:41:18.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:18.292 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:18.339 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:18.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:18.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:18.472 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:18.516 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:18.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:18.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:18.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:19.047 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2060029535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:19.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[180297]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:19.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:19.048 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:19.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2060029535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:19.048 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[186706]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:19.049 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:19.049 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:19.111 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:19.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2060029535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:18 smithi152 ceph-mon[156819]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:19.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:19.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:19.248 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:19.391 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:41:19.391 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:41:19.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:19.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:19.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:19.540 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:19.722 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:41:20.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1518524087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.121 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:19 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1518524087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.122 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:19 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1518524087' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:19 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:41:20.898 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:20.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:21.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:21.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:21.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:41:21.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: osdmap e372: 8 total, 8 up, 4 in 2023-06-10T21:41:21.029 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[186706]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: osdmap e372: 8 total, 8 up, 4 in 2023-06-10T21:41:21.030 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:21.031 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:20 smithi064 ceph-mon[180297]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.032 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:21.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:41:21.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:41:21.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: osdmap e372: 8 total, 8 up, 4 in 2023-06-10T21:41:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:21.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:20 smithi152 ceph-mon[156819]: failed to encode map e372 with expected crc 2023-06-10T21:41:21.367 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:41:21.430 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:21.504 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:21.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:21.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:21.607 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:21.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:21.642 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:21.691 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:21.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:21.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:21.831 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:22.083 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:22.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:22.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:22.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:22.218 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:22.268 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:22.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:22.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:22.405 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:22.469 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:41:22.470 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:41:23.148 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:22] "GET /metrics HTTP/1.1" 200 221718 "" "Prometheus/2.33.4" 2023-06-10T21:41:23.149 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:23 smithi064 ceph-mon[180297]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:23.215 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:23.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:23.320 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:23.352 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:23.400 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:23.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:23 smithi152 ceph-mon[156819]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:23.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:23.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:23 smithi064 ceph-mon[186706]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:23.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:41:23.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:23.540 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:23.692 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:23.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:23.796 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:23.828 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:23.875 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:23.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:23.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:24.012 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:24.058 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:24.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:24.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:24.192 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:24.372 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1372,"stamp":"2023-06-10T21:41:22.120406+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":2379958,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":253,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3573252,"kb_used_data":873380,"kb_used_omap":15,"kb_used_meta":2699824,"kb_avail":371407356,"statfs":{"total":383980142592,"available":380321132544,"internally_reserved":0,"allocated":894341120,"data_stored":890360679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16284,"internal_metadata":2764619876},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":52,"apply_latency_ms":52,"commit_latency_ns":52000000,"apply_latency_ns":52000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-10256384,"num_objects":-157,"num_object_clones":0,"num_object_copies":-471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-691,"num_objects_misplaced":-3779,"num_objects_unfound":0,"num_objects_dirty":-157,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-176096,"num_write_kb":-5635009,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-42441,"num_bytes_recovered":-2525495296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.427088"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":395,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.790600+0000","last_change":"2023-06-10T21:39:50.557548+0000","last_active":"2023-06-10T21:41:15.790600+0000","last_peered":"2023-06-10T21:41:15.790600+0000","last_clean":"2023-06-10T21:41:15.790600+0000","last_became_active":"2023-06-10T21:39:50.557023+0000","last_became_peered":"2023-06-10T21:39:50.557023+0000","last_unstale":"2023-06-10T21:41:15.790600+0000","last_undegraded":"2023-06-10T21:41:15.790600+0000","last_fullsized":"2023-06-10T21:41:15.790600+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:45.328782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701469+0000","last_change":"2023-06-10T21:40:30.243387+0000","last_active":"2023-06-10T21:41:14.701469+0000","last_peered":"2023-06-10T21:41:14.701469+0000","last_clean":"2023-06-10T21:41:14.701469+0000","last_became_active":"2023-06-10T21:40:30.243087+0000","last_became_peered":"2023-06-10T21:40:30.243087+0000","last_unstale":"2023-06-10T21:41:14.701469+0000","last_undegraded":"2023-06-10T21:41:14.701469+0000","last_fullsized":"2023-06-10T21:41:14.701469+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:11.139462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":301,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701197+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:41:14.701197+0000","last_peered":"2023-06-10T21:41:14.701197+0000","last_clean":"2023-06-10T21:41:14.701197+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:41:14.701197+0000","last_undegraded":"2023-06-10T21:41:14.701197+0000","last_fullsized":"2023-06-10T21:41:14.701197+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":241,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703643+0000","last_change":"2023-06-10T21:40:30.287586+0000","last_active":"2023-06-10T21:41:14.703643+0000","last_peered":"2023-06-10T21:41:14.703643+0000","last_clean":"2023-06-10T21:41:14.703643+0000","last_became_active":"2023-06-10T21:40:30.286423+0000","last_became_peered":"2023-06-10T21:40:30.286423+0000","last_unstale":"2023-06-10T21:41:14.703643+0000","last_undegraded":"2023-06-10T21:41:14.703643+0000","last_fullsized":"2023-06-10T21:41:14.703643+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700997+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:41:14.700997+0000","last_peered":"2023-06-10T21:41:14.700997+0000","last_clean":"2023-06-10T21:41:14.700997+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:41:14.700997+0000","last_undegraded":"2023-06-10T21:41:14.700997+0000","last_fullsized":"2023-06-10T21:41:14.700997+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.918627+0000","last_change":"2023-06-10T21:40:30.236647+0000","last_active":"2023-06-10T21:41:14.918627+0000","last_peered":"2023-06-10T21:41:14.918627+0000","last_clean":"2023-06-10T21:41:14.918627+0000","last_became_active":"2023-06-10T21:40:30.236307+0000","last_became_peered":"2023-06-10T21:40:30.236307+0000","last_unstale":"2023-06-10T21:41:14.918627+0000","last_undegraded":"2023-06-10T21:41:14.918627+0000","last_fullsized":"2023-06-10T21:41:14.918627+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":255,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.918508+0000","last_change":"2023-06-10T21:40:30.280815+0000","last_active":"2023-06-10T21:41:14.918508+0000","last_peered":"2023-06-10T21:41:14.918508+0000","last_clean":"2023-06-10T21:41:14.918508+0000","last_became_active":"2023-06-10T21:40:30.280427+0000","last_became_peered":"2023-06-10T21:40:30.280427+0000","last_unstale":"2023-06-10T21:41:14.918508+0000","last_undegraded":"2023-06-10T21:41:14.918508+0000","last_fullsized":"2023-06-10T21:41:14.918508+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701205+0000","last_change":"2023-06-10T21:40:30.245395+0000","last_active":"2023-06-10T21:41:14.701205+0000","last_peered":"2023-06-10T21:41:14.701205+0000","last_clean":"2023-06-10T21:41:14.701205+0000","last_became_active":"2023-06-10T21:40:30.244982+0000","last_became_peered":"2023-06-10T21:40:30.244982+0000","last_unstale":"2023-06-10T21:41:14.701205+0000","last_undegraded":"2023-06-10T21:41:14.701205+0000","last_fullsized":"2023-06-10T21:41:14.701205+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:27:28.313757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":63,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.918084+0000","last_change":"2023-06-10T21:40:30.280871+0000","last_active":"2023-06-10T21:41:14.918084+0000","last_peered":"2023-06-10T21:41:14.918084+0000","last_clean":"2023-06-10T21:41:14.918084+0000","last_became_active":"2023-06-10T21:40:30.280332+0000","last_became_peered":"2023-06-10T21:40:30.280332+0000","last_unstale":"2023-06-10T21:41:14.918084+0000","last_undegraded":"2023-06-10T21:41:14.918084+0000","last_fullsized":"2023-06-10T21:41:14.918084+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":105,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701134+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:41:14.701134+0000","last_peered":"2023-06-10T21:41:14.701134+0000","last_clean":"2023-06-10T21:41:14.701134+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:41:14.701134+0000","last_undegraded":"2023-06-10T21:41:14.701134+0000","last_fullsized":"2023-06-10T21:41:14.701134+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":128,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703167+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:41:14.703167+0000","last_peered":"2023-06-10T21:41:14.703167+0000","last_clean":"2023-06-10T21:41:14.703167+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:41:14.703167+0000","last_undegraded":"2023-06-10T21:41:14.703167+0000","last_fullsized":"2023-06-10T21:41:14.703167+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":355,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.790679+0000","last_change":"2023-06-10T21:39:50.557388+0000","last_active":"2023-06-10T21:41:15.790679+0000","last_peered":"2023-06-10T21:41:15.790679+0000","last_clean":"2023-06-10T21:41:15.790679+0000","last_became_active":"2023-06-10T21:39:50.556659+0000","last_became_peered":"2023-06-10T21:39:50.556659+0000","last_unstale":"2023-06-10T21:41:15.790679+0000","last_undegraded":"2023-06-10T21:41:15.790679+0000","last_fullsized":"2023-06-10T21:41:15.790679+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:32.982830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":279,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703220+0000","last_change":"2023-06-10T21:39:30.845741+0000","last_active":"2023-06-10T21:41:14.703220+0000","last_peered":"2023-06-10T21:41:14.703220+0000","last_clean":"2023-06-10T21:41:14.703220+0000","last_became_active":"2023-06-10T21:39:30.234598+0000","last_became_peered":"2023-06-10T21:39:30.234598+0000","last_unstale":"2023-06-10T21:41:14.703220+0000","last_undegraded":"2023-06-10T21:41:14.703220+0000","last_fullsized":"2023-06-10T21:41:14.703220+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917575+0000","last_change":"2023-06-10T21:39:50.270986+0000","last_active":"2023-06-10T21:41:14.917575+0000","last_peered":"2023-06-10T21:41:14.917575+0000","last_clean":"2023-06-10T21:41:14.917575+0000","last_became_active":"2023-06-10T21:39:50.269843+0000","last_became_peered":"2023-06-10T21:39:50.269843+0000","last_unstale":"2023-06-10T21:41:14.917575+0000","last_undegraded":"2023-06-10T21:41:14.917575+0000","last_fullsized":"2023-06-10T21:41:14.917575+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":415,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702701+0000","last_change":"2023-06-10T21:39:50.781250+0000","last_active":"2023-06-10T21:41:14.702701+0000","last_peered":"2023-06-10T21:41:14.702701+0000","last_clean":"2023-06-10T21:41:14.702701+0000","last_became_active":"2023-06-10T21:39:50.376925+0000","last_became_peered":"2023-06-10T21:39:50.376925+0000","last_unstale":"2023-06-10T21:41:14.702701+0000","last_undegraded":"2023-06-10T21:41:14.702701+0000","last_fullsized":"2023-06-10T21:41:14.702701+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":317,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702347+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:41:14.702347+0000","last_peered":"2023-06-10T21:41:14.702347+0000","last_clean":"2023-06-10T21:41:14.702347+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:41:14.702347+0000","last_undegraded":"2023-06-10T21:41:14.702347+0000","last_fullsized":"2023-06-10T21:41:14.702347+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":241,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703256+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:41:14.703256+0000","last_peered":"2023-06-10T21:41:14.703256+0000","last_clean":"2023-06-10T21:41:14.703256+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:41:14.703256+0000","last_undegraded":"2023-06-10T21:41:14.703256+0000","last_fullsized":"2023-06-10T21:41:14.703256+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":72,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917897+0000","last_change":"2023-06-10T21:40:30.280758+0000","last_active":"2023-06-10T21:41:14.917897+0000","last_peered":"2023-06-10T21:41:14.917897+0000","last_clean":"2023-06-10T21:41:14.917897+0000","last_became_active":"2023-06-10T21:40:30.280270+0000","last_became_peered":"2023-06-10T21:40:30.280270+0000","last_unstale":"2023-06-10T21:41:14.917897+0000","last_undegraded":"2023-06-10T21:41:14.917897+0000","last_fullsized":"2023-06-10T21:41:14.917897+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":383,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703097+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:41:14.703097+0000","last_peered":"2023-06-10T21:41:14.703097+0000","last_clean":"2023-06-10T21:41:14.703097+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:41:14.703097+0000","last_undegraded":"2023-06-10T21:41:14.703097+0000","last_fullsized":"2023-06-10T21:41:14.703097+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699734+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:41:14.699734+0000","last_peered":"2023-06-10T21:41:14.699734+0000","last_clean":"2023-06-10T21:41:14.699734+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:41:14.699734+0000","last_undegraded":"2023-06-10T21:41:14.699734+0000","last_fullsized":"2023-06-10T21:41:14.699734+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":414,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.916527+0000","last_change":"2023-06-10T21:40:30.476009+0000","last_active":"2023-06-10T21:41:14.916527+0000","last_peered":"2023-06-10T21:41:14.916527+0000","last_clean":"2023-06-10T21:41:14.916527+0000","last_became_active":"2023-06-10T21:40:30.237078+0000","last_became_peered":"2023-06-10T21:40:30.237078+0000","last_unstale":"2023-06-10T21:41:14.916527+0000","last_undegraded":"2023-06-10T21:41:14.916527+0000","last_fullsized":"2023-06-10T21:41:14.916527+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700784+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:41:14.700784+0000","last_peered":"2023-06-10T21:41:14.700784+0000","last_clean":"2023-06-10T21:41:14.700784+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:41:14.700784+0000","last_undegraded":"2023-06-10T21:41:14.700784+0000","last_fullsized":"2023-06-10T21:41:14.700784+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":148,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702868+0000","last_change":"2023-06-10T21:40:30.567200+0000","last_active":"2023-06-10T21:41:14.702868+0000","last_peered":"2023-06-10T21:41:14.702868+0000","last_clean":"2023-06-10T21:41:14.702868+0000","last_became_active":"2023-06-10T21:40:30.285607+0000","last_became_peered":"2023-06-10T21:40:30.285607+0000","last_unstale":"2023-06-10T21:41:14.702868+0000","last_undegraded":"2023-06-10T21:41:14.702868+0000","last_fullsized":"2023-06-10T21:41:14.702868+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":63,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.916549+0000","last_change":"2023-06-10T21:40:30.232813+0000","last_active":"2023-06-10T21:41:14.916549+0000","last_peered":"2023-06-10T21:41:14.916549+0000","last_clean":"2023-06-10T21:41:14.916549+0000","last_became_active":"2023-06-10T21:40:30.232432+0000","last_became_peered":"2023-06-10T21:40:30.232432+0000","last_unstale":"2023-06-10T21:41:14.916549+0000","last_undegraded":"2023-06-10T21:41:14.916549+0000","last_fullsized":"2023-06-10T21:41:14.916549+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":148,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703766+0000","last_change":"2023-06-10T21:40:30.576549+0000","last_active":"2023-06-10T21:41:14.703766+0000","last_peered":"2023-06-10T21:41:14.703766+0000","last_clean":"2023-06-10T21:41:14.703766+0000","last_became_active":"2023-06-10T21:40:30.286869+0000","last_became_peered":"2023-06-10T21:40:30.286869+0000","last_unstale":"2023-06-10T21:41:14.703766+0000","last_undegraded":"2023-06-10T21:41:14.703766+0000","last_fullsized":"2023-06-10T21:41:14.703766+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701247+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:41:14.701247+0000","last_peered":"2023-06-10T21:41:14.701247+0000","last_clean":"2023-06-10T21:41:14.701247+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:41:14.701247+0000","last_undegraded":"2023-06-10T21:41:14.701247+0000","last_fullsized":"2023-06-10T21:41:14.701247+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":435,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702467+0000","last_change":"2023-06-10T21:40:30.521312+0000","last_active":"2023-06-10T21:41:14.702467+0000","last_peered":"2023-06-10T21:41:14.702467+0000","last_clean":"2023-06-10T21:41:14.702467+0000","last_became_active":"2023-06-10T21:40:30.285713+0000","last_became_peered":"2023-06-10T21:40:30.285713+0000","last_unstale":"2023-06-10T21:41:14.702467+0000","last_undegraded":"2023-06-10T21:41:14.702467+0000","last_fullsized":"2023-06-10T21:41:14.702467+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702121+0000","last_change":"2023-06-10T21:40:30.343628+0000","last_active":"2023-06-10T21:41:14.702121+0000","last_peered":"2023-06-10T21:41:14.702121+0000","last_clean":"2023-06-10T21:41:14.702121+0000","last_became_active":"2023-06-10T21:40:30.343292+0000","last_became_peered":"2023-06-10T21:40:30.343292+0000","last_unstale":"2023-06-10T21:41:14.702121+0000","last_undegraded":"2023-06-10T21:41:14.702121+0000","last_fullsized":"2023-06-10T21:41:14.702121+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":73,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917595+0000","last_change":"2023-06-10T21:40:30.280834+0000","last_active":"2023-06-10T21:41:14.917595+0000","last_peered":"2023-06-10T21:41:14.917595+0000","last_clean":"2023-06-10T21:41:14.917595+0000","last_became_active":"2023-06-10T21:40:30.280353+0000","last_became_peered":"2023-06-10T21:40:30.280353+0000","last_unstale":"2023-06-10T21:41:14.917595+0000","last_undegraded":"2023-06-10T21:41:14.917595+0000","last_fullsized":"2023-06-10T21:41:14.917595+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700632+0000","last_change":"2023-06-10T21:40:30.272439+0000","last_active":"2023-06-10T21:41:14.700632+0000","last_peered":"2023-06-10T21:41:14.700632+0000","last_clean":"2023-06-10T21:41:14.700632+0000","last_became_active":"2023-06-10T21:40:30.272004+0000","last_became_peered":"2023-06-10T21:40:30.272004+0000","last_unstale":"2023-06-10T21:41:14.700632+0000","last_undegraded":"2023-06-10T21:41:14.700632+0000","last_fullsized":"2023-06-10T21:41:14.700632+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:30.425736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":27,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700545+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:41:14.700545+0000","last_peered":"2023-06-10T21:41:14.700545+0000","last_clean":"2023-06-10T21:41:14.700545+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:41:14.700545+0000","last_undegraded":"2023-06-10T21:41:14.700545+0000","last_fullsized":"2023-06-10T21:41:14.700545+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702272+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:41:14.702272+0000","last_peered":"2023-06-10T21:41:14.702272+0000","last_clean":"2023-06-10T21:41:14.702272+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:41:14.702272+0000","last_undegraded":"2023-06-10T21:41:14.702272+0000","last_fullsized":"2023-06-10T21:41:14.702272+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":175,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699491+0000","last_change":"2023-06-10T21:39:50.465026+0000","last_active":"2023-06-10T21:41:14.699491+0000","last_peered":"2023-06-10T21:41:14.699491+0000","last_clean":"2023-06-10T21:41:14.699491+0000","last_became_active":"2023-06-10T21:39:50.464762+0000","last_became_peered":"2023-06-10T21:39:50.464762+0000","last_unstale":"2023-06-10T21:41:14.699491+0000","last_undegraded":"2023-06-10T21:41:14.699491+0000","last_fullsized":"2023-06-10T21:41:14.699491+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:55:00.850195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":393,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.790589+0000","last_change":"2023-06-10T21:39:50.259203+0000","last_active":"2023-06-10T21:41:15.790589+0000","last_peered":"2023-06-10T21:41:15.790589+0000","last_clean":"2023-06-10T21:41:15.790589+0000","last_became_active":"2023-06-10T21:39:50.258873+0000","last_became_peered":"2023-06-10T21:39:50.258873+0000","last_unstale":"2023-06-10T21:41:15.790589+0000","last_undegraded":"2023-06-10T21:41:15.790589+0000","last_fullsized":"2023-06-10T21:41:15.790589+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:56:41.490096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":371,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.800455+0000","last_change":"2023-06-10T21:39:50.432352+0000","last_active":"2023-06-10T21:41:15.800455+0000","last_peered":"2023-06-10T21:41:15.800455+0000","last_clean":"2023-06-10T21:41:15.800455+0000","last_became_active":"2023-06-10T21:39:50.429920+0000","last_became_peered":"2023-06-10T21:39:50.429920+0000","last_unstale":"2023-06-10T21:41:15.800455+0000","last_undegraded":"2023-06-10T21:41:15.800455+0000","last_fullsized":"2023-06-10T21:41:15.800455+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:14.733503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699636+0000","last_change":"2023-06-10T21:40:30.351796+0000","last_active":"2023-06-10T21:41:14.699636+0000","last_peered":"2023-06-10T21:41:14.699636+0000","last_clean":"2023-06-10T21:41:14.699636+0000","last_became_active":"2023-06-10T21:40:30.350450+0000","last_became_peered":"2023-06-10T21:40:30.350450+0000","last_unstale":"2023-06-10T21:41:14.699636+0000","last_undegraded":"2023-06-10T21:41:14.699636+0000","last_fullsized":"2023-06-10T21:41:14.699636+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:38.306536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702806+0000","last_change":"2023-06-10T21:40:30.279010+0000","last_active":"2023-06-10T21:41:14.702806+0000","last_peered":"2023-06-10T21:41:14.702806+0000","last_clean":"2023-06-10T21:41:14.702806+0000","last_became_active":"2023-06-10T21:40:30.277943+0000","last_became_peered":"2023-06-10T21:40:30.277943+0000","last_unstale":"2023-06-10T21:41:14.702806+0000","last_undegraded":"2023-06-10T21:41:14.702806+0000","last_fullsized":"2023-06-10T21:41:14.702806+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:38:15.175343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":302,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701514+0000","last_change":"2023-06-10T21:39:50.233694+0000","last_active":"2023-06-10T21:41:14.701514+0000","last_peered":"2023-06-10T21:41:14.701514+0000","last_clean":"2023-06-10T21:41:14.701514+0000","last_became_active":"2023-06-10T21:39:50.233429+0000","last_became_peered":"2023-06-10T21:39:50.233429+0000","last_unstale":"2023-06-10T21:41:14.701514+0000","last_undegraded":"2023-06-10T21:41:14.701514+0000","last_fullsized":"2023-06-10T21:41:14.701514+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:15:42.432299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":154,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.703303+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:41:14.703303+0000","last_peered":"2023-06-10T21:41:14.703303+0000","last_clean":"2023-06-10T21:41:14.703303+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:41:14.703303+0000","last_undegraded":"2023-06-10T21:41:14.703303+0000","last_fullsized":"2023-06-10T21:41:14.703303+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":54,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917819+0000","last_change":"2023-06-10T21:40:30.281222+0000","last_active":"2023-06-10T21:41:14.917819+0000","last_peered":"2023-06-10T21:41:14.917819+0000","last_clean":"2023-06-10T21:41:14.917819+0000","last_became_active":"2023-06-10T21:40:30.280846+0000","last_became_peered":"2023-06-10T21:40:30.280846+0000","last_unstale":"2023-06-10T21:41:14.917819+0000","last_undegraded":"2023-06-10T21:41:14.917819+0000","last_fullsized":"2023-06-10T21:41:14.917819+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917164+0000","last_change":"2023-06-10T21:39:50.466302+0000","last_active":"2023-06-10T21:41:14.917164+0000","last_peered":"2023-06-10T21:41:14.917164+0000","last_clean":"2023-06-10T21:41:14.917164+0000","last_became_active":"2023-06-10T21:39:50.465409+0000","last_became_peered":"2023-06-10T21:39:50.465409+0000","last_unstale":"2023-06-10T21:41:14.917164+0000","last_undegraded":"2023-06-10T21:41:14.917164+0000","last_fullsized":"2023-06-10T21:41:14.917164+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":73,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.916925+0000","last_change":"2023-06-10T21:40:30.236317+0000","last_active":"2023-06-10T21:41:14.916925+0000","last_peered":"2023-06-10T21:41:14.916925+0000","last_clean":"2023-06-10T21:41:14.916925+0000","last_became_active":"2023-06-10T21:40:30.235998+0000","last_became_peered":"2023-06-10T21:40:30.235998+0000","last_unstale":"2023-06-10T21:41:14.916925+0000","last_undegraded":"2023-06-10T21:41:14.916925+0000","last_fullsized":"2023-06-10T21:41:14.916925+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":351,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917653+0000","last_change":"2023-06-10T21:40:30.281401+0000","last_active":"2023-06-10T21:41:14.917653+0000","last_peered":"2023-06-10T21:41:14.917653+0000","last_clean":"2023-06-10T21:41:14.917653+0000","last_became_active":"2023-06-10T21:40:30.280966+0000","last_became_peered":"2023-06-10T21:40:30.280966+0000","last_unstale":"2023-06-10T21:41:14.917653+0000","last_undegraded":"2023-06-10T21:41:14.917653+0000","last_fullsized":"2023-06-10T21:41:14.917653+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":235,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.806508+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:41:15.806508+0000","last_peered":"2023-06-10T21:41:15.806508+0000","last_clean":"2023-06-10T21:41:15.806508+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:41:15.806508+0000","last_undegraded":"2023-06-10T21:41:15.806508+0000","last_fullsized":"2023-06-10T21:41:15.806508+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":255,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.800376+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:41:15.800376+0000","last_peered":"2023-06-10T21:41:15.800376+0000","last_clean":"2023-06-10T21:41:15.800376+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:41:15.800376+0000","last_undegraded":"2023-06-10T21:41:15.800376+0000","last_fullsized":"2023-06-10T21:41:15.800376+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":244,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.789528+0000","last_change":"2023-06-10T21:40:30.229244+0000","last_active":"2023-06-10T21:41:15.789528+0000","last_peered":"2023-06-10T21:41:15.789528+0000","last_clean":"2023-06-10T21:41:15.789528+0000","last_became_active":"2023-06-10T21:40:30.228964+0000","last_became_peered":"2023-06-10T21:40:30.228964+0000","last_unstale":"2023-06-10T21:41:15.789528+0000","last_undegraded":"2023-06-10T21:41:15.789528+0000","last_fullsized":"2023-06-10T21:41:15.789528+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:41.629358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":72,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.918549+0000","last_change":"2023-06-10T21:40:30.236712+0000","last_active":"2023-06-10T21:41:14.918549+0000","last_peered":"2023-06-10T21:41:14.918549+0000","last_clean":"2023-06-10T21:41:14.918549+0000","last_became_active":"2023-06-10T21:40:30.236404+0000","last_became_peered":"2023-06-10T21:40:30.236404+0000","last_unstale":"2023-06-10T21:41:14.918549+0000","last_undegraded":"2023-06-10T21:41:14.918549+0000","last_fullsized":"2023-06-10T21:41:14.918549+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":45,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917341+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:41:14.917341+0000","last_peered":"2023-06-10T21:41:14.917341+0000","last_clean":"2023-06-10T21:41:14.917341+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:41:14.917341+0000","last_undegraded":"2023-06-10T21:41:14.917341+0000","last_fullsized":"2023-06-10T21:41:14.917341+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":571,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702503+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:41:14.702503+0000","last_peered":"2023-06-10T21:41:14.702503+0000","last_clean":"2023-06-10T21:41:14.702503+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:41:14.702503+0000","last_undegraded":"2023-06-10T21:41:14.702503+0000","last_fullsized":"2023-06-10T21:41:14.702503+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699287+0000","last_change":"2023-06-10T21:40:30.260288+0000","last_active":"2023-06-10T21:41:14.699287+0000","last_peered":"2023-06-10T21:41:14.699287+0000","last_clean":"2023-06-10T21:41:14.699287+0000","last_became_active":"2023-06-10T21:40:30.259958+0000","last_became_peered":"2023-06-10T21:40:30.259958+0000","last_unstale":"2023-06-10T21:41:14.699287+0000","last_undegraded":"2023-06-10T21:41:14.699287+0000","last_fullsized":"2023-06-10T21:41:14.699287+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:41.077016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":239,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702955+0000","last_change":"2023-06-10T21:40:30.286026+0000","last_active":"2023-06-10T21:41:14.702955+0000","last_peered":"2023-06-10T21:41:14.702955+0000","last_clean":"2023-06-10T21:41:14.702955+0000","last_became_active":"2023-06-10T21:40:30.285670+0000","last_became_peered":"2023-06-10T21:40:30.285670+0000","last_unstale":"2023-06-10T21:41:14.702955+0000","last_undegraded":"2023-06-10T21:41:14.702955+0000","last_fullsized":"2023-06-10T21:41:14.702955+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":408,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702523+0000","last_change":"2023-06-10T21:39:50.237017+0000","last_active":"2023-06-10T21:41:14.702523+0000","last_peered":"2023-06-10T21:41:14.702523+0000","last_clean":"2023-06-10T21:41:14.702523+0000","last_became_active":"2023-06-10T21:39:50.232270+0000","last_became_peered":"2023-06-10T21:39:50.232270+0000","last_unstale":"2023-06-10T21:41:14.702523+0000","last_undegraded":"2023-06-10T21:41:14.702523+0000","last_fullsized":"2023-06-10T21:41:14.702523+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700883+0000","last_change":"2023-06-10T21:40:30.316582+0000","last_active":"2023-06-10T21:41:14.700883+0000","last_peered":"2023-06-10T21:41:14.700883+0000","last_clean":"2023-06-10T21:41:14.700883+0000","last_became_active":"2023-06-10T21:40:30.315657+0000","last_became_peered":"2023-06-10T21:40:30.315657+0000","last_unstale":"2023-06-10T21:41:14.700883+0000","last_undegraded":"2023-06-10T21:41:14.700883+0000","last_fullsized":"2023-06-10T21:41:14.700883+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:08:04.756800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":106,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.793677+0000","last_change":"2023-06-10T21:39:50.559093+0000","last_active":"2023-06-10T21:41:15.793677+0000","last_peered":"2023-06-10T21:41:15.793677+0000","last_clean":"2023-06-10T21:41:15.793677+0000","last_became_active":"2023-06-10T21:39:50.558289+0000","last_became_peered":"2023-06-10T21:39:50.558289+0000","last_unstale":"2023-06-10T21:41:15.793677+0000","last_undegraded":"2023-06-10T21:41:15.793677+0000","last_fullsized":"2023-06-10T21:41:15.793677+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:37.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":402,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702575+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:41:14.702575+0000","last_peered":"2023-06-10T21:41:14.702575+0000","last_clean":"2023-06-10T21:41:14.702575+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:41:14.702575+0000","last_undegraded":"2023-06-10T21:41:14.702575+0000","last_fullsized":"2023-06-10T21:41:14.702575+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":235,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.807023+0000","last_change":"2023-06-10T21:39:50.296322+0000","last_active":"2023-06-10T21:41:15.807023+0000","last_peered":"2023-06-10T21:41:15.807023+0000","last_clean":"2023-06-10T21:41:15.807023+0000","last_became_active":"2023-06-10T21:39:50.295201+0000","last_became_peered":"2023-06-10T21:39:50.295201+0000","last_unstale":"2023-06-10T21:41:15.807023+0000","last_undegraded":"2023-06-10T21:41:15.807023+0000","last_fullsized":"2023-06-10T21:41:15.807023+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:28.403434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":200,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917747+0000","last_change":"2023-06-10T21:39:50.271175+0000","last_active":"2023-06-10T21:41:14.917747+0000","last_peered":"2023-06-10T21:41:14.917747+0000","last_clean":"2023-06-10T21:41:14.917747+0000","last_became_active":"2023-06-10T21:39:50.270302+0000","last_became_peered":"2023-06-10T21:39:50.270302+0000","last_unstale":"2023-06-10T21:41:14.917747+0000","last_undegraded":"2023-06-10T21:41:14.917747+0000","last_fullsized":"2023-06-10T21:41:14.917747+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":364,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700882+0000","last_change":"2023-06-10T21:39:30.266087+0000","last_active":"2023-06-10T21:41:14.700882+0000","last_peered":"2023-06-10T21:41:14.700882+0000","last_clean":"2023-06-10T21:41:14.700882+0000","last_became_active":"2023-06-10T21:39:30.231858+0000","last_became_peered":"2023-06-10T21:39:30.231858+0000","last_unstale":"2023-06-10T21:41:14.700882+0000","last_undegraded":"2023-06-10T21:41:14.700882+0000","last_fullsized":"2023-06-10T21:41:14.700882+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:08:09.982313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":175,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.700029+0000","last_change":"2023-06-10T21:39:50.227162+0000","last_active":"2023-06-10T21:41:14.700029+0000","last_peered":"2023-06-10T21:41:14.700029+0000","last_clean":"2023-06-10T21:41:14.700029+0000","last_became_active":"2023-06-10T21:39:50.226891+0000","last_became_peered":"2023-06-10T21:39:50.226891+0000","last_unstale":"2023-06-10T21:41:14.700029+0000","last_undegraded":"2023-06-10T21:41:14.700029+0000","last_fullsized":"2023-06-10T21:41:14.700029+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:48:59.680224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702156+0000","last_change":"2023-06-10T21:40:30.279185+0000","last_active":"2023-06-10T21:41:14.702156+0000","last_peered":"2023-06-10T21:41:14.702156+0000","last_clean":"2023-06-10T21:41:14.702156+0000","last_became_active":"2023-06-10T21:40:30.278330+0000","last_became_peered":"2023-06-10T21:40:30.278330+0000","last_unstale":"2023-06-10T21:41:14.702156+0000","last_undegraded":"2023-06-10T21:41:14.702156+0000","last_fullsized":"2023-06-10T21:41:14.702156+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:15.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":119,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702911+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:41:14.702911+0000","last_peered":"2023-06-10T21:41:14.702911+0000","last_clean":"2023-06-10T21:41:14.702911+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:41:14.702911+0000","last_undegraded":"2023-06-10T21:41:14.702911+0000","last_fullsized":"2023-06-10T21:41:14.702911+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":218,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702221+0000","last_change":"2023-06-10T21:40:30.580293+0000","last_active":"2023-06-10T21:41:14.702221+0000","last_peered":"2023-06-10T21:41:14.702221+0000","last_clean":"2023-06-10T21:41:14.702221+0000","last_became_active":"2023-06-10T21:40:30.391049+0000","last_became_peered":"2023-06-10T21:40:30.391049+0000","last_unstale":"2023-06-10T21:41:14.702221+0000","last_undegraded":"2023-06-10T21:41:14.702221+0000","last_fullsized":"2023-06-10T21:41:14.702221+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.916459+0000","last_change":"2023-06-10T21:39:30.208137+0000","last_active":"2023-06-10T21:41:14.916459+0000","last_peered":"2023-06-10T21:41:14.916459+0000","last_clean":"2023-06-10T21:41:14.916459+0000","last_became_active":"2023-06-10T21:39:30.207848+0000","last_became_peered":"2023-06-10T21:39:30.207848+0000","last_unstale":"2023-06-10T21:41:14.916459+0000","last_undegraded":"2023-06-10T21:41:14.916459+0000","last_fullsized":"2023-06-10T21:41:14.916459+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":70,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702897+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:41:14.702897+0000","last_peered":"2023-06-10T21:41:14.702897+0000","last_clean":"2023-06-10T21:41:14.702897+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:41:14.702897+0000","last_undegraded":"2023-06-10T21:41:14.702897+0000","last_fullsized":"2023-06-10T21:41:14.702897+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":252,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-06-10T21:41:15.807102+0000","last_change":"2023-06-10T21:40:30.227732+0000","last_active":"2023-06-10T21:41:15.807102+0000","last_peered":"2023-06-10T21:41:15.807102+0000","last_clean":"2023-06-10T21:41:15.807102+0000","last_became_active":"2023-06-10T21:40:30.227380+0000","last_became_peered":"2023-06-10T21:40:30.227380+0000","last_unstale":"2023-06-10T21:41:15.807102+0000","last_undegraded":"2023-06-10T21:41:15.807102+0000","last_fullsized":"2023-06-10T21:41:15.807102+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:33.673485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":58,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917631+0000","last_change":"2023-06-10T21:39:50.304413+0000","last_active":"2023-06-10T21:41:14.917631+0000","last_peered":"2023-06-10T21:41:14.917631+0000","last_clean":"2023-06-10T21:41:14.917631+0000","last_became_active":"2023-06-10T21:39:50.303414+0000","last_became_peered":"2023-06-10T21:39:50.303414+0000","last_unstale":"2023-06-10T21:41:14.917631+0000","last_undegraded":"2023-06-10T21:41:14.917631+0000","last_fullsized":"2023-06-10T21:41:14.917631+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":273,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701275+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:41:14.701275+0000","last_peered":"2023-06-10T21:41:14.701275+0000","last_clean":"2023-06-10T21:41:14.701275+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:41:14.701275+0000","last_undegraded":"2023-06-10T21:41:14.701275+0000","last_fullsized":"2023-06-10T21:41:14.701275+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":75,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.917912+0000","last_change":"2023-06-10T21:40:30.565123+0000","last_active":"2023-06-10T21:41:14.917912+0000","last_peered":"2023-06-10T21:41:14.917912+0000","last_clean":"2023-06-10T21:41:14.917912+0000","last_became_active":"2023-06-10T21:40:30.280290+0000","last_became_peered":"2023-06-10T21:40:30.280290+0000","last_unstale":"2023-06-10T21:41:14.917912+0000","last_undegraded":"2023-06-10T21:41:14.917912+0000","last_fullsized":"2023-06-10T21:41:14.917912+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":329,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699587+0000","last_change":"2023-06-10T21:39:30.245702+0000","last_active":"2023-06-10T21:41:14.699587+0000","last_peered":"2023-06-10T21:41:14.699587+0000","last_clean":"2023-06-10T21:41:14.699587+0000","last_became_active":"2023-06-10T21:39:30.245426+0000","last_became_peered":"2023-06-10T21:39:30.245426+0000","last_unstale":"2023-06-10T21:41:14.699587+0000","last_undegraded":"2023-06-10T21:41:14.699587+0000","last_fullsized":"2023-06-10T21:41:14.699587+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:52:27.316156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.699826+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:41:14.699826+0000","last_peered":"2023-06-10T21:41:14.699826+0000","last_clean":"2023-06-10T21:41:14.699826+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:41:14.699826+0000","last_undegraded":"2023-06-10T21:41:14.699826+0000","last_fullsized":"2023-06-10T21:41:14.699826+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.701695+0000","last_change":"2023-06-10T21:40:30.253755+0000","last_active":"2023-06-10T21:41:14.701695+0000","last_peered":"2023-06-10T21:41:14.701695+0000","last_clean":"2023-06-10T21:41:14.701695+0000","last_became_active":"2023-06-10T21:40:30.253530+0000","last_became_peered":"2023-06-10T21:40:30.253530+0000","last_unstale":"2023-06-10T21:41:14.701695+0000","last_undegraded":"2023-06-10T21:41:14.701695+0000","last_fullsized":"2023-06-10T21:41:14.701695+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:14.839319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":311,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.918426+0000","last_change":"2023-06-10T21:40:30.574471+0000","last_active":"2023-06-10T21:41:14.918426+0000","last_peered":"2023-06-10T21:41:14.918426+0000","last_clean":"2023-06-10T21:41:14.918426+0000","last_became_active":"2023-06-10T21:40:30.281351+0000","last_became_peered":"2023-06-10T21:40:30.281351+0000","last_unstale":"2023-06-10T21:41:14.918426+0000","last_undegraded":"2023-06-10T21:41:14.918426+0000","last_fullsized":"2023-06-10T21:41:14.918426+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":370,"state":"active+clean","last_fresh":"2023-06-10T21:41:14.702880+0000","last_change":"2023-06-10T21:40:30.245261+0000","last_active":"2023-06-10T21:41:14.702880+0000","last_peered":"2023-06-10T21:41:14.702880+0000","last_clean":"2023-06-10T21:41:14.702880+0000","last_became_active":"2023-06-10T21:40:30.244695+0000","last_became_peered":"2023-06-10T21:40:30.244695+0000","last_unstale":"2023-06-10T21:41:14.702880+0000","last_undegraded":"2023-06-10T21:41:14.702880+0000","last_fullsized":"2023-06-10T21:41:14.702880+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:54.784460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":18486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430731,"num_pgs":84,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815364,"kb_used_data":3900,"kb_used_omap":0,"kb_used_meta":811456,"kb_avail":92929788,"statfs":{"total":95995035648,"available":95160102912,"internally_reserved":0,"allocated":3993600,"data_stored":3300880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":830930944},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.986,"15min":4.8600000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":22.277000000000001,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":2.3889999999999998,"5min":2.4079999999999999,"15min":5.165},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.223},"max":{"1min":22.462,"5min":56.628,"15min":185.58000000000001},"last":0.432}]},{"osd":1,"last update":"Sat Jun 10 21:40:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":3.085,"15min":2.8380000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":8.4410000000000007,"5min":60.863,"15min":93.093999999999994},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.6619999999999999,"5min":3.2669999999999999,"15min":2.9060000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":14.161,"5min":61.148000000000003,"15min":93.119},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:40:35 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":4.5250000000000004,"15min":5.4109999999999996},"min":{"1min":0.33500000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.6260000000000003,"5min":137.64099999999999,"15min":172.33600000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.794,"5min":4.2199999999999998,"15min":5.3440000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29499999999999998},"max":{"1min":5.8029999999999999,"5min":136.96600000000001,"15min":172.185},"last":0.505}]},{"osd":3,"last update":"Sat Jun 10 21:40:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3530000000000002,"5min":6.0519999999999996,"15min":4.2560000000000002},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.27000000000000002},"max":{"1min":8.5670000000000002,"5min":147.00700000000001,"15min":147.00700000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.895,"5min":5.9509999999999996,"15min":3.8889999999999998},"min":{"1min":0.35899999999999999,"5min":0.308,"15min":0.308},"max":{"1min":5.7560000000000002,"5min":146.93899999999999,"15min":146.93899999999999},"last":0.54000000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.249},"max":{"1min":1.0289999999999999,"5min":5.859,"15min":5.859},"last":0.441},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.81399999999999995,"15min":0.66700000000000004},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.3290000000000002,"5min":22.77,"15min":22.77},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48399999999999999,"15min":0.64200000000000002},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.8220000000000001,"5min":2.7930000000000001,"15min":12.048},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.97499999999999998,"5min":5.9100000000000001,"15min":6.7530000000000001},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.68300000000000005,"15min":0.72399999999999998},"min":{"1min":0.34699999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":1.847,"5min":10.890000000000001,"15min":13.073},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.59299999999999997,"15min":0.56799999999999995},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0209999999999999,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":2.1280000000000001}]}]},{"osd":6,"up_from":0,"seq":180388626959,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566402,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020378,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075790,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753252,"kb_used_data":4508,"kb_used_omap":7,"kb_used_meta":748728,"kb_avail":92991900,"statfs":{"total":95995035648,"available":95223705600,"internally_reserved":0,"allocated":4616192,"data_stored":3373103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":766697553},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":25,"apply_latency_ms":25,"commit_latency_ns":25000000,"apply_latency_ns":25000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":5.54,"5min":5.9180000000000001,"15min":5.0979999999999999},"min":{"1min":0.19900000000000001,"5min":0.19,"15min":0.19},"max":{"1min":83.733000000000004,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":4.9020000000000001,"5min":5.4669999999999996,"15min":5.173},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":82.322999999999993,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.4720000000000004,"5min":7.6699999999999999,"15min":5.0419999999999998},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.19},"max":{"1min":74.256,"5min":171.286,"15min":171.286},"last":0.26700000000000002},{"interface":"front","average":{"1min":5.6689999999999996,"5min":3.5830000000000002,"15min":3.4420000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.214},"max":{"1min":72.683999999999997,"5min":72.683999999999997,"15min":147.81299999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:41:20 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":5.694,"15min":5.5270000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":70.480999999999995,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":5.4980000000000002,"5min":5.3550000000000004,"15min":5.7530000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":71.144999999999996,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":3.5550000000000002,"15min":3.589},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.3170000000000002,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.766,"5min":5.2670000000000003,"15min":4.4470000000000001},"min":{"1min":0.36899999999999999,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":14.858000000000001,"5min":171.18100000000001,"15min":223.28399999999999},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":3.5470000000000002,"15min":2.274},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":7.3440000000000003,"5min":171.303,"15min":171.303},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.90500000000000003,"5min":3.54,"15min":2.3519999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.359,"5min":171.203,"15min":171.203},"last":0.62}]},{"osd":6,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":3.5830000000000002,"15min":2.4870000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.4169999999999998,"5min":171.441,"15min":171.441},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.9220000000000002,"5min":5.2729999999999997,"15min":3.3340000000000001},"min":{"1min":0.45900000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":14.893000000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":6.7229999999999999,"15min":4.4020000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":7.383,"5min":219.131,"15min":219.131},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.94099999999999995,"5min":6.6310000000000002,"15min":4.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.4489999999999998,"5min":219.19,"15min":219.19},"last":0.57599999999999996}]}]},{"osd":3,"up_from":250,"seq":1073741824149,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1183456,"kb_used_data":673560,"kb_used_omap":7,"kb_used_meta":509880,"kb_avail":92561696,"statfs":{"total":95995035648,"available":94783176704,"internally_reserved":0,"allocated":689725440,"data_stored":688365731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":522117139},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":2.863,"15min":4.4119999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":14.771000000000001,"5min":25.512,"15min":105.09},"last":0.221},{"interface":"front","average":{"1min":1.8049999999999999,"5min":2.8460000000000001,"15min":4.2249999999999996},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.5730000000000004,"5min":25.472999999999999,"15min":99.852999999999994},"last":0.251}]},{"osd":1,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":2.9860000000000002,"15min":3.9399999999999999},"min":{"1min":0.27600000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.249000000000001,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.23599999999999999},{"interface":"front","average":{"1min":3.5209999999999999,"5min":3.6240000000000001,"15min":3.9129999999999998},"min":{"1min":0.29699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":11.846,"5min":41.18,"15min":94.691000000000003},"last":0.28899999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":4.8609999999999998,"5min":4.7050000000000001,"15min":7.2060000000000004},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.452000000000002,"5min":231.07900000000001,"15min":231.07900000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":5.7720000000000002,"5min":6.1849999999999996,"15min":8.7609999999999992},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.22800000000000001},"max":{"1min":18.733000000000001,"5min":231.58600000000001,"15min":231.58600000000001},"last":0.26800000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2729999999999999,"15min":1.407},"min":{"1min":0.433,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":10.368,"15min":13.417},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.9119999999999999,"5min":2.1589999999999998,"15min":2.5739999999999998},"min":{"1min":0.35599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.722,"5min":19.244,"15min":19.728000000000002},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":1.8,"15min":1.583},"min":{"1min":0.41299999999999998,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":9.8680000000000003,"5min":17.298999999999999,"15min":17.298999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.379,"15min":1.506},"min":{"1min":0.434,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":8.5109999999999992,"5min":10.414999999999999,"15min":13.470000000000001},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.2170000000000001,"15min":2.609},"min":{"1min":0.42899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.691000000000001,"5min":19.164999999999999,"15min":19.805},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.5920000000000001,"5min":1.7909999999999999,"15min":1.615},"min":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":9.9719999999999995,"5min":17.327000000000002,"15min":17.327000000000002},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":2.194,"15min":2.8319999999999999},"min":{"1min":0.48299999999999998,"5min":0.379,"15min":0.33900000000000002},"max":{"1min":10.648999999999999,"5min":19.193000000000001,"15min":79.984999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.6349999999999998,"5min":1.897,"15min":1.865},"min":{"1min":0.44800000000000001,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":9.9380000000000006,"5min":17.251000000000001,"15min":80.272999999999996},"last":0.53300000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019415,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823187,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":821180,"kb_used_data":191412,"kb_used_omap":0,"kb_used_meta":629760,"kb_avail":92923972,"statfs":{"total":95995035648,"available":95154147328,"internally_reserved":0,"allocated":196005888,"data_stored":195320965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":644874240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.655,"15min":3.2349999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.4530000000000003,"5min":18.562000000000001,"15min":134.71899999999999},"last":0.504},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.121,"15min":3.4710000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2709999999999999,"5min":27.256,"15min":135.386},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":5.0019999999999998,"5min":2.5059999999999998,"15min":2.6320000000000001},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":60.399999999999999,"5min":60.399999999999999,"15min":71.349000000000004},"last":0.38700000000000001},{"interface":"front","average":{"1min":5.25,"5min":2.319,"15min":2.5899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":59.780999999999999,"5min":59.780999999999999,"15min":71.331999999999994},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":5.9020000000000001,"15min":6.3079999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.789999999999999,"5min":324.21600000000001,"15min":324.21600000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.9670000000000001,"5min":5.54,"15min":6.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.0950000000000006,"5min":324.16300000000001,"15min":324.16300000000001},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:40:23 2023","interfaces":[{"interface":"back","average":{"1min":4.165,"5min":4.4109999999999996,"15min":4.5099999999999998},"min":{"1min":0.41699999999999998,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":13.065,"5min":90.725999999999999,"15min":128.089},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.4430000000000001,"5min":3.9630000000000001,"15min":4.5430000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":8.8219999999999992,"5min":90.180000000000007,"15min":128.24299999999999},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.47899999999999998,"15min":1.135},"min":{"1min":0.27900000000000003,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":5.3460000000000001,"5min":5.3460000000000001,"15min":165.73699999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":1.242},"min":{"1min":0.31,"5min":0.25,"15min":0.25},"max":{"1min":1.266,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.60599999999999998,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.22},"max":{"1min":3.4580000000000002,"5min":5.3739999999999997,"15min":29.760000000000002},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.751,"5min":0.61399999999999999,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.29199999999999998},"max":{"1min":3.4340000000000002,"5min":4.3380000000000001,"15min":28.995999999999999},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:40:23 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.6100000000000001,"15min":1.54},"min":{"1min":0.33400000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":4.3049999999999997,"5min":94.102000000000004,"15min":136.904},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.6599999999999999,"15min":1.5229999999999999},"min":{"1min":0.314,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":5.3179999999999996,"5min":95.653999999999996,"15min":137.661},"last":0.56999999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:41:24.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:41:24.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:24 smithi064 ceph-mon[186706]: from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:24.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:24.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:24 smithi064 ceph-mon[180297]: from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:24.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:24.417 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:24.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:24 smithi152 ceph-mon[156819]: from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:24.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:24.492 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:24.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:24.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:24.598 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:24.601 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:24.623 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:24.633 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:24.649 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:24.676 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:24.701 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:24.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:24.740 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:24.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:24.746 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:24.766 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:24.767 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.767 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.767 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.768 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:41:24.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:24.879 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:24.886 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:24.928 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:24.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:24.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:25.036 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:25.064 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:25.108 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:25.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:25.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:25.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:25.243 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:25.286 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:25.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:25.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:25.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:25 smithi064 ceph-mon[186706]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:25.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1949751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:25.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:25 smithi064 ceph-mon[180297]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:25.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1949751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:25.419 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:25 smithi152 ceph-mon[156819]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:25.419 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1949751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:25.421 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:25.463 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:25.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:25.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:25.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:25.976 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:25.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:26.081 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:26.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:26.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:26.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:26.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:26.396 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:26.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:26.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:26.576 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:26.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:26.650 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:26.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:26.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:26.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:26.795 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:27.168 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_8' created 2023-06-10T21:41:27.187 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:41:27.187 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:41:27.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104494 12.9 12:9ca311c5:::benchmark_data_smithi064_442287_object16444:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.728279144, 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6361784579999998, 2023-06-10T21:41:27.188 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104494 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:41:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177995+0000", 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178006+0000", 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1514e-05 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178024+0000", 2023-06-10T21:41:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7651000000000001e-05 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178035+0000", 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0857e-05 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.188934+0000", 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010898873 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189075+0000", 2023-06-10T21:41:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014062699999999999 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189180+0000", 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105855 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.196756+0000", 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075760269999999999 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207957+0000", 2023-06-10T21:41:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011200676999999999 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245880+0000", 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037923327999999999 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245895+0000", 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4575e-05 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200093+0000", 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95419813399999998 2023-06-10T21:41:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200108+0000", 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5024e-05 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754247+0000", 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5541385060000001 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754441+0000", 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019410900000000001 2023-06-10T21:41:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814067+0000", 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059626237999999998 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814176+0000", 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010860400000000001 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104503 12.a 12:53f91a42:::benchmark_data_smithi064_442287_object16453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.70037870199999, 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.527545849, 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104503 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205896+0000", 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205904+0000", 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9699999999999999e-06 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205916+0000", 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2194999999999999e-05 2023-06-10T21:41:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205924+0000", 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218750+0000", 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012825721999999999 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218889+0000", 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138617 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.219004+0000", 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011551 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.224321+0000", 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053164609999999998 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246026+0000", 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021705486 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246036+0000", 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0294e-05 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202839+0000", 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95680261799999999 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202852+0000", 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459999999999999e-05 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636477+0000", 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.433625159 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636649+0000", 2023-06-10T21:41:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017257199999999999 2023-06-10T21:41:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733342+0000", 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096692527 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733443+0000", 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101641 2023-06-10T21:41:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104492 12.8 12:145682b0:::benchmark_data_smithi064_442287_object16442:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:41:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.67558326400001, 2023-06-10T21:41:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1766732329999998, 2023-06-10T21:41:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104492 2023-06-10T21:41:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230691+0000", 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230699+0000", 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-06-10T21:41:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.237985+0000", 2023-06-10T21:41:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072853450000000004 2023-06-10T21:41:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.238000+0000", 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5784999999999999e-05 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258465+0000", 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020465041999999999 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258473+0000", 2023-06-10T21:41:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9489999999999996e-06 2023-06-10T21:41:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257243+0000", 2023-06-10T21:41:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99876928499999995 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257255+0000", 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2431e-05 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.405789+0000", 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.148533523 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407366+0000", 2023-06-10T21:41:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577572 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104510 12.9 12:91eab998:::benchmark_data_smithi064_442287_object16460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:41:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.64150276300001, 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5536998, 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104510 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264772+0000", 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264780+0000", 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264792+0000", 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1987999999999999e-05 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264800+0000", 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8189999999999998e-06 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264846+0000", 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5782000000000003e-05 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264860+0000", 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4267000000000001e-05 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200139+0000", 2023-06-10T21:41:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93527848099999999 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200148+0000", 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758081+0000", 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5579332859999999 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758234+0000", 2023-06-10T21:41:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015339999999999999 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758918+0000", 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068356899999999997 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.784273+0000", 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025354811000000001 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.785804+0000", 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00153134 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814218+0000", 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028414053000000002 2023-06-10T21:41:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814269+0000", 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.075e-05 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818473+0000", 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042044639999999998 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104512 12.8 12:1b4cbe21:::benchmark_data_smithi064_442287_object16462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.63493368300001, 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1733947009999999, 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104512 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271341+0000", 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271350+0000", 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8789999999999998e-06 2023-06-10T21:41:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271394+0000", 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.384e-05 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271404+0000", 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6309999999999998e-06 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271472+0000", 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7986999999999998e-05 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271488+0000", 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5911000000000001e-05 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257279+0000", 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98579087099999996 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257287+0000", 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9410000000000007e-06 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407591+0000", 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15030481 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408064+0000", 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047220399999999999 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408146+0000", 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2446000000000003e-05 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420391+0000", 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012245285999999999 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423446+0000", 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030550500000000001 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429481+0000", 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060345260000000001 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444541+0000", 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015059771 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444604+0000", 2023-06-10T21:41:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3081000000000001e-05 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444737+0000", 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133468 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104514 12.8 12:1ddd6575:::benchmark_data_smithi064_442287_object16464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 185.62737011799999, 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1660305000000002, 2023-06-10T21:41:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104514 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278905+0000", 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278910+0000", 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278921+0000", 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278927+0000", 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5470000000000004e-06 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278994+0000", 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6595999999999997e-05 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.279006+0000", 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2479e-05 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257307+0000", 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97830109899999995 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257315+0000", 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3119999999999994e-06 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410688+0000", 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15337278 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410832+0000", 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014341000000000001 2023-06-10T21:41:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410967+0000", 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135816 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420402+0000", 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094344049999999999 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423489+0000", 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030871060000000001 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429518+0000", 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060294930000000004 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444782+0000", 2023-06-10T21:41:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015263465 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444832+0000", 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9935999999999998e-05 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444937+0000", 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010481599999999999 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117672 12.10 12:08d95f39:::benchmark_data_smithi064_442287_object29622:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 142.69508199000001, 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9927129360000002, 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117672 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211197+0000", 2023-06-10T21:41:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8619999999999999e-06 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211203+0000", 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0959999999999997e-06 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211208+0000", 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.412e-06 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560435+0000", 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922703799999999 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560446+0000", 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0944999999999999e-05 2023-06-10T21:41:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075800+0000", 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515354077 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075806+0000", 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0939999999999996e-06 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203794+0000", 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1279870999999999 2023-06-10T21:41:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203907+0000", 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113702 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117682 12.18 12:18fc4302:::benchmark_data_smithi064_442287_object29632:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 142.69502699, 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.083346562, 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117682 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211252+0000", 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7800000000000001e-06 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211258+0000", 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.404e-06 2023-06-10T21:41:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211263+0000", 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1460000000000004e-06 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560489+0000", 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922595299999998 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560495+0000", 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2299999999999996e-06 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075893+0000", 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515397806 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075905+0000", 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1467e-05 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183546+0000", 2023-06-10T21:41:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1076415719999999 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183688+0000", 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141898 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183979+0000", 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029059499999999999 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.186643+0000", 2023-06-10T21:41:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026644519999999999 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.231646+0000", 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045002563000000002 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294378+0000", 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062732531999999994 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294444+0000", 2023-06-10T21:41:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5165000000000004e-05 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294496+0000", 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2339999999999997e-05 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294596+0000", 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9959999999999998e-05 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117683 12.10 12:0c4ca780:::benchmark_data_smithi064_442287_object29633:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 142.6950081, 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.12118121, 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117683 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:41:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211270+0000", 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2350000000000002e-06 2023-06-10T21:41:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211276+0000", 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.587e-06 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211281+0000", 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2000000000000002e-06 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560465+0000", 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34918397899999998 2023-06-10T21:41:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560471+0000", 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7729999999999996e-06 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075818+0000", 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5153477930000001 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075823+0000", 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8350000000000003e-06 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294814+0000", 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218991033 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.295436+0000", 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062166299999999995 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313438+0000", 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018001952000000002 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313553+0000", 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115364 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313880+0000", 2023-06-10T21:41:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032708500000000002 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.322782+0000", 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089019919999999992 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331101+0000", 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083185140000000008 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331137+0000", 2023-06-10T21:41:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6034000000000001e-05 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331263+0000", 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012637400000000001 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331298+0000", 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5088e-05 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.332449+0000", 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151032 2023-06-10T21:41:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117693 12.19 12:9f069f6e:::benchmark_data_smithi064_442287_object29643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 142.238588284, 2023-06-10T21:41:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5931760800000001, 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117693 2023-06-10T21:41:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667686+0000", 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667699+0000", 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2989e-05 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667723+0000", 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3802e-05 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667736+0000", 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.375e-05 2023-06-10T21:41:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667797+0000", 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0807000000000002e-05 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667811+0000", 2023-06-10T21:41:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3402e-05 2023-06-10T21:41:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075633+0000", 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4078227939999999 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075643+0000", 2023-06-10T21:41:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5529999999999998e-06 2023-06-10T21:41:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205257+0000", 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1296138060000001 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205964+0000", 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070749699999999997 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253657+0000", 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047692997000000001 2023-06-10T21:41:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253748+0000", 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0437000000000002e-05 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5", 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253982+0000", 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023403700000000001 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.258419+0000", 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044375309999999998 2023-06-10T21:41:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259238+0000", 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081873500000000001 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259276+0000", 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8238999999999998e-05 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259376+0000", 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9351000000000005e-05 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259436+0000", 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0044000000000003e-05 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.260864+0000", 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014284499999999999 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117701 12.10 12:0fd56fd2:::benchmark_data_smithi064_442287_object29651:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-06-10T21:41:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:05.281242+0000", 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 141.62503476099999, 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.05464973, 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117701 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281242+0000", 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281242+0000", 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281240+0000", 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281250+0000", 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0019e-05 2023-06-10T21:41:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281264+0000", 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4132e-05 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.281274+0000", 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0214000000000001e-05 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.329300+0000", 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.048025822000000003 2023-06-10T21:41:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.329308+0000", 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1519999999999992e-06 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075845+0000", 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74653679399999995 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075851+0000", 2023-06-10T21:41:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5409999999999997e-06 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.295467+0000", 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.219615602 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.296141+0000", 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00067440000000000002 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.317632+0000", 2023-06-10T21:41:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021490322999999999 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.317707+0000", 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5071e-05 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.318007+0000", 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030030200000000001 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331044+0000", 2023-06-10T21:41:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013036631 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331177+0000", 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013332799999999999 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331236+0000", 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9138999999999997e-05 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.334332+0000", 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003095988 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.334391+0000", 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9194999999999998e-05 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.335891+0000", 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015000759999999999 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117705 12.18 12:1f7e8315:::benchmark_data_smithi064_442287_object29655:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:05.318634+0000", 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 141.587642332, 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9761253590000001, 2023-06-10T21:41:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117705 2023-06-10T21:41:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318634+0000", 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318634+0000", 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318633+0000", 2023-06-10T21:41:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318639+0000", 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6890000000000001e-06 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318649+0000", 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.039e-05 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.318656+0000", 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3089999999999998e-06 2023-06-10T21:41:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.330455+0000", 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011798368 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.330463+0000", 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3939999999999996e-06 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075925+0000", 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.74546160400000006 2023-06-10T21:41:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075930+0000", 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6620000000000002e-06 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.186694+0000", 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.110763068 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.186848+0000", 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015427799999999999 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.187202+0000", 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00035463299999999998 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.231924+0000", 2023-06-10T21:41:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.044721400000000001 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294387+0000", 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062463022 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294417+0000", 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9692e-05 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294630+0000", 2023-06-10T21:41:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021326400000000001 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294672+0000", 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2391999999999997e-05 2023-06-10T21:41:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294759+0000", 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7083999999999995e-05 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117723 12.10 12:0e10ef0f:::benchmark_data_smithi064_442287_object29673:head [delete] snapc 0=[] ondisk+write+known_if_redirected e344)", 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:05.386493+0000", 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 141.51978290400001, 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9611300200000001, 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117723 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386493+0000", 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386493+0000", 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386492+0000", 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386499+0000", 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9739999999999996e-06 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386513+0000", 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3525e-05 2023-06-10T21:41:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386520+0000", 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0949999999999998e-06 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386575+0000", 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5229999999999998e-05 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.386587+0000", 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1963999999999999e-05 2023-06-10T21:41:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075869+0000", 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.68928190499999997 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075875+0000", 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2079999999999997e-06 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.297888+0000", 2023-06-10T21:41:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.222013207 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.298627+0000", 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073904999999999999 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.321271+0000", 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022643789000000001 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.321368+0000", 2023-06-10T21:41:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7416000000000004e-05 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.321624+0000", 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025582600000000001 2023-06-10T21:41:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331189+0000", 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0095649859999999993 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.332513+0000", 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001324066 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.332555+0000", 2023-06-10T21:41:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1770000000000002e-05 2023-06-10T21:41:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.346019+0000", 2023-06-10T21:41:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013463998 2023-06-10T21:41:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.346080+0000", 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0702999999999998e-05 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.347623+0000", 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0015435729999999999 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136391 12.a 12:50cbddcd:::benchmark_data_smithi064_442287_object48341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:41:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.728811234000005, 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.351712574, 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136391 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177464+0000", 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177470+0000", 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8900000000000004e-06 2023-06-10T21:41:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177481+0000", 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1082e-05 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177488+0000", 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.605e-06 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177545+0000", 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6372999999999997e-05 2023-06-10T21:41:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.182867+0000", 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053217430000000003 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210800+0000", 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027933428 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210903+0000", 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103509 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.211048+0000", 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144902 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.215466+0000", 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044176579999999997 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221482+0000", 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060159760000000001 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257726+0000", 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036244365000000001 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257739+0000", 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2807e-05 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.221998+0000", 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96425910500000001 2023-06-10T21:41:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222010+0000", 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1863e-05 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454898+0000", 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2328882999999999 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454985+0000", 2023-06-10T21:41:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6615999999999999e-05 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529107+0000", 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074122399000000005 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529178+0000", 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0131999999999999e-05 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136405 12.a 12:54ed30e6:::benchmark_data_smithi064_442287_object48355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.689689692000002, 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3845864269999999, 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136405 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216586+0000", 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216589+0000", 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5150000000000001e-06 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216594+0000", 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1039999999999998e-06 2023-06-10T21:41:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216598+0000", 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4460000000000003e-06 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221515+0000", 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049165859999999997 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222958+0000", 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014432430000000001 2023-06-10T21:41:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261169+0000", 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038210604000000002 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261175+0000", 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2589999999999997e-06 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222103+0000", 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092808100000005 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222109+0000", 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3199999999999996e-06 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529563+0000", 2023-06-10T21:41:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307454071 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529785+0000", 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022143400000000001 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549764+0000", 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01997898 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549868+0000", 2023-06-10T21:41:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103929 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.550267+0000", 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000398811 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.568050+0000", 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017783812999999999 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575392+0000", 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007341106 2023-06-10T21:41:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575441+0000", 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9833000000000002e-05 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583056+0000", 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076141569999999999 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598427+0000", 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015371246 2023-06-10T21:41:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598489+0000", 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2738999999999997e-05 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.601173+0000", 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002683506 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136406 12.a 12:5c33d118:::benchmark_data_smithi064_442287_object48356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.689668798, 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4148715210000002, 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136406 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:41:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216609+0000", 2023-06-10T21:41:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1780000000000002e-06 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216614+0000", 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.549e-06 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216618+0000", 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222999+0000", 2023-06-10T21:41:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063813000000000003 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224213+0000", 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001213312 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261137+0000", 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036923926000000003 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261146+0000", 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2450000000000004e-06 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222072+0000", 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092633900000002 2023-06-10T21:41:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222078+0000", 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0990000000000004e-06 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529195+0000", 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307116508 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529519+0000", 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032444499999999997 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583294+0000", 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.053774523999999997 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583424+0000", 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130443 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:41:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583605+0000", 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018088100000000001 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.588407+0000", 2023-06-10T21:41:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048019949999999999 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593807+0000", 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053999369999999996 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593843+0000", 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5809999999999998e-05 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.597698+0000", 2023-06-10T21:41:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038552539999999998 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629308+0000", 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031610232000000002 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629379+0000", 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0947999999999999e-05 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.631479+0000", 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020995969999999999 2023-06-10T21:41:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136411 12.a 12:52d92197:::benchmark_data_smithi064_442287_object48361:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 97.681997597999995, 2023-06-10T21:41:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.391707663, 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136411 2023-06-10T21:41:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224277+0000", 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224285+0000", 2023-06-10T21:41:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9610000000000005e-06 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224297+0000", 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.153e-05 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224306+0000", 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6539999999999994e-06 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224396+0000", 2023-06-10T21:41:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9741999999999996e-05 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224406+0000", 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6120000000000005e-06 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261193+0000", 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036787411999999999 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261198+0000", 2023-06-10T21:41:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2730000000000002e-06 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222130+0000", 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96093205800000003 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.443 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222135+0000", 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6940000000000003e-06 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529810+0000", 2023-06-10T21:41:27.444 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307674896 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.530375+0000", 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056451500000000003 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.445 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565747+0000", 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.035372210000000001 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565873+0000", 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125976 2023-06-10T21:41:27.446 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.566075+0000", 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000202573 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.447 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582894+0000", 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016818204999999999 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582953+0000", 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9228000000000003e-05 2023-06-10T21:41:27.448 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583006+0000", 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3650000000000003e-05 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.449 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583198+0000", 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019192600000000001 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613246+0000", 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030047554000000001 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.450 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613310+0000", 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3886999999999993e-05 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.451 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.615986+0000", 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026765270000000002 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156289 12.19 12:9ee93671:::benchmark_data_smithi064_442287_object68239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:41:27.452 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.710801117999999, 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2463835950000002, 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.453 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156289 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.454 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.455 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195473+0000", 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195481+0000", 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8739999999999995e-06 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195499+0000", 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7822e-05 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195507+0000", 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4540000000000007e-06 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.197882+0000", 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023747109999999998 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.199076+0000", 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011941499999999999 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212744+0000", 2023-06-10T21:41:27.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013667643 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212868+0000", 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124313 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.213164+0000", 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029563400000000002 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231891+0000", 2023-06-10T21:41:27.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018727414000000001 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258125+0000", 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026233570000000001 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258140+0000", 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5158e-05 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211376+0000", 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95323585399999999 2023-06-10T21:41:27.463 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211388+0000", 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2655e-05 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.464 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382314+0000", 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1709252590000001 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382471+0000", 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157326 2023-06-10T21:41:27.465 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441738+0000", 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059266970000000002 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.466 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441859+0000", 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120934 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.467 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156297 12.19 12:9878c605:::benchmark_data_smithi064_442287_object68247:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.695531260000003, 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2780029549999998, 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.468 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156297 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:41:27.469 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.470 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210743+0000", 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210751+0000", 2023-06-10T21:41:27.471 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8199999999999997e-06 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210765+0000", 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3745e-05 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.472 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210773+0000", 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1030000000000002e-06 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.230620+0000", 2023-06-10T21:41:27.473 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019846534999999998 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231381+0000", 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076190000000000003 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.474 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260069+0000", 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028687363 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260080+0000", 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0786e-05 2023-06-10T21:41:27.475 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211426+0000", 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134606499999996 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.476 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211436+0000", 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0145e-05 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441910+0000", 2023-06-10T21:41:27.477 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2304743680000001 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.442245+0000", 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000334514 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.478 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454189+0000", 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011943949000000001 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454304+0000", 2023-06-10T21:41:27.479 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115359 2023-06-10T21:41:27.480 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.480 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.480 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:41:27.480 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454674+0000", 2023-06-10T21:41:27.480 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000370176 2023-06-10T21:41:27.481 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.481 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.481 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.481 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459572+0000", 2023-06-10T21:41:27.482 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048976749999999998 2023-06-10T21:41:27.482 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.482 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.482 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.482 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459582+0000", 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105999999999999e-05 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464585+0000", 2023-06-10T21:41:27.483 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050026599999999999 2023-06-10T21:41:27.484 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.484 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.484 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.484 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487733+0000", 2023-06-10T21:41:27.484 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023147893999999999 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487794+0000", 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1424e-05 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.485 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488748+0000", 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095399799999999996 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.486 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156298 12.19 12:98e24496:::benchmark_data_smithi064_442287_object68248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.695492756, 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2788083060000002, 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:41:27.487 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156298 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:41:27.488 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.489 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210783+0000", 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210786+0000", 2023-06-10T21:41:27.490 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5849999999999998e-06 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210792+0000", 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9750000000000004e-06 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.491 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210797+0000", 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9509999999999997e-06 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260110+0000", 2023-06-10T21:41:27.492 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049313312999999998 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260119+0000", 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9889999999999998e-06 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.493 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211462+0000", 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134341099999997 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211471+0000", 2023-06-10T21:41:27.494 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7490000000000004e-06 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.446320+0000", 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234848945 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.495 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.447323+0000", 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001003122 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457450+0000", 2023-06-10T21:41:27.496 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010126634000000001 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457569+0000", 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119261 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.497 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457915+0000", 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034610199999999999 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459652+0000", 2023-06-10T21:41:27.498 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001736502 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464637+0000", 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049851749999999997 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.499 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.467552+0000", 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002914954 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488815+0000", 2023-06-10T21:41:27.500 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021263659000000001 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488870+0000", 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4682000000000003e-05 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:41:27.501 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.489592+0000", 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072164799999999997 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:41:27.502 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:41:27.503 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:41:27.503 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:27 smithi152 ceph-mon[156819]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:27.503 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1504042026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:27.503 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:27 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:27.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[180297]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:27.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1504042026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:27.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:27.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[186706]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:27.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1504042026' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:27.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:27 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-10T21:41:28.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:41:28.404 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:28.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[180297]: failed to encode map e373 with expected crc 2023-06-10T21:41:28.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-10T21:41:28.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[180297]: osdmap e373: 8 total, 8 up, 4 in 2023-06-10T21:41:28.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[186706]: failed to encode map e373 with expected crc 2023-06-10T21:41:28.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-10T21:41:28.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:28 smithi064 ceph-mon[186706]: osdmap e373: 8 total, 8 up, 4 in 2023-06-10T21:41:28.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:28.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:28.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:28 smithi152 ceph-mon[156819]: failed to encode map e373 with expected crc 2023-06-10T21:41:28.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:28 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-10T21:41:28.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:28 smithi152 ceph-mon[156819]: osdmap e373: 8 total, 8 up, 4 in 2023-06-10T21:41:28.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:41:28.509 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:28.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:28.582 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-06-10T21:41:28.749 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:28.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:28.853 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:28.859 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:28.880 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:28.928 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:28.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:29.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:29.059 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:29.107 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:29.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:29.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:29.211 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:29.239 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:29.289 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[186706]: pgmap v1376: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[186706]: failed to encode map e373 with expected crc 2023-06-10T21:41:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[186706]: osdmap e374: 8 total, 8 up, 4 in 2023-06-10T21:41:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[186706]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[186706]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[180297]: pgmap v1376: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[180297]: failed to encode map e373 with expected crc 2023-06-10T21:41:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[180297]: osdmap e374: 8 total, 8 up, 4 in 2023-06-10T21:41:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[180297]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:29 smithi064 ceph-mon[180297]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:29 smithi152 ceph-mon[156819]: pgmap v1376: 89 pgs: 1 creating+peering, 15 unknown, 73 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:29.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:29 smithi152 ceph-mon[156819]: failed to encode map e373 with expected crc 2023-06-10T21:41:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:29 smithi152 ceph-mon[156819]: osdmap e374: 8 total, 8 up, 4 in 2023-06-10T21:41:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:29 smithi152 ceph-mon[156819]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:29 smithi152 ceph-mon[156819]: failed to encode map e374 with expected crc 2023-06-10T21:41:29.491 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:29.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:29.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:29.627 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:29.672 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:29.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:29.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:29.805 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:29.917 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:29.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:30.021 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:30.049 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:30.185 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:30.193 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:41:30.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:30.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:30.309 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:30.338 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:30.387 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:30.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:30.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[180297]: osdmap e375: 8 total, 8 up, 4 in 2023-06-10T21:41:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2668746941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[180297]: pgmap v1379: 89 pgs: 1 creating+peering, 4 unknown, 84 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[186706]: osdmap e375: 8 total, 8 up, 4 in 2023-06-10T21:41:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2668746941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:30 smithi064 ceph-mon[186706]: pgmap v1379: 89 pgs: 1 creating+peering, 4 unknown, 84 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:30 smithi152 ceph-mon[156819]: osdmap e375: 8 total, 8 up, 4 in 2023-06-10T21:41:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2668746941' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:30 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:30.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:30 smithi152 ceph-mon[156819]: pgmap v1379: 89 pgs: 1 creating+peering, 4 unknown, 84 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:30.469 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:41:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:41:30.469 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:41:30.470 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:41:30.492 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:30.519 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:30.717 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:30.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:30.821 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:30.849 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:31.098 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:31.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:31.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:31.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:41:31.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:31.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:31.380 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:31.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:31.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:31.382 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: osdmap e376: 8 total, 8 up, 4 in 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[186706]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: osdmap e376: 8 total, 8 up, 4 in 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.384 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:31 smithi064 ceph-mon[180297]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:31.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:31.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: osdmap e376: 8 total, 8 up, 4 in 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e375 with expected crc 2023-06-10T21:41:31.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:31 smithi152 ceph-mon[156819]: failed to encode map e376 with expected crc 2023-06-10T21:41:31.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:31.502 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:31.516 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:31.563 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:31.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:31.667 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:31.693 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:31.741 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:31.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:31.845 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:31.877 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:32.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:32.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:32.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:32.321 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:41:32.325 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:41:32.389 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:32.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:32 smithi064 ceph-mon[180297]: failed to encode map e376 with expected crc 2023-06-10T21:41:32.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:32 smithi064 ceph-mon[180297]: pgmap v1381: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:32.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:32 smithi064 ceph-mon[186706]: failed to encode map e376 with expected crc 2023-06-10T21:41:32.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:32 smithi064 ceph-mon[186706]: pgmap v1381: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:32.421 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:32.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:32 smithi152 ceph-mon[156819]: failed to encode map e376 with expected crc 2023-06-10T21:41:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:32 smithi152 ceph-mon[156819]: pgmap v1381: 89 pgs: 1 creating+peering, 88 active+clean; 582 KiB data, 3.3 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:32.904 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:41:32.905 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:41:32.905 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:41:32.905 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:41:32.905 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:41:33.140 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:41:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:32] "GET /metrics HTTP/1.1" 200 221718 "" "Prometheus/2.33.4" 2023-06-10T21:41:33.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:33.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:33.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:33 smithi152 ceph-mon[156819]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:33.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:33 smithi064 ceph-mon[180297]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:33.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:33 smithi064 ceph-mon[186706]: from='client.36738 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:33.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:33.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:41:33.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:33.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:33.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:33.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.833 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.834 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:41:33.935 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:33.980 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1381,"stamp":"2023-06-10T21:41:32.122555+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":2379958,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":238,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3431284,"kb_used_data":731220,"kb_used_omap":15,"kb_used_meta":2700016,"kb_avail":371549324,"statfs":{"total":383980142592,"available":380466507776,"internally_reserved":0,"allocated":748769280,"data_stored":744766806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16284,"internal_metadata":2764816484},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.967210"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":399,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.175876+0000","last_change":"2023-06-10T21:39:50.557548+0000","last_active":"2023-06-10T21:41:29.175876+0000","last_peered":"2023-06-10T21:41:29.175876+0000","last_clean":"2023-06-10T21:41:29.175876+0000","last_became_active":"2023-06-10T21:39:50.557023+0000","last_became_peered":"2023-06-10T21:39:50.557023+0000","last_unstale":"2023-06-10T21:41:29.175876+0000","last_undegraded":"2023-06-10T21:41:29.175876+0000","last_fullsized":"2023-06-10T21:41:29.175876+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:45.328782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716428+0000","last_change":"2023-06-10T21:40:30.243387+0000","last_active":"2023-06-10T21:41:19.716428+0000","last_peered":"2023-06-10T21:41:19.716428+0000","last_clean":"2023-06-10T21:41:19.716428+0000","last_became_active":"2023-06-10T21:40:30.243087+0000","last_became_peered":"2023-06-10T21:40:30.243087+0000","last_unstale":"2023-06-10T21:41:19.716428+0000","last_undegraded":"2023-06-10T21:41:19.716428+0000","last_fullsized":"2023-06-10T21:41:19.716428+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:11.139462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":303,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716712+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:41:19.716712+0000","last_peered":"2023-06-10T21:41:19.716712+0000","last_clean":"2023-06-10T21:41:19.716712+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:41:19.716712+0000","last_undegraded":"2023-06-10T21:41:19.716712+0000","last_fullsized":"2023-06-10T21:41:19.716712+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":244,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161509+0000","last_change":"2023-06-10T21:40:30.287586+0000","last_active":"2023-06-10T21:41:27.161509+0000","last_peered":"2023-06-10T21:41:27.161509+0000","last_clean":"2023-06-10T21:41:27.161509+0000","last_became_active":"2023-06-10T21:40:30.286423+0000","last_became_peered":"2023-06-10T21:40:30.286423+0000","last_unstale":"2023-06-10T21:41:27.161509+0000","last_undegraded":"2023-06-10T21:41:27.161509+0000","last_fullsized":"2023-06-10T21:41:27.161509+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715662+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:41:19.715662+0000","last_peered":"2023-06-10T21:41:19.715662+0000","last_clean":"2023-06-10T21:41:19.715662+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:41:19.715662+0000","last_undegraded":"2023-06-10T21:41:19.715662+0000","last_fullsized":"2023-06-10T21:41:19.715662+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167798+0000","last_change":"2023-06-10T21:40:30.236647+0000","last_active":"2023-06-10T21:41:27.167798+0000","last_peered":"2023-06-10T21:41:27.167798+0000","last_clean":"2023-06-10T21:41:27.167798+0000","last_became_active":"2023-06-10T21:40:30.236307+0000","last_became_peered":"2023-06-10T21:40:30.236307+0000","last_unstale":"2023-06-10T21:41:27.167798+0000","last_undegraded":"2023-06-10T21:41:27.167798+0000","last_fullsized":"2023-06-10T21:41:27.167798+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167668+0000","last_change":"2023-06-10T21:40:30.280815+0000","last_active":"2023-06-10T21:41:27.167668+0000","last_peered":"2023-06-10T21:41:27.167668+0000","last_clean":"2023-06-10T21:41:27.167668+0000","last_became_active":"2023-06-10T21:40:30.280427+0000","last_became_peered":"2023-06-10T21:40:30.280427+0000","last_unstale":"2023-06-10T21:41:27.167668+0000","last_undegraded":"2023-06-10T21:41:27.167668+0000","last_fullsized":"2023-06-10T21:41:27.167668+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715759+0000","last_change":"2023-06-10T21:40:30.245395+0000","last_active":"2023-06-10T21:41:19.715759+0000","last_peered":"2023-06-10T21:41:19.715759+0000","last_clean":"2023-06-10T21:41:19.715759+0000","last_became_active":"2023-06-10T21:40:30.244982+0000","last_became_peered":"2023-06-10T21:40:30.244982+0000","last_unstale":"2023-06-10T21:41:19.715759+0000","last_undegraded":"2023-06-10T21:41:19.715759+0000","last_fullsized":"2023-06-10T21:41:19.715759+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:27:28.313757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167601+0000","last_change":"2023-06-10T21:40:30.280871+0000","last_active":"2023-06-10T21:41:27.167601+0000","last_peered":"2023-06-10T21:41:27.167601+0000","last_clean":"2023-06-10T21:41:27.167601+0000","last_became_active":"2023-06-10T21:40:30.280332+0000","last_became_peered":"2023-06-10T21:40:30.280332+0000","last_unstale":"2023-06-10T21:41:27.167601+0000","last_undegraded":"2023-06-10T21:41:27.167601+0000","last_fullsized":"2023-06-10T21:41:27.167601+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":107,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715975+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:41:19.715975+0000","last_peered":"2023-06-10T21:41:19.715975+0000","last_clean":"2023-06-10T21:41:19.715975+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:41:19.715975+0000","last_undegraded":"2023-06-10T21:41:19.715975+0000","last_fullsized":"2023-06-10T21:41:19.715975+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":131,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161239+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:41:27.161239+0000","last_peered":"2023-06-10T21:41:27.161239+0000","last_clean":"2023-06-10T21:41:27.161239+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:41:27.161239+0000","last_undegraded":"2023-06-10T21:41:27.161239+0000","last_fullsized":"2023-06-10T21:41:27.161239+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":359,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.175934+0000","last_change":"2023-06-10T21:39:50.557388+0000","last_active":"2023-06-10T21:41:29.175934+0000","last_peered":"2023-06-10T21:41:29.175934+0000","last_clean":"2023-06-10T21:41:29.175934+0000","last_became_active":"2023-06-10T21:39:50.556659+0000","last_became_peered":"2023-06-10T21:39:50.556659+0000","last_unstale":"2023-06-10T21:41:29.175934+0000","last_undegraded":"2023-06-10T21:41:29.175934+0000","last_fullsized":"2023-06-10T21:41:29.175934+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:32.982830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":282,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161203+0000","last_change":"2023-06-10T21:39:30.845741+0000","last_active":"2023-06-10T21:41:27.161203+0000","last_peered":"2023-06-10T21:41:27.161203+0000","last_clean":"2023-06-10T21:41:27.161203+0000","last_became_active":"2023-06-10T21:39:30.234598+0000","last_became_peered":"2023-06-10T21:39:30.234598+0000","last_unstale":"2023-06-10T21:41:27.161203+0000","last_undegraded":"2023-06-10T21:41:27.161203+0000","last_fullsized":"2023-06-10T21:41:27.161203+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":113,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167165+0000","last_change":"2023-06-10T21:39:50.270986+0000","last_active":"2023-06-10T21:41:27.167165+0000","last_peered":"2023-06-10T21:41:27.167165+0000","last_clean":"2023-06-10T21:41:27.167165+0000","last_became_active":"2023-06-10T21:39:50.269843+0000","last_became_peered":"2023-06-10T21:39:50.269843+0000","last_unstale":"2023-06-10T21:41:27.167165+0000","last_undegraded":"2023-06-10T21:41:27.167165+0000","last_fullsized":"2023-06-10T21:41:27.167165+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":418,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.160949+0000","last_change":"2023-06-10T21:39:50.781250+0000","last_active":"2023-06-10T21:41:27.160949+0000","last_peered":"2023-06-10T21:41:27.160949+0000","last_clean":"2023-06-10T21:41:27.160949+0000","last_became_active":"2023-06-10T21:39:50.376925+0000","last_became_peered":"2023-06-10T21:39:50.376925+0000","last_unstale":"2023-06-10T21:41:27.160949+0000","last_undegraded":"2023-06-10T21:41:27.160949+0000","last_fullsized":"2023-06-10T21:41:27.160949+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":319,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716757+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:41:19.716757+0000","last_peered":"2023-06-10T21:41:19.716757+0000","last_clean":"2023-06-10T21:41:19.716757+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:41:19.716757+0000","last_undegraded":"2023-06-10T21:41:19.716757+0000","last_fullsized":"2023-06-10T21:41:19.716757+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161044+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:41:27.161044+0000","last_peered":"2023-06-10T21:41:27.161044+0000","last_clean":"2023-06-10T21:41:27.161044+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:41:27.161044+0000","last_undegraded":"2023-06-10T21:41:27.161044+0000","last_fullsized":"2023-06-10T21:41:27.161044+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":75,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167345+0000","last_change":"2023-06-10T21:40:30.280758+0000","last_active":"2023-06-10T21:41:27.167345+0000","last_peered":"2023-06-10T21:41:27.167345+0000","last_clean":"2023-06-10T21:41:27.167345+0000","last_became_active":"2023-06-10T21:40:30.280270+0000","last_became_peered":"2023-06-10T21:40:30.280270+0000","last_unstale":"2023-06-10T21:41:27.167345+0000","last_undegraded":"2023-06-10T21:41:27.167345+0000","last_fullsized":"2023-06-10T21:41:27.167345+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.160906+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:41:27.160906+0000","last_peered":"2023-06-10T21:41:27.160906+0000","last_clean":"2023-06-10T21:41:27.160906+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:41:27.160906+0000","last_undegraded":"2023-06-10T21:41:27.160906+0000","last_fullsized":"2023-06-10T21:41:27.160906+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716564+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:41:19.716564+0000","last_peered":"2023-06-10T21:41:19.716564+0000","last_clean":"2023-06-10T21:41:19.716564+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:41:19.716564+0000","last_undegraded":"2023-06-10T21:41:19.716564+0000","last_fullsized":"2023-06-10T21:41:19.716564+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":417,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.166137+0000","last_change":"2023-06-10T21:40:30.476009+0000","last_active":"2023-06-10T21:41:27.166137+0000","last_peered":"2023-06-10T21:41:27.166137+0000","last_clean":"2023-06-10T21:41:27.166137+0000","last_became_active":"2023-06-10T21:40:30.237078+0000","last_became_peered":"2023-06-10T21:40:30.237078+0000","last_unstale":"2023-06-10T21:41:27.166137+0000","last_undegraded":"2023-06-10T21:41:27.166137+0000","last_fullsized":"2023-06-10T21:41:27.166137+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716613+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:41:19.716613+0000","last_peered":"2023-06-10T21:41:19.716613+0000","last_clean":"2023-06-10T21:41:19.716613+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:41:19.716613+0000","last_undegraded":"2023-06-10T21:41:19.716613+0000","last_fullsized":"2023-06-10T21:41:19.716613+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-06-10T21:41:30.185882+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:41:30.185882+0000","last_peered":"2023-06-10T21:41:30.185882+0000","last_clean":"2023-06-10T21:41:30.185882+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:41:30.185882+0000","last_undegraded":"2023-06-10T21:41:30.185882+0000","last_fullsized":"2023-06-10T21:41:30.185882+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":151,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161501+0000","last_change":"2023-06-10T21:40:30.567200+0000","last_active":"2023-06-10T21:41:27.161501+0000","last_peered":"2023-06-10T21:41:27.161501+0000","last_clean":"2023-06-10T21:41:27.161501+0000","last_became_active":"2023-06-10T21:40:30.285607+0000","last_became_peered":"2023-06-10T21:40:30.285607+0000","last_unstale":"2023-06-10T21:41:27.161501+0000","last_undegraded":"2023-06-10T21:41:27.161501+0000","last_fullsized":"2023-06-10T21:41:27.161501+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.166235+0000","last_change":"2023-06-10T21:40:30.232813+0000","last_active":"2023-06-10T21:41:27.166235+0000","last_peered":"2023-06-10T21:41:27.166235+0000","last_clean":"2023-06-10T21:41:27.166235+0000","last_became_active":"2023-06-10T21:40:30.232432+0000","last_became_peered":"2023-06-10T21:40:30.232432+0000","last_unstale":"2023-06-10T21:41:27.166235+0000","last_undegraded":"2023-06-10T21:41:27.166235+0000","last_fullsized":"2023-06-10T21:41:27.166235+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.181199+0000","last_change":"2023-06-10T21:41:28.181199+0000","last_active":"2023-06-10T21:41:28.181199+0000","last_peered":"2023-06-10T21:41:28.181199+0000","last_clean":"2023-06-10T21:41:28.181199+0000","last_became_active":"2023-06-10T21:41:28.180841+0000","last_became_peered":"2023-06-10T21:41:28.180841+0000","last_unstale":"2023-06-10T21:41:28.181199+0000","last_undegraded":"2023-06-10T21:41:28.181199+0000","last_fullsized":"2023-06-10T21:41:28.181199+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":151,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161762+0000","last_change":"2023-06-10T21:40:30.576549+0000","last_active":"2023-06-10T21:41:27.161762+0000","last_peered":"2023-06-10T21:41:27.161762+0000","last_clean":"2023-06-10T21:41:27.161762+0000","last_became_active":"2023-06-10T21:40:30.286869+0000","last_became_peered":"2023-06-10T21:40:30.286869+0000","last_unstale":"2023-06-10T21:41:27.161762+0000","last_undegraded":"2023-06-10T21:41:27.161762+0000","last_fullsized":"2023-06-10T21:41:27.161762+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716717+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:41:19.716717+0000","last_peered":"2023-06-10T21:41:19.716717+0000","last_clean":"2023-06-10T21:41:19.716717+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:41:19.716717+0000","last_undegraded":"2023-06-10T21:41:19.716717+0000","last_fullsized":"2023-06-10T21:41:19.716717+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"0'0","reported_seq":1,"reported_epoch":373,"state":"creating+peering","last_fresh":"2023-06-10T21:41:27.161157+0000","last_change":"2023-06-10T21:41:27.161157+0000","last_active":"2023-06-10T21:41:27.150417+0000","last_peered":"2023-06-10T21:41:27.150417+0000","last_clean":"2023-06-10T21:41:27.150417+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:41:27.161157+0000","last_undegraded":"2023-06-10T21:41:27.161157+0000","last_fullsized":"2023-06-10T21:41:27.161157+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:34:42.879929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":438,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161342+0000","last_change":"2023-06-10T21:40:30.521312+0000","last_active":"2023-06-10T21:41:27.161342+0000","last_peered":"2023-06-10T21:41:27.161342+0000","last_clean":"2023-06-10T21:41:27.161342+0000","last_became_active":"2023-06-10T21:40:30.285713+0000","last_became_peered":"2023-06-10T21:40:30.285713+0000","last_unstale":"2023-06-10T21:41:27.161342+0000","last_undegraded":"2023-06-10T21:41:27.161342+0000","last_fullsized":"2023-06-10T21:41:27.161342+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.160839+0000","last_change":"2023-06-10T21:40:30.343628+0000","last_active":"2023-06-10T21:41:27.160839+0000","last_peered":"2023-06-10T21:41:27.160839+0000","last_clean":"2023-06-10T21:41:27.160839+0000","last_became_active":"2023-06-10T21:40:30.343292+0000","last_became_peered":"2023-06-10T21:40:30.343292+0000","last_unstale":"2023-06-10T21:41:27.160839+0000","last_undegraded":"2023-06-10T21:41:27.160839+0000","last_fullsized":"2023-06-10T21:41:27.160839+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.177207+0000","last_change":"2023-06-10T21:41:28.177207+0000","last_active":"2023-06-10T21:41:28.177207+0000","last_peered":"2023-06-10T21:41:28.177207+0000","last_clean":"2023-06-10T21:41:28.177207+0000","last_became_active":"2023-06-10T21:41:28.176814+0000","last_became_peered":"2023-06-10T21:41:28.176814+0000","last_unstale":"2023-06-10T21:41:28.177207+0000","last_undegraded":"2023-06-10T21:41:28.177207+0000","last_fullsized":"2023-06-10T21:41:28.177207+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":76,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.166990+0000","last_change":"2023-06-10T21:40:30.280834+0000","last_active":"2023-06-10T21:41:27.166990+0000","last_peered":"2023-06-10T21:41:27.166990+0000","last_clean":"2023-06-10T21:41:27.166990+0000","last_became_active":"2023-06-10T21:40:30.280353+0000","last_became_peered":"2023-06-10T21:40:30.280353+0000","last_unstale":"2023-06-10T21:41:27.166990+0000","last_undegraded":"2023-06-10T21:41:27.166990+0000","last_fullsized":"2023-06-10T21:41:27.166990+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716453+0000","last_change":"2023-06-10T21:40:30.272439+0000","last_active":"2023-06-10T21:41:19.716453+0000","last_peered":"2023-06-10T21:41:19.716453+0000","last_clean":"2023-06-10T21:41:19.716453+0000","last_became_active":"2023-06-10T21:40:30.272004+0000","last_became_peered":"2023-06-10T21:40:30.272004+0000","last_unstale":"2023-06-10T21:41:19.716453+0000","last_undegraded":"2023-06-10T21:41:19.716453+0000","last_fullsized":"2023-06-10T21:41:19.716453+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:30.425736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.178371+0000","last_change":"2023-06-10T21:41:28.178371+0000","last_active":"2023-06-10T21:41:28.178371+0000","last_peered":"2023-06-10T21:41:28.178371+0000","last_clean":"2023-06-10T21:41:28.178371+0000","last_became_active":"2023-06-10T21:41:28.178058+0000","last_became_peered":"2023-06-10T21:41:28.178058+0000","last_unstale":"2023-06-10T21:41:28.178371+0000","last_undegraded":"2023-06-10T21:41:28.178371+0000","last_fullsized":"2023-06-10T21:41:28.178371+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":29,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716350+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:41:19.716350+0000","last_peered":"2023-06-10T21:41:19.716350+0000","last_clean":"2023-06-10T21:41:19.716350+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:41:19.716350+0000","last_undegraded":"2023-06-10T21:41:19.716350+0000","last_fullsized":"2023-06-10T21:41:19.716350+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716177+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:41:19.716177+0000","last_peered":"2023-06-10T21:41:19.716177+0000","last_clean":"2023-06-10T21:41:19.716177+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:41:19.716177+0000","last_undegraded":"2023-06-10T21:41:19.716177+0000","last_fullsized":"2023-06-10T21:41:19.716177+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-06-10T21:41:30.184978+0000","last_change":"2023-06-10T21:41:28.182345+0000","last_active":"2023-06-10T21:41:30.184978+0000","last_peered":"2023-06-10T21:41:30.184978+0000","last_clean":"2023-06-10T21:41:30.184978+0000","last_became_active":"2023-06-10T21:41:28.181851+0000","last_became_peered":"2023-06-10T21:41:28.181851+0000","last_unstale":"2023-06-10T21:41:30.184978+0000","last_undegraded":"2023-06-10T21:41:30.184978+0000","last_fullsized":"2023-06-10T21:41:30.184978+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:07:28.251208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":177,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716368+0000","last_change":"2023-06-10T21:39:50.465026+0000","last_active":"2023-06-10T21:41:19.716368+0000","last_peered":"2023-06-10T21:41:19.716368+0000","last_clean":"2023-06-10T21:41:19.716368+0000","last_became_active":"2023-06-10T21:39:50.464762+0000","last_became_peered":"2023-06-10T21:39:50.464762+0000","last_unstale":"2023-06-10T21:41:19.716368+0000","last_undegraded":"2023-06-10T21:41:19.716368+0000","last_fullsized":"2023-06-10T21:41:19.716368+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:55:00.850195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":397,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.176433+0000","last_change":"2023-06-10T21:39:50.259203+0000","last_active":"2023-06-10T21:41:29.176433+0000","last_peered":"2023-06-10T21:41:29.176433+0000","last_clean":"2023-06-10T21:41:29.176433+0000","last_became_active":"2023-06-10T21:39:50.258873+0000","last_became_peered":"2023-06-10T21:39:50.258873+0000","last_unstale":"2023-06-10T21:41:29.176433+0000","last_undegraded":"2023-06-10T21:41:29.176433+0000","last_fullsized":"2023-06-10T21:41:29.176433+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:56:41.490096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.184243+0000","last_change":"2023-06-10T21:41:28.184243+0000","last_active":"2023-06-10T21:41:28.184243+0000","last_peered":"2023-06-10T21:41:28.184243+0000","last_clean":"2023-06-10T21:41:28.184243+0000","last_became_active":"2023-06-10T21:41:28.183870+0000","last_became_peered":"2023-06-10T21:41:28.183870+0000","last_unstale":"2023-06-10T21:41:28.184243+0000","last_undegraded":"2023-06-10T21:41:28.184243+0000","last_fullsized":"2023-06-10T21:41:28.184243+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":375,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.175806+0000","last_change":"2023-06-10T21:39:50.432352+0000","last_active":"2023-06-10T21:41:29.175806+0000","last_peered":"2023-06-10T21:41:29.175806+0000","last_clean":"2023-06-10T21:41:29.175806+0000","last_became_active":"2023-06-10T21:39:50.429920+0000","last_became_peered":"2023-06-10T21:39:50.429920+0000","last_unstale":"2023-06-10T21:41:29.175806+0000","last_undegraded":"2023-06-10T21:41:29.175806+0000","last_fullsized":"2023-06-10T21:41:29.175806+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:14.733503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716480+0000","last_change":"2023-06-10T21:40:30.351796+0000","last_active":"2023-06-10T21:41:19.716480+0000","last_peered":"2023-06-10T21:41:19.716480+0000","last_clean":"2023-06-10T21:41:19.716480+0000","last_became_active":"2023-06-10T21:40:30.350450+0000","last_became_peered":"2023-06-10T21:40:30.350450+0000","last_unstale":"2023-06-10T21:41:19.716480+0000","last_undegraded":"2023-06-10T21:41:19.716480+0000","last_fullsized":"2023-06-10T21:41:19.716480+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:38.306536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.177644+0000","last_change":"2023-06-10T21:41:28.177644+0000","last_active":"2023-06-10T21:41:28.177644+0000","last_peered":"2023-06-10T21:41:28.177644+0000","last_clean":"2023-06-10T21:41:28.177644+0000","last_became_active":"2023-06-10T21:41:28.177318+0000","last_became_peered":"2023-06-10T21:41:28.177318+0000","last_unstale":"2023-06-10T21:41:28.177644+0000","last_undegraded":"2023-06-10T21:41:28.177644+0000","last_fullsized":"2023-06-10T21:41:28.177644+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715977+0000","last_change":"2023-06-10T21:40:30.279010+0000","last_active":"2023-06-10T21:41:19.715977+0000","last_peered":"2023-06-10T21:41:19.715977+0000","last_clean":"2023-06-10T21:41:19.715977+0000","last_became_active":"2023-06-10T21:40:30.277943+0000","last_became_peered":"2023-06-10T21:40:30.277943+0000","last_unstale":"2023-06-10T21:41:19.715977+0000","last_undegraded":"2023-06-10T21:41:19.715977+0000","last_fullsized":"2023-06-10T21:41:19.715977+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:38:15.175343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":304,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716686+0000","last_change":"2023-06-10T21:39:50.233694+0000","last_active":"2023-06-10T21:41:19.716686+0000","last_peered":"2023-06-10T21:41:19.716686+0000","last_clean":"2023-06-10T21:41:19.716686+0000","last_became_active":"2023-06-10T21:39:50.233429+0000","last_became_peered":"2023-06-10T21:39:50.233429+0000","last_unstale":"2023-06-10T21:41:19.716686+0000","last_undegraded":"2023-06-10T21:41:19.716686+0000","last_fullsized":"2023-06-10T21:41:19.716686+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:15:42.432299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.176942+0000","last_change":"2023-06-10T21:41:28.176942+0000","last_active":"2023-06-10T21:41:28.176942+0000","last_peered":"2023-06-10T21:41:28.176942+0000","last_clean":"2023-06-10T21:41:28.176942+0000","last_became_active":"2023-06-10T21:41:28.176613+0000","last_became_peered":"2023-06-10T21:41:28.176613+0000","last_unstale":"2023-06-10T21:41:28.176942+0000","last_undegraded":"2023-06-10T21:41:28.176942+0000","last_fullsized":"2023-06-10T21:41:28.176942+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":157,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161147+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:41:27.161147+0000","last_peered":"2023-06-10T21:41:27.161147+0000","last_clean":"2023-06-10T21:41:27.161147+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:41:27.161147+0000","last_undegraded":"2023-06-10T21:41:27.161147+0000","last_fullsized":"2023-06-10T21:41:27.161147+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":57,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167168+0000","last_change":"2023-06-10T21:40:30.281222+0000","last_active":"2023-06-10T21:41:27.167168+0000","last_peered":"2023-06-10T21:41:27.167168+0000","last_clean":"2023-06-10T21:41:27.167168+0000","last_became_active":"2023-06-10T21:40:30.280846+0000","last_became_peered":"2023-06-10T21:40:30.280846+0000","last_unstale":"2023-06-10T21:41:27.167168+0000","last_undegraded":"2023-06-10T21:41:27.167168+0000","last_fullsized":"2023-06-10T21:41:27.167168+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167211+0000","last_change":"2023-06-10T21:39:50.466302+0000","last_active":"2023-06-10T21:41:27.167211+0000","last_peered":"2023-06-10T21:41:27.167211+0000","last_clean":"2023-06-10T21:41:27.167211+0000","last_became_active":"2023-06-10T21:39:50.465409+0000","last_became_peered":"2023-06-10T21:39:50.465409+0000","last_unstale":"2023-06-10T21:41:27.167211+0000","last_undegraded":"2023-06-10T21:41:27.167211+0000","last_fullsized":"2023-06-10T21:41:27.167211+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.182320+0000","last_change":"2023-06-10T21:41:28.182320+0000","last_active":"2023-06-10T21:41:28.182320+0000","last_peered":"2023-06-10T21:41:28.182320+0000","last_clean":"2023-06-10T21:41:28.182320+0000","last_became_active":"2023-06-10T21:41:28.181951+0000","last_became_peered":"2023-06-10T21:41:28.181951+0000","last_unstale":"2023-06-10T21:41:28.182320+0000","last_undegraded":"2023-06-10T21:41:28.182320+0000","last_fullsized":"2023-06-10T21:41:28.182320+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":76,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167219+0000","last_change":"2023-06-10T21:40:30.236317+0000","last_active":"2023-06-10T21:41:27.167219+0000","last_peered":"2023-06-10T21:41:27.167219+0000","last_clean":"2023-06-10T21:41:27.167219+0000","last_became_active":"2023-06-10T21:40:30.235998+0000","last_became_peered":"2023-06-10T21:40:30.235998+0000","last_unstale":"2023-06-10T21:41:27.167219+0000","last_undegraded":"2023-06-10T21:41:27.167219+0000","last_fullsized":"2023-06-10T21:41:27.167219+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":354,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167196+0000","last_change":"2023-06-10T21:40:30.281401+0000","last_active":"2023-06-10T21:41:27.167196+0000","last_peered":"2023-06-10T21:41:27.167196+0000","last_clean":"2023-06-10T21:41:27.167196+0000","last_became_active":"2023-06-10T21:40:30.280966+0000","last_became_peered":"2023-06-10T21:40:30.280966+0000","last_unstale":"2023-06-10T21:41:27.167196+0000","last_undegraded":"2023-06-10T21:41:27.167196+0000","last_fullsized":"2023-06-10T21:41:27.167196+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":239,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.176305+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:41:29.176305+0000","last_peered":"2023-06-10T21:41:29.176305+0000","last_clean":"2023-06-10T21:41:29.176305+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:41:29.176305+0000","last_undegraded":"2023-06-10T21:41:29.176305+0000","last_fullsized":"2023-06-10T21:41:29.176305+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.182336+0000","last_change":"2023-06-10T21:41:28.182336+0000","last_active":"2023-06-10T21:41:28.182336+0000","last_peered":"2023-06-10T21:41:28.182336+0000","last_clean":"2023-06-10T21:41:28.182336+0000","last_became_active":"2023-06-10T21:41:28.181961+0000","last_became_peered":"2023-06-10T21:41:28.181961+0000","last_unstale":"2023-06-10T21:41:28.182336+0000","last_undegraded":"2023-06-10T21:41:28.182336+0000","last_fullsized":"2023-06-10T21:41:28.182336+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":259,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.175763+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:41:29.175763+0000","last_peered":"2023-06-10T21:41:29.175763+0000","last_clean":"2023-06-10T21:41:29.175763+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:41:29.175763+0000","last_undegraded":"2023-06-10T21:41:29.175763+0000","last_fullsized":"2023-06-10T21:41:29.175763+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":248,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.175696+0000","last_change":"2023-06-10T21:40:30.229244+0000","last_active":"2023-06-10T21:41:29.175696+0000","last_peered":"2023-06-10T21:41:29.175696+0000","last_clean":"2023-06-10T21:41:29.175696+0000","last_became_active":"2023-06-10T21:40:30.228964+0000","last_became_peered":"2023-06-10T21:40:30.228964+0000","last_unstale":"2023-06-10T21:41:29.175696+0000","last_undegraded":"2023-06-10T21:41:29.175696+0000","last_fullsized":"2023-06-10T21:41:29.175696+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:41.629358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167735+0000","last_change":"2023-06-10T21:40:30.236712+0000","last_active":"2023-06-10T21:41:27.167735+0000","last_peered":"2023-06-10T21:41:27.167735+0000","last_clean":"2023-06-10T21:41:27.167735+0000","last_became_active":"2023-06-10T21:40:30.236404+0000","last_became_peered":"2023-06-10T21:40:30.236404+0000","last_unstale":"2023-06-10T21:41:27.167735+0000","last_undegraded":"2023-06-10T21:41:27.167735+0000","last_fullsized":"2023-06-10T21:41:27.167735+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.181267+0000","last_change":"2023-06-10T21:41:28.181267+0000","last_active":"2023-06-10T21:41:28.181267+0000","last_peered":"2023-06-10T21:41:28.181267+0000","last_clean":"2023-06-10T21:41:28.181267+0000","last_became_active":"2023-06-10T21:41:28.180932+0000","last_became_peered":"2023-06-10T21:41:28.180932+0000","last_unstale":"2023-06-10T21:41:28.181267+0000","last_undegraded":"2023-06-10T21:41:28.181267+0000","last_fullsized":"2023-06-10T21:41:28.181267+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":48,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.166900+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:41:27.166900+0000","last_peered":"2023-06-10T21:41:27.166900+0000","last_clean":"2023-06-10T21:41:27.166900+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:41:27.166900+0000","last_undegraded":"2023-06-10T21:41:27.166900+0000","last_fullsized":"2023-06-10T21:41:27.166900+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":574,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.160617+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:41:27.160617+0000","last_peered":"2023-06-10T21:41:27.160617+0000","last_clean":"2023-06-10T21:41:27.160617+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:41:27.160617+0000","last_undegraded":"2023-06-10T21:41:27.160617+0000","last_fullsized":"2023-06-10T21:41:27.160617+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716113+0000","last_change":"2023-06-10T21:40:30.260288+0000","last_active":"2023-06-10T21:41:19.716113+0000","last_peered":"2023-06-10T21:41:19.716113+0000","last_clean":"2023-06-10T21:41:19.716113+0000","last_became_active":"2023-06-10T21:40:30.259958+0000","last_became_peered":"2023-06-10T21:40:30.259958+0000","last_unstale":"2023-06-10T21:41:19.716113+0000","last_undegraded":"2023-06-10T21:41:19.716113+0000","last_fullsized":"2023-06-10T21:41:19.716113+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:41.077016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":242,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161705+0000","last_change":"2023-06-10T21:40:30.286026+0000","last_active":"2023-06-10T21:41:27.161705+0000","last_peered":"2023-06-10T21:41:27.161705+0000","last_clean":"2023-06-10T21:41:27.161705+0000","last_became_active":"2023-06-10T21:40:30.285670+0000","last_became_peered":"2023-06-10T21:40:30.285670+0000","last_unstale":"2023-06-10T21:41:27.161705+0000","last_undegraded":"2023-06-10T21:41:27.161705+0000","last_fullsized":"2023-06-10T21:41:27.161705+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.183802+0000","last_change":"2023-06-10T21:41:28.183802+0000","last_active":"2023-06-10T21:41:28.183802+0000","last_peered":"2023-06-10T21:41:28.183802+0000","last_clean":"2023-06-10T21:41:28.183802+0000","last_became_active":"2023-06-10T21:41:28.183511+0000","last_became_peered":"2023-06-10T21:41:28.183511+0000","last_unstale":"2023-06-10T21:41:28.183802+0000","last_undegraded":"2023-06-10T21:41:28.183802+0000","last_fullsized":"2023-06-10T21:41:28.183802+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":411,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.160637+0000","last_change":"2023-06-10T21:39:50.237017+0000","last_active":"2023-06-10T21:41:27.160637+0000","last_peered":"2023-06-10T21:41:27.160637+0000","last_clean":"2023-06-10T21:41:27.160637+0000","last_became_active":"2023-06-10T21:39:50.232270+0000","last_became_peered":"2023-06-10T21:39:50.232270+0000","last_unstale":"2023-06-10T21:41:27.160637+0000","last_undegraded":"2023-06-10T21:41:27.160637+0000","last_fullsized":"2023-06-10T21:41:27.160637+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715815+0000","last_change":"2023-06-10T21:40:30.316582+0000","last_active":"2023-06-10T21:41:19.715815+0000","last_peered":"2023-06-10T21:41:19.715815+0000","last_clean":"2023-06-10T21:41:19.715815+0000","last_became_active":"2023-06-10T21:40:30.315657+0000","last_became_peered":"2023-06-10T21:40:30.315657+0000","last_unstale":"2023-06-10T21:41:19.715815+0000","last_undegraded":"2023-06-10T21:41:19.715815+0000","last_fullsized":"2023-06-10T21:41:19.715815+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:08:04.756800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":110,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.176536+0000","last_change":"2023-06-10T21:39:50.559093+0000","last_active":"2023-06-10T21:41:29.176536+0000","last_peered":"2023-06-10T21:41:29.176536+0000","last_clean":"2023-06-10T21:41:29.176536+0000","last_became_active":"2023-06-10T21:39:50.558289+0000","last_became_peered":"2023-06-10T21:39:50.558289+0000","last_unstale":"2023-06-10T21:41:29.176536+0000","last_undegraded":"2023-06-10T21:41:29.176536+0000","last_fullsized":"2023-06-10T21:41:29.176536+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:37.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-06-10T21:41:30.184057+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:41:30.184057+0000","last_peered":"2023-06-10T21:41:30.184057+0000","last_clean":"2023-06-10T21:41:30.184057+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:41:30.184057+0000","last_undegraded":"2023-06-10T21:41:30.184057+0000","last_fullsized":"2023-06-10T21:41:30.184057+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":405,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161431+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:41:27.161431+0000","last_peered":"2023-06-10T21:41:27.161431+0000","last_clean":"2023-06-10T21:41:27.161431+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:41:27.161431+0000","last_undegraded":"2023-06-10T21:41:27.161431+0000","last_fullsized":"2023-06-10T21:41:27.161431+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":239,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.176384+0000","last_change":"2023-06-10T21:39:50.296322+0000","last_active":"2023-06-10T21:41:29.176384+0000","last_peered":"2023-06-10T21:41:29.176384+0000","last_clean":"2023-06-10T21:41:29.176384+0000","last_became_active":"2023-06-10T21:39:50.295201+0000","last_became_peered":"2023-06-10T21:39:50.295201+0000","last_unstale":"2023-06-10T21:41:29.176384+0000","last_undegraded":"2023-06-10T21:41:29.176384+0000","last_fullsized":"2023-06-10T21:41:29.176384+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:28.403434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167261+0000","last_change":"2023-06-10T21:39:50.271175+0000","last_active":"2023-06-10T21:41:27.167261+0000","last_peered":"2023-06-10T21:41:27.167261+0000","last_clean":"2023-06-10T21:41:27.167261+0000","last_became_active":"2023-06-10T21:39:50.270302+0000","last_became_peered":"2023-06-10T21:39:50.270302+0000","last_unstale":"2023-06-10T21:41:27.167261+0000","last_undegraded":"2023-06-10T21:41:27.167261+0000","last_fullsized":"2023-06-10T21:41:27.167261+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"0'0","reported_seq":9,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-06-10T21:41:28.179779+0000","last_change":"2023-06-10T21:41:28.179779+0000","last_active":"2023-06-10T21:41:28.179779+0000","last_peered":"2023-06-10T21:41:28.179779+0000","last_clean":"2023-06-10T21:41:28.179779+0000","last_became_active":"2023-06-10T21:41:28.179455+0000","last_became_peered":"2023-06-10T21:41:28.179455+0000","last_unstale":"2023-06-10T21:41:28.179779+0000","last_undegraded":"2023-06-10T21:41:28.179779+0000","last_fullsized":"2023-06-10T21:41:28.179779+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":366,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716613+0000","last_change":"2023-06-10T21:39:30.266087+0000","last_active":"2023-06-10T21:41:19.716613+0000","last_peered":"2023-06-10T21:41:19.716613+0000","last_clean":"2023-06-10T21:41:19.716613+0000","last_became_active":"2023-06-10T21:39:30.231858+0000","last_became_peered":"2023-06-10T21:39:30.231858+0000","last_unstale":"2023-06-10T21:41:19.716613+0000","last_undegraded":"2023-06-10T21:41:19.716613+0000","last_fullsized":"2023-06-10T21:41:19.716613+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:08:09.982313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":177,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716812+0000","last_change":"2023-06-10T21:39:50.227162+0000","last_active":"2023-06-10T21:41:19.716812+0000","last_peered":"2023-06-10T21:41:19.716812+0000","last_clean":"2023-06-10T21:41:19.716812+0000","last_became_active":"2023-06-10T21:39:50.226891+0000","last_became_peered":"2023-06-10T21:39:50.226891+0000","last_unstale":"2023-06-10T21:41:19.716812+0000","last_undegraded":"2023-06-10T21:41:19.716812+0000","last_fullsized":"2023-06-10T21:41:19.716812+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:48:59.680224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716838+0000","last_change":"2023-06-10T21:40:30.279185+0000","last_active":"2023-06-10T21:41:19.716838+0000","last_peered":"2023-06-10T21:41:19.716838+0000","last_clean":"2023-06-10T21:41:19.716838+0000","last_became_active":"2023-06-10T21:40:30.278330+0000","last_became_peered":"2023-06-10T21:40:30.278330+0000","last_unstale":"2023-06-10T21:41:19.716838+0000","last_undegraded":"2023-06-10T21:41:19.716838+0000","last_fullsized":"2023-06-10T21:41:19.716838+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:15.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-06-10T21:41:30.183474+0000","last_change":"2023-06-10T21:41:28.176991+0000","last_active":"2023-06-10T21:41:30.183474+0000","last_peered":"2023-06-10T21:41:30.183474+0000","last_clean":"2023-06-10T21:41:30.183474+0000","last_became_active":"2023-06-10T21:41:28.176650+0000","last_became_peered":"2023-06-10T21:41:28.176650+0000","last_unstale":"2023-06-10T21:41:30.183474+0000","last_undegraded":"2023-06-10T21:41:30.183474+0000","last_fullsized":"2023-06-10T21:41:30.183474+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:14.567179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":122,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161518+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:41:27.161518+0000","last_peered":"2023-06-10T21:41:27.161518+0000","last_clean":"2023-06-10T21:41:27.161518+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:41:27.161518+0000","last_undegraded":"2023-06-10T21:41:27.161518+0000","last_fullsized":"2023-06-10T21:41:27.161518+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":221,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161395+0000","last_change":"2023-06-10T21:40:30.580293+0000","last_active":"2023-06-10T21:41:27.161395+0000","last_peered":"2023-06-10T21:41:27.161395+0000","last_clean":"2023-06-10T21:41:27.161395+0000","last_became_active":"2023-06-10T21:40:30.391049+0000","last_became_peered":"2023-06-10T21:40:30.391049+0000","last_unstale":"2023-06-10T21:41:27.161395+0000","last_undegraded":"2023-06-10T21:41:27.161395+0000","last_fullsized":"2023-06-10T21:41:27.161395+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.166121+0000","last_change":"2023-06-10T21:39:30.208137+0000","last_active":"2023-06-10T21:41:27.166121+0000","last_peered":"2023-06-10T21:41:27.166121+0000","last_clean":"2023-06-10T21:41:27.166121+0000","last_became_active":"2023-06-10T21:39:30.207848+0000","last_became_peered":"2023-06-10T21:39:30.207848+0000","last_unstale":"2023-06-10T21:41:27.166121+0000","last_undegraded":"2023-06-10T21:41:27.166121+0000","last_fullsized":"2023-06-10T21:41:27.166121+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":73,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.161102+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:41:27.161102+0000","last_peered":"2023-06-10T21:41:27.161102+0000","last_clean":"2023-06-10T21:41:27.161102+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:41:27.161102+0000","last_undegraded":"2023-06-10T21:41:27.161102+0000","last_fullsized":"2023-06-10T21:41:27.161102+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":256,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-06-10T21:41:29.176421+0000","last_change":"2023-06-10T21:40:30.227732+0000","last_active":"2023-06-10T21:41:29.176421+0000","last_peered":"2023-06-10T21:41:29.176421+0000","last_clean":"2023-06-10T21:41:29.176421+0000","last_became_active":"2023-06-10T21:40:30.227380+0000","last_became_peered":"2023-06-10T21:40:30.227380+0000","last_unstale":"2023-06-10T21:41:29.176421+0000","last_undegraded":"2023-06-10T21:41:29.176421+0000","last_fullsized":"2023-06-10T21:41:29.176421+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:33.673485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167075+0000","last_change":"2023-06-10T21:39:50.304413+0000","last_active":"2023-06-10T21:41:27.167075+0000","last_peered":"2023-06-10T21:41:27.167075+0000","last_clean":"2023-06-10T21:41:27.167075+0000","last_became_active":"2023-06-10T21:39:50.303414+0000","last_became_peered":"2023-06-10T21:39:50.303414+0000","last_unstale":"2023-06-10T21:41:27.167075+0000","last_undegraded":"2023-06-10T21:41:27.167075+0000","last_fullsized":"2023-06-10T21:41:27.167075+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":276,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.162623+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:41:27.162623+0000","last_peered":"2023-06-10T21:41:27.162623+0000","last_clean":"2023-06-10T21:41:27.162623+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:41:27.162623+0000","last_undegraded":"2023-06-10T21:41:27.162623+0000","last_fullsized":"2023-06-10T21:41:27.162623+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":78,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167413+0000","last_change":"2023-06-10T21:40:30.565123+0000","last_active":"2023-06-10T21:41:27.167413+0000","last_peered":"2023-06-10T21:41:27.167413+0000","last_clean":"2023-06-10T21:41:27.167413+0000","last_became_active":"2023-06-10T21:40:30.280290+0000","last_became_peered":"2023-06-10T21:40:30.280290+0000","last_unstale":"2023-06-10T21:41:27.167413+0000","last_undegraded":"2023-06-10T21:41:27.167413+0000","last_fullsized":"2023-06-10T21:41:27.167413+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":332,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.162500+0000","last_change":"2023-06-10T21:39:30.245702+0000","last_active":"2023-06-10T21:41:27.162500+0000","last_peered":"2023-06-10T21:41:27.162500+0000","last_clean":"2023-06-10T21:41:27.162500+0000","last_became_active":"2023-06-10T21:39:30.245426+0000","last_became_peered":"2023-06-10T21:39:30.245426+0000","last_unstale":"2023-06-10T21:41:27.162500+0000","last_undegraded":"2023-06-10T21:41:27.162500+0000","last_fullsized":"2023-06-10T21:41:27.162500+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:52:27.316156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716654+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:41:19.716654+0000","last_peered":"2023-06-10T21:41:19.716654+0000","last_clean":"2023-06-10T21:41:19.716654+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:41:19.716654+0000","last_undegraded":"2023-06-10T21:41:19.716654+0000","last_fullsized":"2023-06-10T21:41:19.716654+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.715840+0000","last_change":"2023-06-10T21:40:30.253755+0000","last_active":"2023-06-10T21:41:19.715840+0000","last_peered":"2023-06-10T21:41:19.715840+0000","last_clean":"2023-06-10T21:41:19.715840+0000","last_became_active":"2023-06-10T21:40:30.253530+0000","last_became_peered":"2023-06-10T21:40:30.253530+0000","last_unstale":"2023-06-10T21:41:19.715840+0000","last_undegraded":"2023-06-10T21:41:19.715840+0000","last_fullsized":"2023-06-10T21:41:19.715840+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:14.839319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":314,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-06-10T21:41:27.167740+0000","last_change":"2023-06-10T21:40:30.574471+0000","last_active":"2023-06-10T21:41:27.167740+0000","last_peered":"2023-06-10T21:41:27.167740+0000","last_clean":"2023-06-10T21:41:27.167740+0000","last_became_active":"2023-06-10T21:40:30.281351+0000","last_became_peered":"2023-06-10T21:40:30.281351+0000","last_unstale":"2023-06-10T21:41:27.167740+0000","last_undegraded":"2023-06-10T21:41:27.167740+0000","last_fullsized":"2023-06-10T21:41:27.167740+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-06-10T21:41:19.716048+0000","last_change":"2023-06-10T21:40:30.245261+0000","last_active":"2023-06-10T21:41:19.716048+0000","last_peered":"2023-06-10T21:41:19.716048+0000","last_clean":"2023-06-10T21:41:19.716048+0000","last_became_active":"2023-06-10T21:40:30.244695+0000","last_became_peered":"2023-06-10T21:40:30.244695+0000","last_unstale":"2023-06-10T21:41:19.716048+0000","last_undegraded":"2023-06-10T21:41:19.716048+0000","last_fullsized":"2023-06-10T21:41:19.716048+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:54.784460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":18486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430733,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815452,"kb_used_data":3924,"kb_used_omap":0,"kb_used_meta":811520,"kb_avail":92929700,"statfs":{"total":95995035648,"available":95160012800,"internally_reserved":0,"allocated":4018176,"data_stored":3323681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":830996480},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.986,"15min":4.8600000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":22.277000000000001,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.3889999999999998,"5min":2.4079999999999999,"15min":5.165},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.223},"max":{"1min":22.462,"5min":56.628,"15min":185.58000000000001},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.9660000000000002,"15min":2.976},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":38.078000000000003,"5min":60.863,"15min":93.093999999999994},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.476,"5min":3.2919999999999998,"15min":3.1000000000000001},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":38.341000000000001,"5min":61.148000000000003,"15min":93.119},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:40:35 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":4.5250000000000004,"15min":5.4109999999999996},"min":{"1min":0.33500000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":6.6260000000000003,"5min":137.64099999999999,"15min":172.33600000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.794,"5min":4.2199999999999998,"15min":5.3440000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29499999999999998},"max":{"1min":5.8029999999999999,"5min":136.96600000000001,"15min":172.185},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":4.3460000000000001,"15min":4.2649999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":24.375,"5min":102.25700000000001,"15min":147.00700000000001},"last":0.436},{"interface":"front","average":{"1min":2.347,"5min":4.1070000000000002,"15min":3.923},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":26.460000000000001,"5min":103.401,"15min":146.93899999999999},"last":0.505}]},{"osd":4,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.249},"max":{"1min":1.0289999999999999,"5min":5.859,"15min":5.859},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.81399999999999995,"15min":0.66700000000000004},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.3290000000000002,"5min":22.77,"15min":22.77},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48399999999999999,"15min":0.64200000000000002},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.8220000000000001,"5min":2.7930000000000001,"15min":12.048},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.97499999999999998,"5min":5.9100000000000001,"15min":6.7530000000000001},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.68300000000000005,"15min":0.72399999999999998},"min":{"1min":0.34699999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":1.847,"5min":10.890000000000001,"15min":13.073},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.59299999999999997,"15min":0.56799999999999995},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0209999999999999,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.57899999999999996}]}]},{"osd":6,"up_from":0,"seq":180388626961,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566404,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020380,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075792,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753296,"kb_used_data":4552,"kb_used_omap":7,"kb_used_meta":748728,"kb_avail":92991856,"statfs":{"total":95995035648,"available":95223660544,"internally_reserved":0,"allocated":4661248,"data_stored":3404793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":766697553},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":5.54,"5min":5.9180000000000001,"15min":5.0979999999999999},"min":{"1min":0.19900000000000001,"5min":0.19,"15min":0.19},"max":{"1min":83.733000000000004,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":4.9020000000000001,"5min":5.4669999999999996,"15min":5.173},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":82.322999999999993,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":7.4720000000000004,"5min":7.6699999999999999,"15min":5.0419999999999998},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.19},"max":{"1min":74.256,"5min":171.286,"15min":171.286},"last":0.72099999999999997},{"interface":"front","average":{"1min":5.6689999999999996,"5min":3.5830000000000002,"15min":3.4420000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.214},"max":{"1min":72.683999999999997,"5min":72.683999999999997,"15min":147.81299999999999},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:41:20 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":5.694,"15min":5.5270000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":70.480999999999995,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":5.4980000000000002,"5min":5.3550000000000004,"15min":5.7530000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":71.144999999999996,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.94499999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":3.5550000000000002,"15min":3.589},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.3170000000000002,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.766,"5min":5.2670000000000003,"15min":4.4470000000000001},"min":{"1min":0.36899999999999999,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":14.858000000000001,"5min":171.18100000000001,"15min":223.28399999999999},"last":0.81100000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":3.5470000000000002,"15min":2.274},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":7.3440000000000003,"5min":171.303,"15min":171.303},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.90500000000000003,"5min":3.54,"15min":2.3519999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.359,"5min":171.203,"15min":171.203},"last":0.621}]},{"osd":6,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":3.5830000000000002,"15min":2.4870000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.4169999999999998,"5min":171.441,"15min":171.441},"last":0.68799999999999994},{"interface":"front","average":{"1min":2.9220000000000002,"5min":5.2729999999999997,"15min":3.3340000000000001},"min":{"1min":0.45900000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":14.893000000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":0.76500000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":6.7229999999999999,"15min":4.4020000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":7.383,"5min":219.131,"15min":219.131},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.94099999999999995,"5min":6.6310000000000002,"15min":4.46},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.4489999999999998,"5min":219.19,"15min":219.19},"last":0.55000000000000004}]}]},{"osd":3,"up_from":250,"seq":1073741824151,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1079604,"kb_used_data":569708,"kb_used_omap":7,"kb_used_meta":509880,"kb_avail":92665548,"statfs":{"total":95995035648,"available":94889521152,"internally_reserved":0,"allocated":583380992,"data_stored":582016166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":522117139},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":2.863,"15min":4.4119999999999999},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":14.771000000000001,"5min":25.512,"15min":105.09},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.8049999999999999,"5min":2.8460000000000001,"15min":4.2249999999999996},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.5730000000000004,"5min":25.472999999999999,"15min":99.852999999999994},"last":0.499}]},{"osd":1,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":2.9860000000000002,"15min":3.9399999999999999},"min":{"1min":0.27600000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.249000000000001,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.58899999999999997},{"interface":"front","average":{"1min":3.5209999999999999,"5min":3.6240000000000001,"15min":3.9129999999999998},"min":{"1min":0.29699999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":11.846,"5min":41.18,"15min":94.691000000000003},"last":0.626}]},{"osd":2,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":4.8609999999999998,"5min":4.7050000000000001,"15min":7.2060000000000004},"min":{"1min":0.33500000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.452000000000002,"5min":231.07900000000001,"15min":231.07900000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":5.7720000000000002,"5min":6.1849999999999996,"15min":8.7609999999999992},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.22800000000000001},"max":{"1min":18.733000000000001,"5min":231.58600000000001,"15min":231.58600000000001},"last":0.53700000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.2729999999999999,"15min":1.407},"min":{"1min":0.433,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.1909999999999998,"5min":10.368,"15min":13.417},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.9119999999999999,"5min":2.1589999999999998,"15min":2.5739999999999998},"min":{"1min":0.35599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.722,"5min":19.244,"15min":19.728000000000002},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":1.8,"15min":1.583},"min":{"1min":0.41299999999999998,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":9.8680000000000003,"5min":17.298999999999999,"15min":17.298999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.379,"15min":1.506},"min":{"1min":0.434,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":8.5109999999999992,"5min":10.414999999999999,"15min":13.470000000000001},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.2170000000000001,"15min":2.609},"min":{"1min":0.42899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.691000000000001,"5min":19.164999999999999,"15min":19.805},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.5920000000000001,"5min":1.7909999999999999,"15min":1.615},"min":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":9.9719999999999995,"5min":17.327000000000002,"15min":17.327000000000002},"last":0.84699999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:40:27 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":2.194,"15min":2.8319999999999999},"min":{"1min":0.48299999999999998,"5min":0.379,"15min":0.33900000000000002},"max":{"1min":10.648999999999999,"5min":19.193000000000001,"15min":79.984999999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.6349999999999998,"5min":1.897,"15min":1.865},"min":{"1min":0.44800000000000001,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":9.9380000000000006,"5min":17.251000000000001,"15min":80.272999999999996},"last":0.82899999999999996}]}]},{"osd":4,"up_from":0,"seq":128849019417,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823189,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782932,"kb_used_data":153036,"kb_used_omap":0,"kb_used_meta":629888,"kb_avail":92962220,"statfs":{"total":95995035648,"available":95193313280,"internally_reserved":0,"allocated":156708864,"data_stored":156022166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":645005312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.655,"15min":3.2349999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.4530000000000003,"5min":18.562000000000001,"15min":134.71899999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.121,"15min":3.4710000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2709999999999999,"5min":27.256,"15min":135.386},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":5.0019999999999998,"5min":2.5059999999999998,"15min":2.6320000000000001},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":60.399999999999999,"5min":60.399999999999999,"15min":71.349000000000004},"last":0.42799999999999999},{"interface":"front","average":{"1min":5.25,"5min":2.319,"15min":2.5899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":59.780999999999999,"5min":59.780999999999999,"15min":71.331999999999994},"last":1.746}]},{"osd":2,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":5.9020000000000001,"15min":6.3079999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.789999999999999,"5min":324.21600000000001,"15min":324.21600000000001},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.9670000000000001,"5min":5.54,"15min":6.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.0950000000000006,"5min":324.16300000000001,"15min":324.16300000000001},"last":0.63200000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":4.5609999999999999,"15min":4.3949999999999996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.2510000000000003,"5min":90.725999999999999,"15min":128.089},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.9390000000000001,"15min":4.3959999999999999},"min":{"1min":0.39000000000000001,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":5.907,"5min":90.180000000000007,"15min":128.24299999999999},"last":2.2719999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.47899999999999998,"15min":1.135},"min":{"1min":0.27900000000000003,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":5.3460000000000001,"5min":5.3460000000000001,"15min":165.73699999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":1.242},"min":{"1min":0.31,"5min":0.25,"15min":0.25},"max":{"1min":1.266,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.498}]},{"osd":6,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.60599999999999998,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.22},"max":{"1min":3.4580000000000002,"5min":5.3739999999999997,"15min":29.760000000000002},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.751,"5min":0.61399999999999999,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.29199999999999998},"max":{"1min":3.4340000000000002,"5min":4.3380000000000001,"15min":28.995999999999999},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.6359999999999999,"15min":1.46},"min":{"1min":0.33900000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":1.179,"5min":94.102000000000004,"15min":136.904},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6850000000000001,"15min":1.4930000000000001},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":2.754,"5min":95.653999999999996,"15min":137.661},"last":0.59499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:41:33.986 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:41:34.088 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:34.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:34.195 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:34.223 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:34.266 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:34.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:34.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:34.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:34.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2829760839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:34.351 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[180297]: pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:34.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:34.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:34.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2829760839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:34.352 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:34 smithi064 ceph-mon[186706]: pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:34.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:34.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2829760839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:34.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:34 smithi152 ceph-mon[156819]: pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:34.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:34.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:34.921 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:34.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:35.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:35.078 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:35.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:35.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:35.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:35.212 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:35.217 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-06-10T21:41:35.258 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:35.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:35.362 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:35.392 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2157204063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2157204063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:35.435 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:41:35.436 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:41:35.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2157204063' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:35.456 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:41:36.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:41:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[186706]: failed to encode map e377 with expected crc 2023-06-10T21:41:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:41:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[186706]: osdmap e377: 8 total, 8 up, 4 in 2023-06-10T21:41:36.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[186706]: pgmap v1384: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[180297]: failed to encode map e377 with expected crc 2023-06-10T21:41:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:41:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[180297]: osdmap e377: 8 total, 8 up, 4 in 2023-06-10T21:41:36.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:36 smithi064 ceph-mon[180297]: pgmap v1384: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:36.436 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:36.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:36 smithi152 ceph-mon[156819]: failed to encode map e377 with expected crc 2023-06-10T21:41:36.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:41:36.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:36 smithi152 ceph-mon[156819]: osdmap e377: 8 total, 8 up, 4 in 2023-06-10T21:41:36.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:36 smithi152 ceph-mon[156819]: pgmap v1384: 89 pgs: 89 active+clean; 582 KiB data, 3.2 GiB used, 354 GiB / 358 GiB avail 2023-06-10T21:41:36.525 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:36.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:36.629 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:36.659 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:36.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:41:37.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:37.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:37.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:37.113 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:37.142 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:37.186 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:37.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:37.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:37.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:37 smithi152 ceph-mon[156819]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:37 smithi152 ceph-mon[156819]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.316 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:37.341 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-06-10T21:41:37.343 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:41:37.343 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:41:37.389 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:41:37.389 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:41:37.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_501823 2023-06-10T21:41:37.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:41:37.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:41:37.431 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:37.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:37.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:37.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:37 smithi064 ceph-mon[180297]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:37 smithi064 ceph-mon[180297]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:37 smithi064 ceph-mon[186706]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:37 smithi064 ceph-mon[186706]: failed to encode map e377 with expected crc 2023-06-10T21:41:37.540 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:37.596 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:37.679 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:37.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:37.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:37.839 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:37.925 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:37.930 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:37.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:38.039 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:38.097 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:41:38.097 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:41:38.106 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:38.351 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:38 smithi064 ceph-mon[186706]: pgmap v1385: 89 pgs: 89 active+clean; 2.0 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 184 KiB/s wr, 2 op/s 2023-06-10T21:41:38.352 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:38 smithi064 ceph-mon[180297]: pgmap v1385: 89 pgs: 89 active+clean; 2.0 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 184 KiB/s wr, 2 op/s 2023-06-10T21:41:38.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 16 3210 3194 199.613 199.625 0.00557031 0.00496691 2023-06-10T21:41:38.408 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:38.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:38.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:38 smithi152 ceph-mon[156819]: pgmap v1385: 89 pgs: 89 active+clean; 2.0 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 184 KiB/s wr, 2 op/s 2023-06-10T21:41:38.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:38.572 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:38.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:38.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:38.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:38.890 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:41:38.890 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1972,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":377,"num_osds":8,"num_up_osds":8,"osd_up_since":1686432536,"num_in_osds":4,"osd_in_since":1686433189,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":47,"data_bytes":2103182,"bytes_used":3395764224,"bytes_avail":380584378368,"bytes_total":383980142592,"write_bytes_sec":188368,"read_op_per_sec":0,"write_op_per_sec":2},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":28,"modified":"2023-06-10T21:39:46.060793+0000","services":{}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-10T21:41:38.935 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:39.040 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:39.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:39.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:39.201 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:39.262 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:41:39.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6236 6220 194.355 189.125 0.00332837 0.00488741 2023-06-10T21:41:39.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:41:39.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/482039345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:41:39.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:39 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:39.569 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:41:39.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:41:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/482039345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:41:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:41:39.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/482039345' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:41:39.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:39 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:41:40.226 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:40.390 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 15 9270 9255 192.793 189.688 0.00500303 0.00518086 2023-06-10T21:41:40.490 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: osdmap e378: 8 total, 8 up, 4 in 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.491 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:40 smithi152 ceph-mon[156819]: pgmap v1387: 89 pgs: 89 active+clean; 242 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 30 MiB/s wr, 482 op/s 2023-06-10T21:41:40.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:40.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: osdmap e378: 8 total, 8 up, 4 in 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[186706]: pgmap v1387: 89 pgs: 89 active+clean; 242 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 30 MiB/s wr, 482 op/s 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3982903148' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: osdmap e378: 8 total, 8 up, 4 in 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: failed to encode map e378 with expected crc 2023-06-10T21:41:40.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:40 smithi064 ceph-mon[180297]: pgmap v1387: 89 pgs: 89 active+clean; 242 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 30 MiB/s wr, 482 op/s 2023-06-10T21:41:41.391 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 12253 12237 191.146 186.375 0.00603547 0.00521765 2023-06-10T21:41:41.504 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:41.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:41.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:41 smithi064 ceph-mon[180297]: failed to encode map e378 with expected crc 2023-06-10T21:41:41.510 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:41 smithi064 ceph-mon[186706]: failed to encode map e378 with expected crc 2023-06-10T21:41:41.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:41.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:41.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:41 smithi152 ceph-mon[156819]: failed to encode map e378 with expected crc 2023-06-10T21:41:42.391 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 14829 14813 185.113 161 0.0035929 0.00539587 2023-06-10T21:41:42.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:42 smithi064 ceph-mon[186706]: pgmap v1388: 89 pgs: 89 active+clean; 395 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 49 MiB/s wr, 789 op/s 2023-06-10T21:41:42.585 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:42 smithi064 ceph-mon[180297]: pgmap v1388: 89 pgs: 89 active+clean; 395 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 49 MiB/s wr, 789 op/s 2023-06-10T21:41:42.618 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-06-10T21:41:42.619 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:41:42.665 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:42.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:42 smithi152 ceph-mon[156819]: pgmap v1388: 89 pgs: 89 active+clean; 395 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 49 MiB/s wr, 789 op/s 2023-06-10T21:41:42.774 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:42.845 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:43.019 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:43.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:43.128 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:42] "GET /metrics HTTP/1.1" 200 224065 "" "Prometheus/2.33.4" 2023-06-10T21:41:43.175 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:43.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:43.211 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:43.248 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:43.291 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:43.291 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:43.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:43.331 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:43.357 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:43.369 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:43.392 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17257 17241 179.518 151.75 0.00532888 0.00556456 2023-06-10T21:41:43.399 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:43.408 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:43.441 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.453 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.454 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:41:43.462 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:43.477 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2335889870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:43.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2335889870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:43.564 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:43.567 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2335889870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:43.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:43.610 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:41:43.611 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:41:43.611 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:41:43.611 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:41:43.611 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:41:43.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:43.726 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:43.871 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:43.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:43.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:44.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:44.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:44.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:44.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:44.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:44.285 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:44.374 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:44.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:44.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:44.393 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 19745 19729 176.069 155.5 0.00509238 0.0056743 2023-06-10T21:41:44.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:44.526 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:44 smithi064 ceph-mon[186706]: pgmap v1389: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:41:44.526 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:44 smithi064 ceph-mon[180297]: pgmap v1389: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:41:44.529 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:44.627 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:44.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:44.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:41:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:44 smithi152 ceph-mon[156819]: pgmap v1389: 89 pgs: 89 active+clean; 1.0 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:41:44.732 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:44.824 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:44.963 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:44.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:45.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:45.118 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:45.276 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:45.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:45.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:41:45.385 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:45.393 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 22086 22070 172.349 146.312 0.00622039 0.00579614 2023-06-10T21:41:45.436 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:45.836 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:46.189 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:46.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:46.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:46.297 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:46.304 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:46.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:46.393 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 23830 23814 165.311 109 0.0138994 0.00604299 2023-06-10T21:41:46.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:46.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:46.560 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:46.623 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:46.697 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:46.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:46.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:46.803 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:46.846 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:46.937 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:46.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:46.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:47.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:47.103 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:47.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:47 smithi152 ceph-mon[156819]: pgmap v1390: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-06-10T21:41:47.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:47.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:47.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:47.217 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:41:47.226 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:41:47.315 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:47.364 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:47.393 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 25894 25878 161.68 129 0.00352886 0.00617894 2023-06-10T21:41:47.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:47 smithi064 ceph-mon[180297]: pgmap v1390: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-06-10T21:41:47.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:47 smithi064 ceph-mon[186706]: pgmap v1390: 89 pgs: 89 active+clean; 1.2 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-06-10T21:41:47.460 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:47.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:47.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:47.565 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:47.624 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:47.716 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:47.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:47.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:47.877 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:47.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:41:48.158 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:48.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:48.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:48.320 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:48.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 27881 27865 158.27 124.188 0.00723251 0.00627658 2023-06-10T21:41:48.806 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:41:48.806 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:41:48.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157960 12.c 12:3ac2c392:::benchmark_data_smithi064_442287_object69910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.335923+0000", 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.455940952999995, 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0602751260000001, 2023-06-10T21:41:48.807 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157960 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335923+0000", 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335923+0000", 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335922+0000", 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2320000000000007e-06 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335931+0000", 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0492000000000001e-05 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335942+0000", 2023-06-10T21:41:48.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2800000000000003e-06 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.335947+0000", 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7646999999999998e-05 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.336015+0000", 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3001e-05 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.336028+0000", 2023-06-10T21:41:48.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0276030190000001 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.363631+0000", 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012513499999999999 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.814 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.363756+0000", 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036831599999999999 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.364124+0000", 2023-06-10T21:41:48.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01851918 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382643+0000", 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013186557999999999 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395830+0000", 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00031238099999999998 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396142+0000", 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9200000000000002e-05 2023-06-10T21:41:48.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396171+0000", 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7025e-05 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396198+0000", 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0602751260000001 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.819 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157972 12.c 12:32176147:::benchmark_data_smithi064_442287_object69922:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.366721+0000", 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.425143278999997, 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.029551232, 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.820 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157972 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366721+0000", 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366721+0000", 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366720+0000", 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7309999999999996e-06 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366726+0000", 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3499999999999999e-05 2023-06-10T21:41:48.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366740+0000", 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8999999999999997e-06 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366745+0000", 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3279000000000003e-05 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366798+0000", 2023-06-10T21:41:48.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8850000000000001e-06 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.366805+0000", 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0097715819999999 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.376576+0000", 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3455999999999995e-05 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.376660+0000", 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9279999999999999e-05 2023-06-10T21:41:48.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.376749+0000", 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005897663 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382647+0000", 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013212776000000001 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395860+0000", 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036663499999999998 2023-06-10T21:41:48.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396226+0000", 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.476e-05 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396251+0000", 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1166e-05 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396272+0000", 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.029551232 2023-06-10T21:41:48.831 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157974 12.c 12:37161017:::benchmark_data_smithi064_442287_object69924:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.369743+0000", 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.422121571999995, 2023-06-10T21:41:48.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.026646409, 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157974 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.833 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369743+0000", 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369743+0000", 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369742+0000", 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4909999999999996e-06 2023-06-10T21:41:48.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369747+0000", 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3283e-05 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369761+0000", 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3210000000000004e-06 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369766+0000", 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2403000000000005e-05 2023-06-10T21:41:48.837 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369838+0000", 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3759999999999996e-06 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.369845+0000", 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.007922669 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.377767+0000", 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2959999999999994e-05 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.377830+0000", 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2813999999999998e-05 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.377903+0000", 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.004748515 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382652+0000", 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013237254 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395889+0000", 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045718100000000003 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396346+0000", 2023-06-10T21:41:48.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3859e-05 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396370+0000", 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9131e-05 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396389+0000", 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.026646409 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157987 12.c 12:3c656fb8:::benchmark_data_smithi064_442287_object69937:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.387723+0000", 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.404141553000002, 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.008747007, 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.845 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157987 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387723+0000", 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387723+0000", 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387721+0000", 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.365e-06 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387727+0000", 2023-06-10T21:41:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0115000000000001e-05 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387737+0000", 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5980000000000002e-06 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387744+0000", 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3472000000000003e-05 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387827+0000", 2023-06-10T21:41:48.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1959e-05 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.387839+0000", 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99093165599999999 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.378771+0000", 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3732999999999994e-05 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.378845+0000", 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000102605 2023-06-10T21:41:48.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.378947+0000", 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067679319999999999 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.385715+0000", 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010195119000000001 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395910+0000", 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00050434500000000001 2023-06-10T21:41:48.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396415+0000", 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8377e-05 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396443+0000", 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6548000000000001e-05 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396470+0000", 2023-06-10T21:41:48.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.008747007 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157994 12.c 12:3d1f418d:::benchmark_data_smithi064_442287_object69944:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.407828+0000", 2023-06-10T21:41:48.857 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.384036015000007, 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98875809100000001, 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.858 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157994 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407828+0000", 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407828+0000", 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407826+0000", 2023-06-10T21:41:48.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0278000000000001e-05 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407836+0000", 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1637999999999998e-05 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407858+0000", 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9659999999999996e-06 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407866+0000", 2023-06-10T21:41:48.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3452000000000001e-05 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407939+0000", 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0436e-05 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407950+0000", 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97188242599999997 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.379832+0000", 2023-06-10T21:41:48.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000115037 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.379947+0000", 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015224999999999999 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.380099+0000", 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056225670000000002 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.385722+0000", 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010210067999999999 2023-06-10T21:41:48.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395932+0000", 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00061125899999999996 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396543+0000", 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3485999999999998e-05 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396567+0000", 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9686000000000001e-05 2023-06-10T21:41:48.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396586+0000", 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.98875809100000001 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:157995 12.c 12:3d54d5e1:::benchmark_data_smithi064_442287_object69945:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.407884+0000", 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.383980355000006, 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.988781617, 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.870 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 157995 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407884+0000", 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407884+0000", 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:48.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407883+0000", 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.326e-06 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407887+0000", 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6439999999999997e-06 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407892+0000", 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9480000000000001e-06 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.407896+0000", 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00048042899999999999 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.408377+0000", 2023-06-10T21:41:48.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0457e-05 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.408387+0000", 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97312061599999999 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.381508+0000", 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6970000000000003e-05 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.381585+0000", 2023-06-10T21:41:48.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2563000000000001e-05 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.381677+0000", 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0066375699999999998 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.388315+0000", 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076401619999999998 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395955+0000", 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066344600000000004 2023-06-10T21:41:48.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396618+0000", 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8028e-05 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396646+0000", 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9046999999999999e-05 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396666+0000", 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.988781617 2023-06-10T21:41:48.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158001 12.c 12:341268c6:::benchmark_data_smithi064_442287_object69951:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.421151+0000", 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.370713112000004, 2023-06-10T21:41:48.882 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97783773100000004, 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158001 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.883 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421151+0000", 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.884 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421151+0000", 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421150+0000", 2023-06-10T21:41:48.885 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1019999999999999e-06 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421157+0000", 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4336e-05 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.886 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421171+0000", 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6669999999999996e-06 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421179+0000", 2023-06-10T21:41:48.887 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7892999999999994e-05 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421257+0000", 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1146e-05 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.888 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421268+0000", 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96142141800000003 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382689+0000", 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104169 2023-06-10T21:41:48.889 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382793+0000", 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012978999999999999 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.890 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.382923+0000", 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053986579999999998 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.388322+0000", 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076626130000000004 2023-06-10T21:41:48.891 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395984+0000", 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029014179999999998 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.892 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.398886+0000", 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0312000000000002e-05 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.398936+0000", 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2753999999999997e-05 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.893 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.398989+0000", 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97783773100000004 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.894 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158002 12.c 12:3f9e54e2:::benchmark_data_smithi064_442287_object69952:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.421197+0000", 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.370667346999994, 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97787884899999999, 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.895 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158002 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.896 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421197+0000", 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421197+0000", 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:48.897 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421196+0000", 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4429999999999999e-06 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.898 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421200+0000", 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3529999999999999e-05 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421213+0000", 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5860000000000004e-06 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.899 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421219+0000", 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000462196 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421681+0000", 2023-06-10T21:41:48.900 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0424e-05 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.421691+0000", 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96239929400000002 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.901 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.384091+0000", 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8712000000000001e-05 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.384169+0000", 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000141602 2023-06-10T21:41:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.384311+0000", 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0076008710000000004 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.391912+0000", 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0040961560000000001 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396008+0000", 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030211539999999999 2023-06-10T21:41:48.904 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399029+0000", 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6781999999999999e-05 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.905 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399056+0000", 2023-06-10T21:41:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9707e-05 2023-06-10T21:41:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.907 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399076+0000", 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.97787884899999999 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.908 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158010 12.c 12:3d18baaf:::benchmark_data_smithi064_442287_object69960:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.427661+0000", 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.364202840999994, 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.971550629, 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.909 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158010 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.910 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427661+0000", 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427661+0000", 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.911 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427660+0000", 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3229999999999998e-06 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427668+0000", 2023-06-10T21:41:48.912 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5299999999999999e-05 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427683+0000", 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0849999999999999e-06 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.913 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427690+0000", 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0171000000000003e-05 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427761+0000", 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1043000000000001e-05 2023-06-10T21:41:48.914 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427772+0000", 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95798573600000003 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.915 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.385757+0000", 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7043000000000001e-05 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.385834+0000", 2023-06-10T21:41:48.916 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0204999999999994e-05 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.385925+0000", 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005992575 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.917 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.391917+0000", 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041068500000000004 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396024+0000", 2023-06-10T21:41:48.918 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003122611 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399147+0000", 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2976000000000003e-05 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.919 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399190+0000", 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2399999999999999e-05 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399212+0000", 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.971550629 2023-06-10T21:41:48.920 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158012 12.c 12:3941ffe3:::benchmark_data_smithi064_442287_object69962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.427707+0000", 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.364157156999994, 2023-06-10T21:41:48.921 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.971586913, 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158012 2023-06-10T21:41:48.922 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427707+0000", 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.923 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427707+0000", 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427707+0000", 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4319999999999999e-06 2023-06-10T21:41:48.924 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427710+0000", 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.255e-06 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.925 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427717+0000", 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4859999999999999e-06 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.427722+0000", 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00046923599999999999 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.926 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.428191+0000", 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3769999999999996e-06 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.428199+0000", 2023-06-10T21:41:48.927 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95892900700000006 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.387128+0000", 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9482999999999999e-05 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.928 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.387198+0000", 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7047999999999998e-05 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.387275+0000", 2023-06-10T21:41:48.929 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071160160000000002 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394391+0000", 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017080820000000001 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.930 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396099+0000", 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031464409999999998 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399245+0000", 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7854000000000001e-05 2023-06-10T21:41:48.931 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399273+0000", 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0754000000000002e-05 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.932 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.399294+0000", 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.971586913 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.933 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158041 12.c 12:34c7c365:::benchmark_data_smithi064_442287_object69991:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.495558+0000", 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.296305856000004, 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90761042199999997, 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.934 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158041 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495558+0000", 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.935 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495558+0000", 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.936 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495557+0000", 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0469999999999999e-06 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495563+0000", 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2582000000000001e-05 2023-06-10T21:41:48.937 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495576+0000", 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2033000000000001e-05 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.938 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495598+0000", 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7876e-05 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495646+0000", 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.17e-06 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.939 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.495653+0000", 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.89270719399999998 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.388360+0000", 2023-06-10T21:41:48.940 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5767999999999998e-05 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.388426+0000", 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6660000000000001e-05 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.941 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.388502+0000", 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005896592 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394399+0000", 2023-06-10T21:41:48.942 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0017225509999999999 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396122+0000", 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069523049999999998 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.943 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.403074+0000", 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3090000000000002e-05 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.403127+0000", 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1922999999999998e-05 2023-06-10T21:41:48.944 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.403169+0000", 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90761042199999997 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.945 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158054 12.c 12:3f51d3de:::benchmark_data_smithi064_442287_object70004:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.543537+0000", 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.248327392999997, 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86104665999999996, 2023-06-10T21:41:48.946 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158054 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.947 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543537+0000", 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.948 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543537+0000", 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543536+0000", 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3890000000000004e-06 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.949 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543541+0000", 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2587e-05 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543554+0000", 2023-06-10T21:41:48.950 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7840000000000003e-06 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543558+0000", 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7094000000000005e-05 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.951 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543625+0000", 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4919999999999993e-06 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.543634+0000", 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84596719799999998 2023-06-10T21:41:48.952 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.389601+0000", 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012283300000000001 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.953 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.389724+0000", 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016699099999999999 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.389891+0000", 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0045168059999999999 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.954 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394408+0000", 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018930380000000001 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.955 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396301+0000", 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082045360000000001 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404505+0000", 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2138999999999998e-05 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.956 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404547+0000", 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6034000000000001e-05 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404584+0000", 2023-06-10T21:41:48.957 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86104665999999996 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158058 12.c 12:3329a548:::benchmark_data_smithi064_442287_object70008:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.564010+0000", 2023-06-10T21:41:48.958 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.227853796000005, 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84068279000000001, 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158058 2023-06-10T21:41:48.959 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564010+0000", 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.960 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564010+0000", 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999967 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564007+0000", 2023-06-10T21:41:48.961 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.668e-05 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564024+0000", 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6738e-05 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.962 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564060+0000", 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1379999999999998e-06 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564065+0000", 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0390999999999999e-05 2023-06-10T21:41:48.963 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564116+0000", 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3659999999999992e-06 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.964 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.564124+0000", 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82678187800000003 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.390906+0000", 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3571000000000003e-05 2023-06-10T21:41:48.965 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.390990+0000", 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3177999999999995e-05 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.966 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.391073+0000", 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044658379999999998 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395539+0000", 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00078532799999999996 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.967 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396324+0000", 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082949100000000008 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404619+0000", 2023-06-10T21:41:48.968 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8884999999999998e-05 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404658+0000", 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5432999999999998e-05 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.969 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.404693+0000", 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84068279000000001 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.970 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158063 12.c 12:3554050c:::benchmark_data_smithi064_442287_object70013:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.582053+0000", 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.209811431999995, 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82328223700000003, 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.971 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158063 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582053+0000", 2023-06-10T21:41:48.972 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582053+0000", 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.973 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582051+0000", 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4729999999999997e-06 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582058+0000", 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3393e-05 2023-06-10T21:41:48.974 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582071+0000", 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1869999999999996e-06 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.975 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582076+0000", 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0266000000000003e-05 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582127+0000", 2023-06-10T21:41:48.976 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6829999999999997e-06 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.582135+0000", 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80981832399999998 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.977 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.979 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.391954+0000", 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2575000000000001e-05 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.392036+0000", 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7383000000000001e-05 2023-06-10T21:41:48.980 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.392114+0000", 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034345030000000002 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.981 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395548+0000", 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094948099999999996 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396498+0000", 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087422139999999999 2023-06-10T21:41:48.982 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405240+0000", 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8585999999999999e-05 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.983 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405288+0000", 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6578e-05 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405335+0000", 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.82328223700000003 2023-06-10T21:41:48.984 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158068 12.c 12:3d6279ca:::benchmark_data_smithi064_442287_object70018:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.604673+0000", 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.187191631999994, 2023-06-10T21:41:48.985 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80077622100000001, 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158068 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.986 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604673+0000", 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:48.987 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604673+0000", 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604671+0000", 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5480000000000004e-06 2023-06-10T21:41:48.988 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604679+0000", 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4674999999999999e-05 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:48.989 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604693+0000", 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8799999999999996e-06 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604699+0000", 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9899000000000003e-05 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.990 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604769+0000", 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3499999999999999e-05 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:48.991 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.604783+0000", 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.788480654 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.393263+0000", 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116513 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.992 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.393380+0000", 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147491 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.393527+0000", 2023-06-10T21:41:48.993 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022636650000000002 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395791+0000", 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00073042300000000001 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.994 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396521+0000", 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0088475859999999993 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405369+0000", 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3899000000000001e-05 2023-06-10T21:41:48.995 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405413+0000", 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6069000000000003e-05 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:48.996 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405449+0000", 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.80077622100000001 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.997 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:158070 12.c 12:38b97b1c:::benchmark_data_smithi064_442287_object70020:head [delete] snapc 0=[] ondisk+write+known_if_redirected e363)", 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:35.615112+0000", 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 73.176752508000007, 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79039863399999999, 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:41:48.998 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 158070 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615112+0000", 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:48.999 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615112+0000", 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:49.000 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615110+0000", 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6899999999999992e-06 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615118+0000", 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6864999999999999e-05 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.001 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615135+0000", 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4880000000000001e-06 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615142+0000", 2023-06-10T21:41:49.002 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1678000000000002e-05 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615194+0000", 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4020000000000008e-06 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.003 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:35.615203+0000", 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.77924099499999999 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394444+0000", 2023-06-10T21:41:49.004 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1818999999999998e-05 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394516+0000", 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4357000000000006e-05 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.005 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.394600+0000", 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00126603 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.395867+0000", 2023-06-10T21:41:49.006 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00082607299999999998 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.396693+0000", 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0087756699999999993 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.007 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405468+0000", 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0475999999999998e-05 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405489+0000", 2023-06-10T21:41:49.008 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1633000000000001e-05 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:36.405510+0000", 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.79039863399999999 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.009 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:22413 13.9 13:9a7a4c5d:::benchmark_data_smithi064_501823_object22412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:41:45.535074+0000", 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2567906249999998, 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20927949300000001, 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:49.010 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 22413 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.011 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535074+0000", 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535074+0000", 2023-06-10T21:41:49.012 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535072+0000", 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2577000000000003e-05 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.013 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535105+0000", 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2673999999999999e-05 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535117+0000", 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3990000000000001e-06 2023-06-10T21:41:49.014 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535124+0000", 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5359999999999998e-05 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.015 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535189+0000", 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000130615 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535320+0000", 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015796399999999999 2023-06-10T21:41:49.016 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535478+0000", 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021054340000000002 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:49.017 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537583+0000", 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024628229999999998 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.540046+0000", 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20419585000000001 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.018 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744242+0000", 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6172999999999999e-05 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744298+0000", 2023-06-10T21:41:49.019 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5096000000000002e-05 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744353+0000", 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20927949300000001 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.020 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:22414 13.5 13:ac5cb05e:::benchmark_data_smithi064_501823_object22413:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:41:45.535141+0000", 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2567237379999998, 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.207939228, 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:49.021 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 22414 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:49.022 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535141+0000", 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535141+0000", 2023-06-10T21:41:49.023 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535140+0000", 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6185000000000001e-05 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.024 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535166+0000", 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3000000000000004e-06 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535173+0000", 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3959999999999999e-06 2023-06-10T21:41:49.025 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535178+0000", 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000231303 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.026 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535409+0000", 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000108937 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535518+0000", 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014162300000000001 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.027 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.535660+0000", 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001392994 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537053+0000", 2023-06-10T21:41:49.028 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001339851 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.538393+0000", 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20456865699999999 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.029 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.742961+0000", 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3720000000000006e-05 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.743035+0000", 2023-06-10T21:41:49.030 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4851000000000001e-05 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.743080+0000", 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.207939228 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.031 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:22418 13.9 13:94ea532f:::benchmark_data_smithi064_501823_object22417:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:41:45.537542+0000", 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2543225740000001, 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20693848200000001, 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:49.032 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 22418 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:49.033 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537542+0000", 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537542+0000", 2023-06-10T21:41:49.034 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537540+0000", 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5989999999999997e-05 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.035 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537606+0000", 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2449999999999999e-05 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:49.036 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537619+0000", 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5560000000000004e-06 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537625+0000", 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7661000000000003e-05 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.037 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537673+0000", 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015364099999999999 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537826+0000", 2023-06-10T21:41:49.038 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143456 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537970+0000", 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021113669999999998 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.039 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.540081+0000", 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001387587 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.541469+0000", 2023-06-10T21:41:49.040 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.202920712 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744390+0000", 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8656999999999998e-05 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.041 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744438+0000", 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1974000000000002e-05 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.744480+0000", 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20693848200000001 2023-06-10T21:41:49.042 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:22421 13.2 13:4ee66a26:::benchmark_data_smithi064_501823_object22420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e378)", 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:41:45.537751+0000", 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 3.2541132780000002, 2023-06-10T21:41:49.043 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20852878, 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 22421 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.044 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537751+0000", 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.045 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537751+0000", 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537750+0000", 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6606e-05 2023-06-10T21:41:49.046 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537807+0000", 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3840000000000003e-06 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:41:49.047 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537816+0000", 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1940000000000003e-06 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.537822+0000", 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027251400000000001 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.048 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.538095+0000", 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119986 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.538215+0000", 2023-06-10T21:41:49.049 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014581800000000001 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.538361+0000", 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025251309999999999 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.050 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.540886+0000", 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066248800000000003 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.541548+0000", 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20465076200000001 2023-06-10T21:41:49.051 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.746199+0000", 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4982999999999999e-05 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:41:49.052 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.746244+0000", 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5629000000000001e-05 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:41:45.746280+0000", 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20852878 2023-06-10T21:41:49.053 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.054 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.054 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:41:49.054 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:41:49.054 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:41:49.055 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:49.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:49.056 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1390,"stamp":"2023-06-10T21:41:46.126334+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1280841614,"num_objects":19559,"num_object_clones":0,"num_object_copies":58677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19559,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":19796,"num_write_kb":1252686,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":2379958,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19730,"ondisk_log_size":19730,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":6635924,"kb_used_data":3874292,"kb_used_omap":15,"kb_used_meta":2761584,"kb_avail":368344684,"statfs":{"total":383980142592,"available":377184956416,"internally_reserved":0,"allocated":3967275008,"data_stored":3963235397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16284,"internal_metadata":2827862116},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1280245760,"num_objects":19535,"num_object_clones":0,"num_object_copies":58605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19535,"num_write_kb":1250240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002491"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":401,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.710745+0000","last_change":"2023-06-10T21:39:50.557548+0000","last_active":"2023-06-10T21:41:36.710745+0000","last_peered":"2023-06-10T21:41:36.710745+0000","last_clean":"2023-06-10T21:41:36.710745+0000","last_became_active":"2023-06-10T21:39:50.557023+0000","last_became_peered":"2023-06-10T21:39:50.557023+0000","last_unstale":"2023-06-10T21:41:36.710745+0000","last_undegraded":"2023-06-10T21:41:36.710745+0000","last_fullsized":"2023-06-10T21:41:36.710745+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:45.328782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400433+0000","last_change":"2023-06-10T21:40:30.243387+0000","last_active":"2023-06-10T21:41:36.400433+0000","last_peered":"2023-06-10T21:41:36.400433+0000","last_clean":"2023-06-10T21:41:36.400433+0000","last_became_active":"2023-06-10T21:40:30.243087+0000","last_became_peered":"2023-06-10T21:40:30.243087+0000","last_unstale":"2023-06-10T21:41:36.400433+0000","last_undegraded":"2023-06-10T21:41:36.400433+0000","last_fullsized":"2023-06-10T21:41:36.400433+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:11.139462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":308,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399802+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:41:36.399802+0000","last_peered":"2023-06-10T21:41:36.399802+0000","last_clean":"2023-06-10T21:41:36.399802+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:41:36.399802+0000","last_undegraded":"2023-06-10T21:41:36.399802+0000","last_fullsized":"2023-06-10T21:41:36.399802+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":248,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020379+0000","last_change":"2023-06-10T21:40:30.287586+0000","last_active":"2023-06-10T21:41:36.020379+0000","last_peered":"2023-06-10T21:41:36.020379+0000","last_clean":"2023-06-10T21:41:36.020379+0000","last_became_active":"2023-06-10T21:40:30.286423+0000","last_became_peered":"2023-06-10T21:40:30.286423+0000","last_unstale":"2023-06-10T21:41:36.020379+0000","last_undegraded":"2023-06-10T21:41:36.020379+0000","last_fullsized":"2023-06-10T21:41:36.020379+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399286+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:41:36.399286+0000","last_peered":"2023-06-10T21:41:36.399286+0000","last_clean":"2023-06-10T21:41:36.399286+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:41:36.399286+0000","last_undegraded":"2023-06-10T21:41:36.399286+0000","last_fullsized":"2023-06-10T21:41:36.399286+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":79,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.716018+0000","last_change":"2023-06-10T21:40:30.236647+0000","last_active":"2023-06-10T21:41:36.716018+0000","last_peered":"2023-06-10T21:41:36.716018+0000","last_clean":"2023-06-10T21:41:36.716018+0000","last_became_active":"2023-06-10T21:40:30.236307+0000","last_became_peered":"2023-06-10T21:40:30.236307+0000","last_unstale":"2023-06-10T21:41:36.716018+0000","last_undegraded":"2023-06-10T21:41:36.716018+0000","last_fullsized":"2023-06-10T21:41:36.716018+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715901+0000","last_change":"2023-06-10T21:40:30.280815+0000","last_active":"2023-06-10T21:41:36.715901+0000","last_peered":"2023-06-10T21:41:36.715901+0000","last_clean":"2023-06-10T21:41:36.715901+0000","last_became_active":"2023-06-10T21:40:30.280427+0000","last_became_peered":"2023-06-10T21:40:30.280427+0000","last_unstale":"2023-06-10T21:41:36.715901+0000","last_undegraded":"2023-06-10T21:41:36.715901+0000","last_fullsized":"2023-06-10T21:41:36.715901+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399387+0000","last_change":"2023-06-10T21:40:30.245395+0000","last_active":"2023-06-10T21:41:36.399387+0000","last_peered":"2023-06-10T21:41:36.399387+0000","last_clean":"2023-06-10T21:41:36.399387+0000","last_became_active":"2023-06-10T21:40:30.244982+0000","last_became_peered":"2023-06-10T21:40:30.244982+0000","last_unstale":"2023-06-10T21:41:36.399387+0000","last_undegraded":"2023-06-10T21:41:36.399387+0000","last_fullsized":"2023-06-10T21:41:36.399387+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:27:28.313757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715825+0000","last_change":"2023-06-10T21:40:30.280871+0000","last_active":"2023-06-10T21:41:36.715825+0000","last_peered":"2023-06-10T21:41:36.715825+0000","last_clean":"2023-06-10T21:41:36.715825+0000","last_became_active":"2023-06-10T21:40:30.280332+0000","last_became_peered":"2023-06-10T21:40:30.280332+0000","last_unstale":"2023-06-10T21:41:36.715825+0000","last_undegraded":"2023-06-10T21:41:36.715825+0000","last_fullsized":"2023-06-10T21:41:36.715825+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":112,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399769+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:41:36.399769+0000","last_peered":"2023-06-10T21:41:36.399769+0000","last_clean":"2023-06-10T21:41:36.399769+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:41:36.399769+0000","last_undegraded":"2023-06-10T21:41:36.399769+0000","last_fullsized":"2023-06-10T21:41:36.399769+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.019998+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:41:36.019998+0000","last_peered":"2023-06-10T21:41:36.019998+0000","last_clean":"2023-06-10T21:41:36.019998+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:41:36.019998+0000","last_undegraded":"2023-06-10T21:41:36.019998+0000","last_fullsized":"2023-06-10T21:41:36.019998+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":361,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.710758+0000","last_change":"2023-06-10T21:39:50.557388+0000","last_active":"2023-06-10T21:41:36.710758+0000","last_peered":"2023-06-10T21:41:36.710758+0000","last_clean":"2023-06-10T21:41:36.710758+0000","last_became_active":"2023-06-10T21:39:50.556659+0000","last_became_peered":"2023-06-10T21:39:50.556659+0000","last_unstale":"2023-06-10T21:41:36.710758+0000","last_undegraded":"2023-06-10T21:41:36.710758+0000","last_fullsized":"2023-06-10T21:41:36.710758+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:32.982830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":286,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020034+0000","last_change":"2023-06-10T21:39:30.845741+0000","last_active":"2023-06-10T21:41:36.020034+0000","last_peered":"2023-06-10T21:41:36.020034+0000","last_clean":"2023-06-10T21:41:36.020034+0000","last_became_active":"2023-06-10T21:39:30.234598+0000","last_became_peered":"2023-06-10T21:39:30.234598+0000","last_unstale":"2023-06-10T21:41:36.020034+0000","last_undegraded":"2023-06-10T21:41:36.020034+0000","last_fullsized":"2023-06-10T21:41:36.020034+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":117,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715481+0000","last_change":"2023-06-10T21:39:50.270986+0000","last_active":"2023-06-10T21:41:36.715481+0000","last_peered":"2023-06-10T21:41:36.715481+0000","last_clean":"2023-06-10T21:41:36.715481+0000","last_became_active":"2023-06-10T21:39:50.269843+0000","last_became_peered":"2023-06-10T21:39:50.269843+0000","last_unstale":"2023-06-10T21:41:36.715481+0000","last_undegraded":"2023-06-10T21:41:36.715481+0000","last_fullsized":"2023-06-10T21:41:36.715481+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":422,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020052+0000","last_change":"2023-06-10T21:39:50.781250+0000","last_active":"2023-06-10T21:41:36.020052+0000","last_peered":"2023-06-10T21:41:36.020052+0000","last_clean":"2023-06-10T21:41:36.020052+0000","last_became_active":"2023-06-10T21:39:50.376925+0000","last_became_peered":"2023-06-10T21:39:50.376925+0000","last_unstale":"2023-06-10T21:41:36.020052+0000","last_undegraded":"2023-06-10T21:41:36.020052+0000","last_fullsized":"2023-06-10T21:41:36.020052+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400102+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:41:36.400102+0000","last_peered":"2023-06-10T21:41:36.400102+0000","last_clean":"2023-06-10T21:41:36.400102+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:41:36.400102+0000","last_undegraded":"2023-06-10T21:41:36.400102+0000","last_fullsized":"2023-06-10T21:41:36.400102+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":248,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.019936+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:41:36.019936+0000","last_peered":"2023-06-10T21:41:36.019936+0000","last_clean":"2023-06-10T21:41:36.019936+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:41:36.019936+0000","last_undegraded":"2023-06-10T21:41:36.019936+0000","last_fullsized":"2023-06-10T21:41:36.019936+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":79,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715765+0000","last_change":"2023-06-10T21:40:30.280758+0000","last_active":"2023-06-10T21:41:36.715765+0000","last_peered":"2023-06-10T21:41:36.715765+0000","last_clean":"2023-06-10T21:41:36.715765+0000","last_became_active":"2023-06-10T21:40:30.280270+0000","last_became_peered":"2023-06-10T21:40:30.280270+0000","last_unstale":"2023-06-10T21:41:36.715765+0000","last_undegraded":"2023-06-10T21:41:36.715765+0000","last_fullsized":"2023-06-10T21:41:36.715765+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":390,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.019868+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:41:36.019868+0000","last_peered":"2023-06-10T21:41:36.019868+0000","last_clean":"2023-06-10T21:41:36.019868+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:41:36.019868+0000","last_undegraded":"2023-06-10T21:41:36.019868+0000","last_fullsized":"2023-06-10T21:41:36.019868+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400395+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:41:36.400395+0000","last_peered":"2023-06-10T21:41:36.400395+0000","last_clean":"2023-06-10T21:41:36.400395+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:41:36.400395+0000","last_undegraded":"2023-06-10T21:41:36.400395+0000","last_fullsized":"2023-06-10T21:41:36.400395+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":421,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.714924+0000","last_change":"2023-06-10T21:40:30.476009+0000","last_active":"2023-06-10T21:41:36.714924+0000","last_peered":"2023-06-10T21:41:36.714924+0000","last_clean":"2023-06-10T21:41:36.714924+0000","last_became_active":"2023-06-10T21:40:30.237078+0000","last_became_peered":"2023-06-10T21:40:30.237078+0000","last_unstale":"2023-06-10T21:41:36.714924+0000","last_undegraded":"2023-06-10T21:41:36.714924+0000","last_fullsized":"2023-06-10T21:41:36.714924+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400558+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:41:36.400558+0000","last_peered":"2023-06-10T21:41:36.400558+0000","last_clean":"2023-06-10T21:41:36.400558+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:41:36.400558+0000","last_undegraded":"2023-06-10T21:41:36.400558+0000","last_fullsized":"2023-06-10T21:41:36.400558+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"378'1442","reported_seq":1456,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:45.492601+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:41:45.492601+0000","last_peered":"2023-06-10T21:41:45.492601+0000","last_clean":"2023-06-10T21:41:45.492601+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:41:45.492601+0000","last_undegraded":"2023-06-10T21:41:45.492601+0000","last_fullsized":"2023-06-10T21:41:45.492601+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1442,"log_dups_size":0,"ondisk_log_size":1442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94502912,"num_objects":1442,"num_object_clones":0,"num_object_copies":4326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1442,"num_write_kb":92288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":155,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020481+0000","last_change":"2023-06-10T21:40:30.567200+0000","last_active":"2023-06-10T21:41:36.020481+0000","last_peered":"2023-06-10T21:41:36.020481+0000","last_clean":"2023-06-10T21:41:36.020481+0000","last_became_active":"2023-06-10T21:40:30.285607+0000","last_became_peered":"2023-06-10T21:40:30.285607+0000","last_unstale":"2023-06-10T21:41:36.020481+0000","last_undegraded":"2023-06-10T21:41:36.020481+0000","last_fullsized":"2023-06-10T21:41:36.020481+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.714934+0000","last_change":"2023-06-10T21:40:30.232813+0000","last_active":"2023-06-10T21:41:36.714934+0000","last_peered":"2023-06-10T21:41:36.714934+0000","last_clean":"2023-06-10T21:41:36.714934+0000","last_became_active":"2023-06-10T21:40:30.232432+0000","last_became_peered":"2023-06-10T21:40:30.232432+0000","last_unstale":"2023-06-10T21:41:36.714934+0000","last_undegraded":"2023-06-10T21:41:36.714934+0000","last_fullsized":"2023-06-10T21:41:36.714934+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"378'1259","reported_seq":1272,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.116948+0000","last_change":"2023-06-10T21:41:28.181199+0000","last_active":"2023-06-10T21:41:44.116948+0000","last_peered":"2023-06-10T21:41:44.116948+0000","last_clean":"2023-06-10T21:41:44.116948+0000","last_became_active":"2023-06-10T21:41:28.180841+0000","last_became_peered":"2023-06-10T21:41:28.180841+0000","last_unstale":"2023-06-10T21:41:44.116948+0000","last_undegraded":"2023-06-10T21:41:44.116948+0000","last_fullsized":"2023-06-10T21:41:44.116948+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1259,"log_dups_size":0,"ondisk_log_size":1259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1259,"num_write_kb":80576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":155,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020478+0000","last_change":"2023-06-10T21:40:30.576549+0000","last_active":"2023-06-10T21:41:36.020478+0000","last_peered":"2023-06-10T21:41:36.020478+0000","last_clean":"2023-06-10T21:41:36.020478+0000","last_became_active":"2023-06-10T21:40:30.286869+0000","last_became_peered":"2023-06-10T21:40:30.286869+0000","last_unstale":"2023-06-10T21:41:36.020478+0000","last_undegraded":"2023-06-10T21:41:36.020478+0000","last_fullsized":"2023-06-10T21:41:36.020478+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399905+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:41:36.399905+0000","last_peered":"2023-06-10T21:41:36.399905+0000","last_clean":"2023-06-10T21:41:36.399905+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:41:36.399905+0000","last_undegraded":"2023-06-10T21:41:36.399905+0000","last_fullsized":"2023-06-10T21:41:36.399905+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"378'942","reported_seq":956,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:42.448326+0000","last_change":"2023-06-10T21:41:28.177829+0000","last_active":"2023-06-10T21:41:42.448326+0000","last_peered":"2023-06-10T21:41:42.448326+0000","last_clean":"2023-06-10T21:41:42.448326+0000","last_became_active":"2023-06-10T21:41:28.177535+0000","last_became_peered":"2023-06-10T21:41:28.177535+0000","last_unstale":"2023-06-10T21:41:42.448326+0000","last_undegraded":"2023-06-10T21:41:42.448326+0000","last_fullsized":"2023-06-10T21:41:42.448326+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":942,"log_dups_size":0,"ondisk_log_size":942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:34:42.879929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61734912,"num_objects":942,"num_object_clones":0,"num_object_copies":2826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":942,"num_write_kb":60288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":442,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020130+0000","last_change":"2023-06-10T21:40:30.521312+0000","last_active":"2023-06-10T21:41:36.020130+0000","last_peered":"2023-06-10T21:41:36.020130+0000","last_clean":"2023-06-10T21:41:36.020130+0000","last_became_active":"2023-06-10T21:40:30.285713+0000","last_became_peered":"2023-06-10T21:40:30.285713+0000","last_unstale":"2023-06-10T21:41:36.020130+0000","last_undegraded":"2023-06-10T21:41:36.020130+0000","last_fullsized":"2023-06-10T21:41:36.020130+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020048+0000","last_change":"2023-06-10T21:40:30.343628+0000","last_active":"2023-06-10T21:41:36.020048+0000","last_peered":"2023-06-10T21:41:36.020048+0000","last_clean":"2023-06-10T21:41:36.020048+0000","last_became_active":"2023-06-10T21:40:30.343292+0000","last_became_peered":"2023-06-10T21:40:30.343292+0000","last_unstale":"2023-06-10T21:41:36.020048+0000","last_undegraded":"2023-06-10T21:41:36.020048+0000","last_fullsized":"2023-06-10T21:41:36.020048+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"378'1208","reported_seq":1220,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.054596+0000","last_change":"2023-06-10T21:41:28.177207+0000","last_active":"2023-06-10T21:41:44.054596+0000","last_peered":"2023-06-10T21:41:44.054596+0000","last_clean":"2023-06-10T21:41:44.054596+0000","last_became_active":"2023-06-10T21:41:28.176814+0000","last_became_peered":"2023-06-10T21:41:28.176814+0000","last_unstale":"2023-06-10T21:41:44.054596+0000","last_undegraded":"2023-06-10T21:41:44.054596+0000","last_fullsized":"2023-06-10T21:41:44.054596+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1208,"log_dups_size":0,"ondisk_log_size":1208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79167488,"num_objects":1208,"num_object_clones":0,"num_object_copies":3624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1208,"num_write_kb":77312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":80,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715376+0000","last_change":"2023-06-10T21:40:30.280834+0000","last_active":"2023-06-10T21:41:36.715376+0000","last_peered":"2023-06-10T21:41:36.715376+0000","last_clean":"2023-06-10T21:41:36.715376+0000","last_became_active":"2023-06-10T21:40:30.280353+0000","last_became_peered":"2023-06-10T21:40:30.280353+0000","last_unstale":"2023-06-10T21:41:36.715376+0000","last_undegraded":"2023-06-10T21:41:36.715376+0000","last_fullsized":"2023-06-10T21:41:36.715376+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399972+0000","last_change":"2023-06-10T21:40:30.272439+0000","last_active":"2023-06-10T21:41:36.399972+0000","last_peered":"2023-06-10T21:41:36.399972+0000","last_clean":"2023-06-10T21:41:36.399972+0000","last_became_active":"2023-06-10T21:40:30.272004+0000","last_became_peered":"2023-06-10T21:40:30.272004+0000","last_unstale":"2023-06-10T21:41:36.399972+0000","last_undegraded":"2023-06-10T21:41:36.399972+0000","last_fullsized":"2023-06-10T21:41:36.399972+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:30.425736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"378'1171","reported_seq":1184,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.066520+0000","last_change":"2023-06-10T21:41:28.178371+0000","last_active":"2023-06-10T21:41:44.066520+0000","last_peered":"2023-06-10T21:41:44.066520+0000","last_clean":"2023-06-10T21:41:44.066520+0000","last_became_active":"2023-06-10T21:41:28.178058+0000","last_became_peered":"2023-06-10T21:41:28.178058+0000","last_unstale":"2023-06-10T21:41:44.066520+0000","last_undegraded":"2023-06-10T21:41:44.066520+0000","last_fullsized":"2023-06-10T21:41:44.066520+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1171,"log_dups_size":0,"ondisk_log_size":1171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76742656,"num_objects":1171,"num_object_clones":0,"num_object_copies":3513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1171,"num_write_kb":74944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":34,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399909+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:41:36.399909+0000","last_peered":"2023-06-10T21:41:36.399909+0000","last_clean":"2023-06-10T21:41:36.399909+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:41:36.399909+0000","last_undegraded":"2023-06-10T21:41:36.399909+0000","last_fullsized":"2023-06-10T21:41:36.399909+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400068+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:41:36.400068+0000","last_peered":"2023-06-10T21:41:36.400068+0000","last_clean":"2023-06-10T21:41:36.400068+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:41:36.400068+0000","last_undegraded":"2023-06-10T21:41:36.400068+0000","last_fullsized":"2023-06-10T21:41:36.400068+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"378'1330","reported_seq":1344,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:45.494872+0000","last_change":"2023-06-10T21:41:28.182345+0000","last_active":"2023-06-10T21:41:45.494872+0000","last_peered":"2023-06-10T21:41:45.494872+0000","last_clean":"2023-06-10T21:41:45.494872+0000","last_became_active":"2023-06-10T21:41:28.181851+0000","last_became_peered":"2023-06-10T21:41:28.181851+0000","last_unstale":"2023-06-10T21:41:45.494872+0000","last_undegraded":"2023-06-10T21:41:45.494872+0000","last_fullsized":"2023-06-10T21:41:45.494872+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1330,"log_dups_size":0,"ondisk_log_size":1330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:07:28.251208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87162880,"num_objects":1330,"num_object_clones":0,"num_object_copies":3990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1330,"num_write_kb":85120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":182,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399778+0000","last_change":"2023-06-10T21:39:50.465026+0000","last_active":"2023-06-10T21:41:36.399778+0000","last_peered":"2023-06-10T21:41:36.399778+0000","last_clean":"2023-06-10T21:41:36.399778+0000","last_became_active":"2023-06-10T21:39:50.464762+0000","last_became_peered":"2023-06-10T21:39:50.464762+0000","last_unstale":"2023-06-10T21:41:36.399778+0000","last_undegraded":"2023-06-10T21:41:36.399778+0000","last_fullsized":"2023-06-10T21:41:36.399778+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:55:00.850195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":399,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.711158+0000","last_change":"2023-06-10T21:39:50.259203+0000","last_active":"2023-06-10T21:41:36.711158+0000","last_peered":"2023-06-10T21:41:36.711158+0000","last_clean":"2023-06-10T21:41:36.711158+0000","last_became_active":"2023-06-10T21:39:50.258873+0000","last_became_peered":"2023-06-10T21:39:50.258873+0000","last_unstale":"2023-06-10T21:41:36.711158+0000","last_undegraded":"2023-06-10T21:41:36.711158+0000","last_fullsized":"2023-06-10T21:41:36.711158+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:56:41.490096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"378'1144","reported_seq":1157,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.107584+0000","last_change":"2023-06-10T21:41:28.184243+0000","last_active":"2023-06-10T21:41:44.107584+0000","last_peered":"2023-06-10T21:41:44.107584+0000","last_clean":"2023-06-10T21:41:44.107584+0000","last_became_active":"2023-06-10T21:41:28.183870+0000","last_became_peered":"2023-06-10T21:41:28.183870+0000","last_unstale":"2023-06-10T21:41:44.107584+0000","last_undegraded":"2023-06-10T21:41:44.107584+0000","last_fullsized":"2023-06-10T21:41:44.107584+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1144,"log_dups_size":0,"ondisk_log_size":1144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1144,"num_write_kb":73216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":377,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.710732+0000","last_change":"2023-06-10T21:39:50.432352+0000","last_active":"2023-06-10T21:41:36.710732+0000","last_peered":"2023-06-10T21:41:36.710732+0000","last_clean":"2023-06-10T21:41:36.710732+0000","last_became_active":"2023-06-10T21:39:50.429920+0000","last_became_peered":"2023-06-10T21:39:50.429920+0000","last_unstale":"2023-06-10T21:41:36.710732+0000","last_undegraded":"2023-06-10T21:41:36.710732+0000","last_fullsized":"2023-06-10T21:41:36.710732+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:14.733503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400303+0000","last_change":"2023-06-10T21:40:30.351796+0000","last_active":"2023-06-10T21:41:36.400303+0000","last_peered":"2023-06-10T21:41:36.400303+0000","last_clean":"2023-06-10T21:41:36.400303+0000","last_became_active":"2023-06-10T21:40:30.350450+0000","last_became_peered":"2023-06-10T21:40:30.350450+0000","last_unstale":"2023-06-10T21:41:36.400303+0000","last_undegraded":"2023-06-10T21:41:36.400303+0000","last_fullsized":"2023-06-10T21:41:36.400303+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:38.306536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"378'1178","reported_seq":1191,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.064468+0000","last_change":"2023-06-10T21:41:28.177644+0000","last_active":"2023-06-10T21:41:44.064468+0000","last_peered":"2023-06-10T21:41:44.064468+0000","last_clean":"2023-06-10T21:41:44.064468+0000","last_became_active":"2023-06-10T21:41:28.177318+0000","last_became_peered":"2023-06-10T21:41:28.177318+0000","last_unstale":"2023-06-10T21:41:44.064468+0000","last_undegraded":"2023-06-10T21:41:44.064468+0000","last_fullsized":"2023-06-10T21:41:44.064468+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1178,"log_dups_size":0,"ondisk_log_size":1178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77201408,"num_objects":1178,"num_object_clones":0,"num_object_copies":3534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1178,"num_write_kb":75392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399578+0000","last_change":"2023-06-10T21:40:30.279010+0000","last_active":"2023-06-10T21:41:36.399578+0000","last_peered":"2023-06-10T21:41:36.399578+0000","last_clean":"2023-06-10T21:41:36.399578+0000","last_became_active":"2023-06-10T21:40:30.277943+0000","last_became_peered":"2023-06-10T21:40:30.277943+0000","last_unstale":"2023-06-10T21:41:36.399578+0000","last_undegraded":"2023-06-10T21:41:36.399578+0000","last_fullsized":"2023-06-10T21:41:36.399578+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:38:15.175343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":309,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400667+0000","last_change":"2023-06-10T21:39:50.233694+0000","last_active":"2023-06-10T21:41:36.400667+0000","last_peered":"2023-06-10T21:41:36.400667+0000","last_clean":"2023-06-10T21:41:36.400667+0000","last_became_active":"2023-06-10T21:39:50.233429+0000","last_became_peered":"2023-06-10T21:39:50.233429+0000","last_unstale":"2023-06-10T21:41:36.400667+0000","last_undegraded":"2023-06-10T21:41:36.400667+0000","last_fullsized":"2023-06-10T21:41:36.400667+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:15:42.432299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"378'1179","reported_seq":1192,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.044435+0000","last_change":"2023-06-10T21:41:28.176942+0000","last_active":"2023-06-10T21:41:44.044435+0000","last_peered":"2023-06-10T21:41:44.044435+0000","last_clean":"2023-06-10T21:41:44.044435+0000","last_became_active":"2023-06-10T21:41:28.176613+0000","last_became_peered":"2023-06-10T21:41:28.176613+0000","last_unstale":"2023-06-10T21:41:44.044435+0000","last_undegraded":"2023-06-10T21:41:44.044435+0000","last_fullsized":"2023-06-10T21:41:44.044435+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1179,"log_dups_size":0,"ondisk_log_size":1179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77266944,"num_objects":1179,"num_object_clones":0,"num_object_copies":3537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1179,"num_write_kb":75456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":161,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020031+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:41:36.020031+0000","last_peered":"2023-06-10T21:41:36.020031+0000","last_clean":"2023-06-10T21:41:36.020031+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:41:36.020031+0000","last_undegraded":"2023-06-10T21:41:36.020031+0000","last_fullsized":"2023-06-10T21:41:36.020031+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":61,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715651+0000","last_change":"2023-06-10T21:40:30.281222+0000","last_active":"2023-06-10T21:41:36.715651+0000","last_peered":"2023-06-10T21:41:36.715651+0000","last_clean":"2023-06-10T21:41:36.715651+0000","last_became_active":"2023-06-10T21:40:30.280846+0000","last_became_peered":"2023-06-10T21:40:30.280846+0000","last_unstale":"2023-06-10T21:41:36.715651+0000","last_undegraded":"2023-06-10T21:41:36.715651+0000","last_fullsized":"2023-06-10T21:41:36.715651+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715446+0000","last_change":"2023-06-10T21:39:50.466302+0000","last_active":"2023-06-10T21:41:36.715446+0000","last_peered":"2023-06-10T21:41:36.715446+0000","last_clean":"2023-06-10T21:41:36.715446+0000","last_became_active":"2023-06-10T21:39:50.465409+0000","last_became_peered":"2023-06-10T21:39:50.465409+0000","last_unstale":"2023-06-10T21:41:36.715446+0000","last_undegraded":"2023-06-10T21:41:36.715446+0000","last_fullsized":"2023-06-10T21:41:36.715446+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"378'1185","reported_seq":1198,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.108121+0000","last_change":"2023-06-10T21:41:28.182320+0000","last_active":"2023-06-10T21:41:44.108121+0000","last_peered":"2023-06-10T21:41:44.108121+0000","last_clean":"2023-06-10T21:41:44.108121+0000","last_became_active":"2023-06-10T21:41:28.181951+0000","last_became_peered":"2023-06-10T21:41:28.181951+0000","last_unstale":"2023-06-10T21:41:44.108121+0000","last_undegraded":"2023-06-10T21:41:44.108121+0000","last_fullsized":"2023-06-10T21:41:44.108121+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1185,"log_dups_size":0,"ondisk_log_size":1185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77660160,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1185,"num_write_kb":75840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":80,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715349+0000","last_change":"2023-06-10T21:40:30.236317+0000","last_active":"2023-06-10T21:41:36.715349+0000","last_peered":"2023-06-10T21:41:36.715349+0000","last_clean":"2023-06-10T21:41:36.715349+0000","last_became_active":"2023-06-10T21:40:30.235998+0000","last_became_peered":"2023-06-10T21:40:30.235998+0000","last_unstale":"2023-06-10T21:41:36.715349+0000","last_undegraded":"2023-06-10T21:41:36.715349+0000","last_fullsized":"2023-06-10T21:41:36.715349+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":358,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715480+0000","last_change":"2023-06-10T21:40:30.281401+0000","last_active":"2023-06-10T21:41:36.715480+0000","last_peered":"2023-06-10T21:41:36.715480+0000","last_clean":"2023-06-10T21:41:36.715480+0000","last_became_active":"2023-06-10T21:40:30.280966+0000","last_became_peered":"2023-06-10T21:40:30.280966+0000","last_unstale":"2023-06-10T21:41:36.715480+0000","last_undegraded":"2023-06-10T21:41:36.715480+0000","last_fullsized":"2023-06-10T21:41:36.715480+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":241,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.711175+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:41:36.711175+0000","last_peered":"2023-06-10T21:41:36.711175+0000","last_clean":"2023-06-10T21:41:36.711175+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:41:36.711175+0000","last_undegraded":"2023-06-10T21:41:36.711175+0000","last_fullsized":"2023-06-10T21:41:36.711175+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"378'1169","reported_seq":1182,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.113255+0000","last_change":"2023-06-10T21:41:28.182336+0000","last_active":"2023-06-10T21:41:44.113255+0000","last_peered":"2023-06-10T21:41:44.113255+0000","last_clean":"2023-06-10T21:41:44.113255+0000","last_became_active":"2023-06-10T21:41:28.181961+0000","last_became_peered":"2023-06-10T21:41:28.181961+0000","last_unstale":"2023-06-10T21:41:44.113255+0000","last_undegraded":"2023-06-10T21:41:44.113255+0000","last_fullsized":"2023-06-10T21:41:44.113255+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1169,"log_dups_size":0,"ondisk_log_size":1169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1169,"num_write_kb":74816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":261,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.710657+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:41:36.710657+0000","last_peered":"2023-06-10T21:41:36.710657+0000","last_clean":"2023-06-10T21:41:36.710657+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:41:36.710657+0000","last_undegraded":"2023-06-10T21:41:36.710657+0000","last_fullsized":"2023-06-10T21:41:36.710657+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":251,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.245350+0000","last_change":"2023-06-10T21:40:30.229244+0000","last_active":"2023-06-10T21:41:39.245350+0000","last_peered":"2023-06-10T21:41:39.245350+0000","last_clean":"2023-06-10T21:41:39.245350+0000","last_became_active":"2023-06-10T21:40:30.228964+0000","last_became_peered":"2023-06-10T21:40:30.228964+0000","last_unstale":"2023-06-10T21:41:39.245350+0000","last_undegraded":"2023-06-10T21:41:39.245350+0000","last_fullsized":"2023-06-10T21:41:39.245350+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:41.629358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":79,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715945+0000","last_change":"2023-06-10T21:40:30.236712+0000","last_active":"2023-06-10T21:41:36.715945+0000","last_peered":"2023-06-10T21:41:36.715945+0000","last_clean":"2023-06-10T21:41:36.715945+0000","last_became_active":"2023-06-10T21:40:30.236404+0000","last_became_peered":"2023-06-10T21:40:30.236404+0000","last_unstale":"2023-06-10T21:41:36.715945+0000","last_undegraded":"2023-06-10T21:41:36.715945+0000","last_fullsized":"2023-06-10T21:41:36.715945+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"378'1121","reported_seq":1134,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.063796+0000","last_change":"2023-06-10T21:41:28.181267+0000","last_active":"2023-06-10T21:41:44.063796+0000","last_peered":"2023-06-10T21:41:44.063796+0000","last_clean":"2023-06-10T21:41:44.063796+0000","last_became_active":"2023-06-10T21:41:28.180932+0000","last_became_peered":"2023-06-10T21:41:28.180932+0000","last_unstale":"2023-06-10T21:41:44.063796+0000","last_undegraded":"2023-06-10T21:41:44.063796+0000","last_fullsized":"2023-06-10T21:41:44.063796+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1121,"log_dups_size":0,"ondisk_log_size":1121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73465856,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1121,"num_write_kb":71744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":52,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715305+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:41:36.715305+0000","last_peered":"2023-06-10T21:41:36.715305+0000","last_clean":"2023-06-10T21:41:36.715305+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:41:36.715305+0000","last_undegraded":"2023-06-10T21:41:36.715305+0000","last_fullsized":"2023-06-10T21:41:36.715305+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":578,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.019587+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:41:36.019587+0000","last_peered":"2023-06-10T21:41:36.019587+0000","last_clean":"2023-06-10T21:41:36.019587+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:41:36.019587+0000","last_undegraded":"2023-06-10T21:41:36.019587+0000","last_fullsized":"2023-06-10T21:41:36.019587+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399697+0000","last_change":"2023-06-10T21:40:30.260288+0000","last_active":"2023-06-10T21:41:36.399697+0000","last_peered":"2023-06-10T21:41:36.399697+0000","last_clean":"2023-06-10T21:41:36.399697+0000","last_became_active":"2023-06-10T21:40:30.259958+0000","last_became_peered":"2023-06-10T21:40:30.259958+0000","last_unstale":"2023-06-10T21:41:36.399697+0000","last_undegraded":"2023-06-10T21:41:36.399697+0000","last_fullsized":"2023-06-10T21:41:36.399697+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:41.077016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":246,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020222+0000","last_change":"2023-06-10T21:40:30.286026+0000","last_active":"2023-06-10T21:41:36.020222+0000","last_peered":"2023-06-10T21:41:36.020222+0000","last_clean":"2023-06-10T21:41:36.020222+0000","last_became_active":"2023-06-10T21:40:30.285670+0000","last_became_peered":"2023-06-10T21:40:30.285670+0000","last_unstale":"2023-06-10T21:41:36.020222+0000","last_undegraded":"2023-06-10T21:41:36.020222+0000","last_fullsized":"2023-06-10T21:41:36.020222+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"378'1181","reported_seq":1194,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.116460+0000","last_change":"2023-06-10T21:41:28.183802+0000","last_active":"2023-06-10T21:41:44.116460+0000","last_peered":"2023-06-10T21:41:44.116460+0000","last_clean":"2023-06-10T21:41:44.116460+0000","last_became_active":"2023-06-10T21:41:28.183511+0000","last_became_peered":"2023-06-10T21:41:28.183511+0000","last_unstale":"2023-06-10T21:41:44.116460+0000","last_undegraded":"2023-06-10T21:41:44.116460+0000","last_fullsized":"2023-06-10T21:41:44.116460+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1181,"log_dups_size":0,"ondisk_log_size":1181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1181,"num_write_kb":75584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":415,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.019595+0000","last_change":"2023-06-10T21:39:50.237017+0000","last_active":"2023-06-10T21:41:36.019595+0000","last_peered":"2023-06-10T21:41:36.019595+0000","last_clean":"2023-06-10T21:41:36.019595+0000","last_became_active":"2023-06-10T21:39:50.232270+0000","last_became_peered":"2023-06-10T21:39:50.232270+0000","last_unstale":"2023-06-10T21:41:36.019595+0000","last_undegraded":"2023-06-10T21:41:36.019595+0000","last_fullsized":"2023-06-10T21:41:36.019595+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399496+0000","last_change":"2023-06-10T21:40:30.316582+0000","last_active":"2023-06-10T21:41:36.399496+0000","last_peered":"2023-06-10T21:41:36.399496+0000","last_clean":"2023-06-10T21:41:36.399496+0000","last_became_active":"2023-06-10T21:40:30.315657+0000","last_became_peered":"2023-06-10T21:40:30.315657+0000","last_unstale":"2023-06-10T21:41:36.399496+0000","last_undegraded":"2023-06-10T21:41:36.399496+0000","last_fullsized":"2023-06-10T21:41:36.399496+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:08:04.756800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":113,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.244794+0000","last_change":"2023-06-10T21:39:50.559093+0000","last_active":"2023-06-10T21:41:39.244794+0000","last_peered":"2023-06-10T21:41:39.244794+0000","last_clean":"2023-06-10T21:41:39.244794+0000","last_became_active":"2023-06-10T21:39:50.558289+0000","last_became_peered":"2023-06-10T21:39:50.558289+0000","last_unstale":"2023-06-10T21:41:39.244794+0000","last_undegraded":"2023-06-10T21:41:39.244794+0000","last_fullsized":"2023-06-10T21:41:39.244794+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:37.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"378'1392","reported_seq":1406,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:45.505694+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:41:45.505694+0000","last_peered":"2023-06-10T21:41:45.505694+0000","last_clean":"2023-06-10T21:41:45.505694+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:41:45.505694+0000","last_undegraded":"2023-06-10T21:41:45.505694+0000","last_fullsized":"2023-06-10T21:41:45.505694+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1392,"log_dups_size":0,"ondisk_log_size":1392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91226112,"num_objects":1392,"num_object_clones":0,"num_object_copies":4176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1392,"num_write_kb":89088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":409,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020218+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:41:36.020218+0000","last_peered":"2023-06-10T21:41:36.020218+0000","last_clean":"2023-06-10T21:41:36.020218+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:41:36.020218+0000","last_undegraded":"2023-06-10T21:41:36.020218+0000","last_fullsized":"2023-06-10T21:41:36.020218+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":241,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.711303+0000","last_change":"2023-06-10T21:39:50.296322+0000","last_active":"2023-06-10T21:41:36.711303+0000","last_peered":"2023-06-10T21:41:36.711303+0000","last_clean":"2023-06-10T21:41:36.711303+0000","last_became_active":"2023-06-10T21:39:50.295201+0000","last_became_peered":"2023-06-10T21:39:50.295201+0000","last_unstale":"2023-06-10T21:41:36.711303+0000","last_undegraded":"2023-06-10T21:41:36.711303+0000","last_fullsized":"2023-06-10T21:41:36.711303+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:28.403434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715557+0000","last_change":"2023-06-10T21:39:50.271175+0000","last_active":"2023-06-10T21:41:36.715557+0000","last_peered":"2023-06-10T21:41:36.715557+0000","last_clean":"2023-06-10T21:41:36.715557+0000","last_became_active":"2023-06-10T21:39:50.270302+0000","last_became_peered":"2023-06-10T21:39:50.270302+0000","last_unstale":"2023-06-10T21:41:36.715557+0000","last_undegraded":"2023-06-10T21:41:36.715557+0000","last_fullsized":"2023-06-10T21:41:36.715557+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"378'1200","reported_seq":1213,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:44.062253+0000","last_change":"2023-06-10T21:41:28.179779+0000","last_active":"2023-06-10T21:41:44.062253+0000","last_peered":"2023-06-10T21:41:44.062253+0000","last_clean":"2023-06-10T21:41:44.062253+0000","last_became_active":"2023-06-10T21:41:28.179455+0000","last_became_peered":"2023-06-10T21:41:28.179455+0000","last_unstale":"2023-06-10T21:41:44.062253+0000","last_undegraded":"2023-06-10T21:41:44.062253+0000","last_fullsized":"2023-06-10T21:41:44.062253+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1200,"log_dups_size":0,"ondisk_log_size":1200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78643200,"num_objects":1200,"num_object_clones":0,"num_object_copies":3600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1200,"num_write_kb":76800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":371,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400623+0000","last_change":"2023-06-10T21:39:30.266087+0000","last_active":"2023-06-10T21:41:36.400623+0000","last_peered":"2023-06-10T21:41:36.400623+0000","last_clean":"2023-06-10T21:41:36.400623+0000","last_became_active":"2023-06-10T21:39:30.231858+0000","last_became_peered":"2023-06-10T21:39:30.231858+0000","last_unstale":"2023-06-10T21:41:36.400623+0000","last_undegraded":"2023-06-10T21:41:36.400623+0000","last_fullsized":"2023-06-10T21:41:36.400623+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:08:09.982313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":182,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400544+0000","last_change":"2023-06-10T21:39:50.227162+0000","last_active":"2023-06-10T21:41:36.400544+0000","last_peered":"2023-06-10T21:41:36.400544+0000","last_clean":"2023-06-10T21:41:36.400544+0000","last_became_active":"2023-06-10T21:39:50.226891+0000","last_became_peered":"2023-06-10T21:39:50.226891+0000","last_unstale":"2023-06-10T21:41:36.400544+0000","last_undegraded":"2023-06-10T21:41:36.400544+0000","last_fullsized":"2023-06-10T21:41:36.400544+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:48:59.680224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399521+0000","last_change":"2023-06-10T21:40:30.279185+0000","last_active":"2023-06-10T21:41:36.399521+0000","last_peered":"2023-06-10T21:41:36.399521+0000","last_clean":"2023-06-10T21:41:36.399521+0000","last_became_active":"2023-06-10T21:40:30.278330+0000","last_became_peered":"2023-06-10T21:40:30.278330+0000","last_unstale":"2023-06-10T21:41:36.399521+0000","last_undegraded":"2023-06-10T21:41:36.399521+0000","last_fullsized":"2023-06-10T21:41:36.399521+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:15.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"378'1434","reported_seq":1448,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:45.503679+0000","last_change":"2023-06-10T21:41:28.176991+0000","last_active":"2023-06-10T21:41:45.503679+0000","last_peered":"2023-06-10T21:41:45.503679+0000","last_clean":"2023-06-10T21:41:45.503679+0000","last_became_active":"2023-06-10T21:41:28.176650+0000","last_became_peered":"2023-06-10T21:41:28.176650+0000","last_unstale":"2023-06-10T21:41:45.503679+0000","last_undegraded":"2023-06-10T21:41:45.503679+0000","last_fullsized":"2023-06-10T21:41:45.503679+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":1434,"log_dups_size":0,"ondisk_log_size":1434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:14.567179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93978624,"num_objects":1434,"num_object_clones":0,"num_object_copies":4302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1434,"num_write_kb":91776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":126,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020479+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:41:36.020479+0000","last_peered":"2023-06-10T21:41:36.020479+0000","last_clean":"2023-06-10T21:41:36.020479+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:41:36.020479+0000","last_undegraded":"2023-06-10T21:41:36.020479+0000","last_fullsized":"2023-06-10T21:41:36.020479+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":225,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020285+0000","last_change":"2023-06-10T21:40:30.580293+0000","last_active":"2023-06-10T21:41:36.020285+0000","last_peered":"2023-06-10T21:41:36.020285+0000","last_clean":"2023-06-10T21:41:36.020285+0000","last_became_active":"2023-06-10T21:40:30.391049+0000","last_became_peered":"2023-06-10T21:40:30.391049+0000","last_unstale":"2023-06-10T21:41:36.020285+0000","last_undegraded":"2023-06-10T21:41:36.020285+0000","last_fullsized":"2023-06-10T21:41:36.020285+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.714993+0000","last_change":"2023-06-10T21:39:30.208137+0000","last_active":"2023-06-10T21:41:36.714993+0000","last_peered":"2023-06-10T21:41:36.714993+0000","last_clean":"2023-06-10T21:41:36.714993+0000","last_became_active":"2023-06-10T21:39:30.207848+0000","last_became_peered":"2023-06-10T21:39:30.207848+0000","last_unstale":"2023-06-10T21:41:36.714993+0000","last_undegraded":"2023-06-10T21:41:36.714993+0000","last_fullsized":"2023-06-10T21:41:36.714993+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":77,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.020158+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:41:36.020158+0000","last_peered":"2023-06-10T21:41:36.020158+0000","last_clean":"2023-06-10T21:41:36.020158+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:41:36.020158+0000","last_undegraded":"2023-06-10T21:41:36.020158+0000","last_fullsized":"2023-06-10T21:41:36.020158+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":258,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.711265+0000","last_change":"2023-06-10T21:40:30.227732+0000","last_active":"2023-06-10T21:41:36.711265+0000","last_peered":"2023-06-10T21:41:36.711265+0000","last_clean":"2023-06-10T21:41:36.711265+0000","last_became_active":"2023-06-10T21:40:30.227380+0000","last_became_peered":"2023-06-10T21:40:30.227380+0000","last_unstale":"2023-06-10T21:41:36.711265+0000","last_undegraded":"2023-06-10T21:41:36.711265+0000","last_fullsized":"2023-06-10T21:41:36.711265+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:33.673485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":65,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715411+0000","last_change":"2023-06-10T21:39:50.304413+0000","last_active":"2023-06-10T21:41:36.715411+0000","last_peered":"2023-06-10T21:41:36.715411+0000","last_clean":"2023-06-10T21:41:36.715411+0000","last_became_active":"2023-06-10T21:39:50.303414+0000","last_became_peered":"2023-06-10T21:39:50.303414+0000","last_unstale":"2023-06-10T21:41:36.715411+0000","last_undegraded":"2023-06-10T21:41:36.715411+0000","last_fullsized":"2023-06-10T21:41:36.715411+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":281,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400014+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:41:36.400014+0000","last_peered":"2023-06-10T21:41:36.400014+0000","last_clean":"2023-06-10T21:41:36.400014+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:41:36.400014+0000","last_undegraded":"2023-06-10T21:41:36.400014+0000","last_fullsized":"2023-06-10T21:41:36.400014+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":82,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715806+0000","last_change":"2023-06-10T21:40:30.565123+0000","last_active":"2023-06-10T21:41:36.715806+0000","last_peered":"2023-06-10T21:41:36.715806+0000","last_clean":"2023-06-10T21:41:36.715806+0000","last_became_active":"2023-06-10T21:40:30.280290+0000","last_became_peered":"2023-06-10T21:40:30.280290+0000","last_unstale":"2023-06-10T21:41:36.715806+0000","last_undegraded":"2023-06-10T21:41:36.715806+0000","last_fullsized":"2023-06-10T21:41:36.715806+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":337,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399800+0000","last_change":"2023-06-10T21:39:30.245702+0000","last_active":"2023-06-10T21:41:36.399800+0000","last_peered":"2023-06-10T21:41:36.399800+0000","last_clean":"2023-06-10T21:41:36.399800+0000","last_became_active":"2023-06-10T21:39:30.245426+0000","last_became_peered":"2023-06-10T21:39:30.245426+0000","last_unstale":"2023-06-10T21:41:36.399800+0000","last_undegraded":"2023-06-10T21:41:36.399800+0000","last_fullsized":"2023-06-10T21:41:36.399800+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:52:27.316156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.400434+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:41:36.400434+0000","last_peered":"2023-06-10T21:41:36.400434+0000","last_clean":"2023-06-10T21:41:36.400434+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:41:36.400434+0000","last_undegraded":"2023-06-10T21:41:36.400434+0000","last_fullsized":"2023-06-10T21:41:36.400434+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399604+0000","last_change":"2023-06-10T21:40:30.253755+0000","last_active":"2023-06-10T21:41:36.399604+0000","last_peered":"2023-06-10T21:41:36.399604+0000","last_clean":"2023-06-10T21:41:36.399604+0000","last_became_active":"2023-06-10T21:40:30.253530+0000","last_became_peered":"2023-06-10T21:40:30.253530+0000","last_unstale":"2023-06-10T21:41:36.399604+0000","last_undegraded":"2023-06-10T21:41:36.399604+0000","last_fullsized":"2023-06-10T21:41:36.399604+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:14.839319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":318,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.715909+0000","last_change":"2023-06-10T21:40:30.574471+0000","last_active":"2023-06-10T21:41:36.715909+0000","last_peered":"2023-06-10T21:41:36.715909+0000","last_clean":"2023-06-10T21:41:36.715909+0000","last_became_active":"2023-06-10T21:40:30.281351+0000","last_became_peered":"2023-06-10T21:40:30.281351+0000","last_unstale":"2023-06-10T21:41:36.715909+0000","last_undegraded":"2023-06-10T21:41:36.715909+0000","last_fullsized":"2023-06-10T21:41:36.715909+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":30,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-06-10T21:41:36.399628+0000","last_change":"2023-06-10T21:40:30.245261+0000","last_active":"2023-06-10T21:41:36.399628+0000","last_peered":"2023-06-10T21:41:36.399628+0000","last_clean":"2023-06-10T21:41:36.399628+0000","last_became_active":"2023-06-10T21:40:30.244695+0000","last_became_peered":"2023-06-10T21:40:30.244695+0000","last_unstale":"2023-06-10T21:41:36.399628+0000","last_undegraded":"2023-06-10T21:41:36.399628+0000","last_fullsized":"2023-06-10T21:41:36.399628+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:54.784460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1280245760,"num_objects":19535,"num_object_clones":0,"num_object_copies":58605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19535,"num_write_kb":1250240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3719102464,"data_stored":3719102464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19535,"ondisk_log_size":19535,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":18486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430736,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1598780,"kb_used_data":774516,"kb_used_omap":0,"kb_used_meta":824256,"kb_avail":92146372,"statfs":{"total":95995035648,"available":94357884928,"internally_reserved":0,"allocated":793104384,"data_stored":792407991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":844038144},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.986,"15min":4.8600000000000003},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":22.277000000000001,"5min":56.639000000000003,"15min":184.83699999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.3889999999999998,"5min":2.4079999999999999,"15min":5.165},"min":{"1min":0.28599999999999998,"5min":0.25,"15min":0.223},"max":{"1min":22.462,"5min":56.628,"15min":185.58000000000001},"last":0.98799999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.9660000000000002,"15min":2.976},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":38.078000000000003,"5min":60.863,"15min":93.093999999999994},"last":1.0509999999999999},{"interface":"front","average":{"1min":3.476,"5min":3.2919999999999998,"15min":3.1000000000000001},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":38.341000000000001,"5min":61.148000000000003,"15min":93.119},"last":2.5419999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:41:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.867,"15min":4.9009999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":24.106000000000002,"5min":137.64099999999999,"15min":172.33600000000001},"last":1.2629999999999999},{"interface":"front","average":{"1min":2.4089999999999998,"5min":4.54,"15min":4.8369999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29499999999999998},"max":{"1min":24.132999999999999,"5min":136.96600000000001,"15min":172.185},"last":2.6389999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":4.3460000000000001,"15min":4.2649999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":24.375,"5min":102.25700000000001,"15min":147.00700000000001},"last":1.349},{"interface":"front","average":{"1min":2.347,"5min":4.1070000000000002,"15min":3.923},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":26.460000000000001,"5min":103.401,"15min":146.93899999999999},"last":2.4870000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.249},"max":{"1min":1.0289999999999999,"5min":5.859,"15min":5.859},"last":2.6000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.81399999999999995,"15min":0.66700000000000004},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.3290000000000002,"5min":22.77,"15min":22.77},"last":1.173}]},{"osd":5,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48399999999999999,"15min":0.64200000000000002},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.8220000000000001,"5min":2.7930000000000001,"15min":12.048},"last":1.389},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.97499999999999998,"5min":5.9100000000000001,"15min":6.7530000000000001},"last":2.8610000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.68300000000000005,"15min":0.72399999999999998},"min":{"1min":0.34699999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":1.847,"5min":10.890000000000001,"15min":13.073},"last":1.29},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.59299999999999997,"15min":0.56799999999999995},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0209999999999999,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":2.8959999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626964,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566406,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020383,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075795,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761652,"kb_used_data":998956,"kb_used_omap":7,"kb_used_meta":762680,"kb_avail":91983500,"statfs":{"total":95995035648,"available":94191104000,"internally_reserved":0,"allocated":1022930944,"data_stored":1021661943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":780984401},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":5.54,"5min":5.9180000000000001,"15min":5.0979999999999999},"min":{"1min":0.19900000000000001,"5min":0.19,"15min":0.19},"max":{"1min":83.733000000000004,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":4.9020000000000001,"5min":5.4669999999999996,"15min":5.173},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":82.322999999999993,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":5.9130000000000003,"5min":8.6440000000000001,"15min":5.3680000000000003},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.19},"max":{"1min":85.709999999999994,"5min":171.286,"15min":171.286},"last":1.4530000000000001},{"interface":"front","average":{"1min":6.3300000000000001,"5min":4.7469999999999999,"15min":3.827},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":85.501999999999995,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:41:20 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":5.694,"15min":5.5270000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":70.480999999999995,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":5.4980000000000002,"5min":5.3550000000000004,"15min":5.7530000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":71.144999999999996,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.52300000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":3.4969999999999999,"15min":3.5449999999999999},"min":{"1min":0.36699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.093,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.2060000000000004,"15min":4.4390000000000001},"min":{"1min":0.35599999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.423,"5min":171.18100000000001,"15min":223.28399999999999},"last":0.502}]},{"osd":5,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.613,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.7759999999999998,"5min":171.303,"15min":171.303},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.5430000000000001,"15min":2.3279999999999998},"min":{"1min":0.38,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1280000000000001,"5min":171.203,"15min":171.203},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":3.585,"15min":2.4420000000000002},"min":{"1min":0.48599999999999999,"5min":0.307,"15min":0.307},"max":{"1min":2.02,"5min":171.441,"15min":171.441},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.86099999999999999,"5min":5.2610000000000001,"15min":3.3090000000000002},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":3.4630000000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":3.7360000000000002,"15min":4.4139999999999997},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":5.7400000000000002,"5min":171.26499999999999,"15min":219.131},"last":1.6100000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":3.5670000000000002,"15min":4.4219999999999997},"min":{"1min":0.41599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.0489999999999999,"5min":171.52000000000001,"15min":219.19},"last":1.5940000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824154,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1509744,"kb_used_data":989992,"kb_used_omap":7,"kb_used_meta":519736,"kb_avail":92235408,"statfs":{"total":95995035648,"available":94449057792,"internally_reserved":0,"allocated":1013751808,"data_stored":1012366187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":532209683},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":3.3039999999999998,"15min":4.234},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.744,"5min":30.744,"15min":105.09},"last":1.276},{"interface":"front","average":{"1min":3.7559999999999998,"5min":3.3029999999999999,"15min":4.1200000000000001},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":30.805,"5min":30.805,"15min":99.852999999999994},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.9830000000000001,"15min":3.819},"min":{"1min":0.23599999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.6509999999999998,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.325,"5min":3.5960000000000001,"15min":3.7690000000000001},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.9649999999999999,"5min":41.18,"15min":94.691000000000003},"last":1.419}]},{"osd":2,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":4.8869999999999996,"15min":6.4669999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.206,"5min":231.07900000000001,"15min":231.07900000000001},"last":4.7080000000000002},{"interface":"front","average":{"1min":1.8879999999999999,"5min":6.1260000000000003,"15min":7.9299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":7.6909999999999998,"5min":231.58600000000001,"15min":231.58600000000001},"last":1.363}]},{"osd":4,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.379,"15min":1.407},"min":{"1min":0.32100000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.0820000000000007,"5min":10.368,"15min":13.417},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.2309999999999999,"15min":2.4990000000000001},"min":{"1min":0.38400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.6150000000000002,"5min":19.244,"15min":19.728000000000002},"last":0.92200000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.847,"15min":1.5489999999999999},"min":{"1min":0.42899999999999999,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":3.8559999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":1.347},{"interface":"front","average":{"1min":1.409,"5min":1.4830000000000001,"15min":1.5049999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":9.1210000000000004,"5min":10.414999999999999,"15min":13.470000000000001},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":2.254,"15min":2.52},"min":{"1min":0.46800000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.5270000000000001,"5min":19.164999999999999,"15min":19.805},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.839,"15min":1.5800000000000001},"min":{"1min":0.49399999999999999,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":3.8180000000000001,"5min":17.327000000000002,"15min":17.327000000000002},"last":1.3169999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.3330000000000002,"15min":2.778},"min":{"1min":0.50800000000000001,"5min":0.379,"15min":0.33900000000000002},"max":{"1min":7.5789999999999997,"5min":19.193000000000001,"15min":79.984999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.9350000000000001,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":3.6960000000000002,"5min":17.251000000000001,"15min":80.272999999999996},"last":1.379}]}]},{"osd":4,"up_from":0,"seq":128849019420,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823192,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1765748,"kb_used_data":1110828,"kb_used_omap":0,"kb_used_meta":654912,"kb_avail":91979404,"statfs":{"total":95995035648,"available":94186909696,"internally_reserved":0,"allocated":1137487872,"data_stored":1136799276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":670629888},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.655,"15min":3.2349999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.4530000000000003,"5min":18.562000000000001,"15min":134.71899999999999},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.121,"15min":3.4710000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2709999999999999,"5min":27.256,"15min":135.386},"last":0.73299999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":5.0019999999999998,"5min":2.5059999999999998,"15min":2.6320000000000001},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":60.399999999999999,"5min":60.399999999999999,"15min":71.349000000000004},"last":0.80700000000000005},{"interface":"front","average":{"1min":5.25,"5min":2.319,"15min":2.5899999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":59.780999999999999,"5min":59.780999999999999,"15min":71.331999999999994},"last":0.76600000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":5.9020000000000001,"15min":6.3079999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":15.789999999999999,"5min":324.21600000000001,"15min":324.21600000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.9670000000000001,"5min":5.54,"15min":6.1070000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.0950000000000006,"5min":324.16300000000001,"15min":324.16300000000001},"last":0.79000000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":4.5609999999999999,"15min":4.3949999999999996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.2510000000000003,"5min":90.725999999999999,"15min":128.089},"last":1.135},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.9390000000000001,"15min":4.3959999999999999},"min":{"1min":0.39000000000000001,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":5.907,"5min":90.180000000000007,"15min":128.24299999999999},"last":0.70599999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.47899999999999998,"15min":1.135},"min":{"1min":0.27900000000000003,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":5.3460000000000001,"5min":5.3460000000000001,"15min":165.73699999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.52600000000000002,"15min":1.242},"min":{"1min":0.31,"5min":0.25,"15min":0.25},"max":{"1min":1.266,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.67400000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.60599999999999998,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.22},"max":{"1min":3.4580000000000002,"5min":5.3739999999999997,"15min":29.760000000000002},"last":1.337},{"interface":"front","average":{"1min":0.751,"5min":0.61399999999999999,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.29199999999999998},"max":{"1min":3.4340000000000002,"5min":4.3380000000000001,"15min":28.995999999999999},"last":1.077}]},{"osd":7,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.6359999999999999,"15min":1.46},"min":{"1min":0.33900000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":1.179,"5min":94.102000000000004,"15min":136.904},"last":1.1060000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6850000000000001,"15min":1.4930000000000001},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":2.754,"5min":95.653999999999996,"15min":137.661},"last":1.3680000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1097334784,"data_stored":1097334784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":678297600,"data_stored":678297600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1080557568,"data_stored":1080557568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":862912512,"data_stored":862912512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:41:49.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:49.205 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:49.215 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:49.378 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:49.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:49.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 29759 29743 154.858 117.375 0.010907 0.00645218 2023-06-10T21:41:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:49 smithi152 ceph-mon[156819]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:49 smithi152 ceph-mon[156819]: pgmap v1391: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-06-10T21:41:49.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:49.485 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:49.550 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:49.573 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[180297]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:49.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[180297]: pgmap v1391: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-06-10T21:41:49.574 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:49.574 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[186706]: from='client.56650 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:49.574 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[186706]: pgmap v1391: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-06-10T21:41:49.575 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:41:49.644 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:49.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:49.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:49.805 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:50.017 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:50.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:50.172 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:50.191 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:41:50.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:41:50.342 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:50.345 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:50.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:50.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:50.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 31643 31627 152.004 117.75 0.00659161 0.00657307 2023-06-10T21:41:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:41:50.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:50.491 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:50.535 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:50.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:50.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:50.671 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:50.728 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:51.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: pgmap v1392: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-06-10T21:41:51.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:51.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:41:51.383 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:51.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: osdmap e379: 8 total, 8 up, 4 in 2023-06-10T21:41:51.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.384 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:51 smithi152 ceph-mon[156819]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 32647 32631 145.629 62.75 0.00900981 0.00683413 2023-06-10T21:41:51.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: pgmap v1392: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: osdmap e379: 8 total, 8 up, 4 in 2023-06-10T21:41:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[186706]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:51.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: pgmap v1392: 89 pgs: 89 active+clean; 1.7 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2174117116' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: osdmap e379: 8 total, 8 up, 4 in 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:51 smithi064 ceph-mon[180297]: failed to encode map e379 with expected crc 2023-06-10T21:41:51.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:51.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:51.811 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:51.854 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:51.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:51.958 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:51.984 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:52.027 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:52.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:52.062 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:41:52.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:52.226 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:52.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:52.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:52.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:52.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 33576 33560 139.792 58.0625 0.00836076 0.00714607 2023-06-10T21:41:52.473 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:41:52.494 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:41:52.528 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:52.528 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:41:52.567 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:41:52.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:41:52.616 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:41:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:52.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:41:52.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:41:52.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:41:52.686 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:52.718 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:41:52.723 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:41:52.752 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:41:52.759 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.760 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.761 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:41:52.800 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:53.049 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:41:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:41:52] "GET /metrics HTTP/1.1" 200 224168 "" "Prometheus/2.33.4" 2023-06-10T21:41:53.088 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:41:53.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:53.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:41:53.117 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:41:53.117 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:41:53.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:53.267 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:53.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 35316 35300 137.852 108.75 0.00772169 0.00724847 2023-06-10T21:41:53.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:53 smithi064 ceph-mon[180297]: pgmap v1394: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-06-10T21:41:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1030205442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:53.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:53 smithi064 ceph-mon[186706]: pgmap v1394: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-06-10T21:41:53.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1030205442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:53 smithi152 ceph-mon[156819]: pgmap v1394: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-06-10T21:41:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1030205442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:41:53.457 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:53.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:53.561 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:53.605 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:53.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:41:54.076 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:54.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:54.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:54.234 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:41:54.235 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:41:54.270 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:54.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 36719 36703 134.901 87.6875 0.00904482 0.00740813 2023-06-10T21:41:54.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:41:54.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:41:54.561 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:54.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:54.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:54.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:54.703 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:54.748 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:54.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:54.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:54.879 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:54.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:54.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:55.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:55.106 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:55.151 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:55.215 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:55.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:55.323 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:41:55.350 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:55 smithi152 ceph-mon[156819]: pgmap v1395: 89 pgs: 89 active+clean; 2.1 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:41:55.362 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:55.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 15 37862 37847 131.379 71.5 0.00468292 0.00760692 2023-06-10T21:41:55.507 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:55.550 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:41:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:55.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:41:55.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:55 smithi064 ceph-mon[186706]: pgmap v1395: 89 pgs: 89 active+clean; 2.1 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:41:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:55 smithi064 ceph-mon[180297]: pgmap v1395: 89 pgs: 89 active+clean; 2.1 GiB data, 9.2 GiB used, 348 GiB / 358 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-06-10T21:41:55.663 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:41:55.711 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:56.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:41:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:56.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:41:56.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:41:56.256 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:41:56.298 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:56.394 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 39341 39325 129.326 92.375 0.0102205 0.00772663 2023-06-10T21:41:56.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:41:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:56.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:41:56.598 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:41:56.604 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:41:56.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:56.670 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:56.714 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:41:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:56.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:41:56.818 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:56.847 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:57.351 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:57 smithi152 ceph-mon[156819]: pgmap v1396: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:41:57.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:41:57.393312+0000 min lat: 0.0011044 max lat: 0.300894 avg lat: 0.00792346 2023-06-10T21:41:57.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:41:57.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40387 40371 126.129 65.375 0.00460636 0.00792346 2023-06-10T21:41:57.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:57 smithi064 ceph-mon[186706]: pgmap v1396: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:41:57.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:57 smithi064 ceph-mon[180297]: pgmap v1396: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-06-10T21:41:57.966 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:41:57.966 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:41:58.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 41247 41231 122.683 53.75 0.00966454 0.00814557 2023-06-10T21:41:58.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:58 smithi064 ceph-mon[186706]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:58.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:58 smithi064 ceph-mon[180297]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:58.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:58 smithi152 ceph-mon[156819]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:41:58.917 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:58.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:58.926 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1396,"stamp":"2023-06-10T21:41:56.130873+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2399672206,"num_objects":36631,"num_object_clones":0,"num_object_copies":109893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36631,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":36868,"num_write_kb":2345294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":2379958,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36802,"ondisk_log_size":36802,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":9987196,"kb_used_data":7186524,"kb_used_omap":15,"kb_used_meta":2800624,"kb_avail":364993412,"statfs":{"total":383980142592,"available":373753253888,"internally_reserved":0,"allocated":7359000576,"data_stored":7354951447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16284,"internal_metadata":2867839076},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,6,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1118830592,"num_objects":17072,"num_object_clones":0,"num_object_copies":51216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17072,"num_write_kb":1092608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004539"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":402,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.244142+0000","last_change":"2023-06-10T21:39:50.557548+0000","last_active":"2023-06-10T21:41:39.244142+0000","last_peered":"2023-06-10T21:41:39.244142+0000","last_clean":"2023-06-10T21:41:39.244142+0000","last_became_active":"2023-06-10T21:39:50.557023+0000","last_became_peered":"2023-06-10T21:39:50.557023+0000","last_unstale":"2023-06-10T21:41:39.244142+0000","last_undegraded":"2023-06-10T21:41:39.244142+0000","last_fullsized":"2023-06-10T21:41:39.244142+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:45.328782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248327+0000","last_change":"2023-06-10T21:40:30.243387+0000","last_active":"2023-06-10T21:41:39.248327+0000","last_peered":"2023-06-10T21:41:39.248327+0000","last_clean":"2023-06-10T21:41:39.248327+0000","last_became_active":"2023-06-10T21:40:30.243087+0000","last_became_peered":"2023-06-10T21:40:30.243087+0000","last_unstale":"2023-06-10T21:41:39.248327+0000","last_undegraded":"2023-06-10T21:41:39.248327+0000","last_fullsized":"2023-06-10T21:41:39.248327+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:11.139462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":309,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248811+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:41:39.248811+0000","last_peered":"2023-06-10T21:41:39.248811+0000","last_clean":"2023-06-10T21:41:39.248811+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:41:39.248811+0000","last_undegraded":"2023-06-10T21:41:39.248811+0000","last_fullsized":"2023-06-10T21:41:39.248811+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":249,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249467+0000","last_change":"2023-06-10T21:40:30.287586+0000","last_active":"2023-06-10T21:41:39.249467+0000","last_peered":"2023-06-10T21:41:39.249467+0000","last_clean":"2023-06-10T21:41:39.249467+0000","last_became_active":"2023-06-10T21:40:30.286423+0000","last_became_peered":"2023-06-10T21:40:30.286423+0000","last_unstale":"2023-06-10T21:41:39.249467+0000","last_undegraded":"2023-06-10T21:41:39.249467+0000","last_fullsized":"2023-06-10T21:41:39.249467+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247198+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:41:39.247198+0000","last_peered":"2023-06-10T21:41:39.247198+0000","last_clean":"2023-06-10T21:41:39.247198+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:41:39.247198+0000","last_undegraded":"2023-06-10T21:41:39.247198+0000","last_fullsized":"2023-06-10T21:41:39.247198+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.253196+0000","last_change":"2023-06-10T21:40:30.236647+0000","last_active":"2023-06-10T21:41:39.253196+0000","last_peered":"2023-06-10T21:41:39.253196+0000","last_clean":"2023-06-10T21:41:39.253196+0000","last_became_active":"2023-06-10T21:40:30.236307+0000","last_became_peered":"2023-06-10T21:40:30.236307+0000","last_unstale":"2023-06-10T21:41:39.253196+0000","last_undegraded":"2023-06-10T21:41:39.253196+0000","last_fullsized":"2023-06-10T21:41:39.253196+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":263,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252884+0000","last_change":"2023-06-10T21:40:30.280815+0000","last_active":"2023-06-10T21:41:39.252884+0000","last_peered":"2023-06-10T21:41:39.252884+0000","last_clean":"2023-06-10T21:41:39.252884+0000","last_became_active":"2023-06-10T21:40:30.280427+0000","last_became_peered":"2023-06-10T21:40:30.280427+0000","last_unstale":"2023-06-10T21:41:39.252884+0000","last_undegraded":"2023-06-10T21:41:39.252884+0000","last_fullsized":"2023-06-10T21:41:39.252884+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247349+0000","last_change":"2023-06-10T21:40:30.245395+0000","last_active":"2023-06-10T21:41:39.247349+0000","last_peered":"2023-06-10T21:41:39.247349+0000","last_clean":"2023-06-10T21:41:39.247349+0000","last_became_active":"2023-06-10T21:40:30.244982+0000","last_became_peered":"2023-06-10T21:40:30.244982+0000","last_unstale":"2023-06-10T21:41:39.247349+0000","last_undegraded":"2023-06-10T21:41:39.247349+0000","last_fullsized":"2023-06-10T21:41:39.247349+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:27:28.313757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":71,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.253364+0000","last_change":"2023-06-10T21:40:30.280871+0000","last_active":"2023-06-10T21:41:39.253364+0000","last_peered":"2023-06-10T21:41:39.253364+0000","last_clean":"2023-06-10T21:41:39.253364+0000","last_became_active":"2023-06-10T21:40:30.280332+0000","last_became_peered":"2023-06-10T21:40:30.280332+0000","last_unstale":"2023-06-10T21:41:39.253364+0000","last_undegraded":"2023-06-10T21:41:39.253364+0000","last_fullsized":"2023-06-10T21:41:39.253364+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":113,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248714+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:41:39.248714+0000","last_peered":"2023-06-10T21:41:39.248714+0000","last_clean":"2023-06-10T21:41:39.248714+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:41:39.248714+0000","last_undegraded":"2023-06-10T21:41:39.248714+0000","last_fullsized":"2023-06-10T21:41:39.248714+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":136,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249002+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:41:39.249002+0000","last_peered":"2023-06-10T21:41:39.249002+0000","last_clean":"2023-06-10T21:41:39.249002+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:41:39.249002+0000","last_undegraded":"2023-06-10T21:41:39.249002+0000","last_fullsized":"2023-06-10T21:41:39.249002+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":362,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.244251+0000","last_change":"2023-06-10T21:39:50.557388+0000","last_active":"2023-06-10T21:41:39.244251+0000","last_peered":"2023-06-10T21:41:39.244251+0000","last_clean":"2023-06-10T21:41:39.244251+0000","last_became_active":"2023-06-10T21:39:50.556659+0000","last_became_peered":"2023-06-10T21:39:50.556659+0000","last_unstale":"2023-06-10T21:41:39.244251+0000","last_undegraded":"2023-06-10T21:41:39.244251+0000","last_fullsized":"2023-06-10T21:41:39.244251+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:32.982830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":287,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249529+0000","last_change":"2023-06-10T21:39:30.845741+0000","last_active":"2023-06-10T21:41:39.249529+0000","last_peered":"2023-06-10T21:41:39.249529+0000","last_clean":"2023-06-10T21:41:39.249529+0000","last_became_active":"2023-06-10T21:39:30.234598+0000","last_became_peered":"2023-06-10T21:39:30.234598+0000","last_unstale":"2023-06-10T21:41:39.249529+0000","last_undegraded":"2023-06-10T21:41:39.249529+0000","last_fullsized":"2023-06-10T21:41:39.249529+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":118,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252067+0000","last_change":"2023-06-10T21:39:50.270986+0000","last_active":"2023-06-10T21:41:39.252067+0000","last_peered":"2023-06-10T21:41:39.252067+0000","last_clean":"2023-06-10T21:41:39.252067+0000","last_became_active":"2023-06-10T21:39:50.269843+0000","last_became_peered":"2023-06-10T21:39:50.269843+0000","last_unstale":"2023-06-10T21:41:39.252067+0000","last_undegraded":"2023-06-10T21:41:39.252067+0000","last_fullsized":"2023-06-10T21:41:39.252067+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":423,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249382+0000","last_change":"2023-06-10T21:39:50.781250+0000","last_active":"2023-06-10T21:41:39.249382+0000","last_peered":"2023-06-10T21:41:39.249382+0000","last_clean":"2023-06-10T21:41:39.249382+0000","last_became_active":"2023-06-10T21:39:50.376925+0000","last_became_peered":"2023-06-10T21:39:50.376925+0000","last_unstale":"2023-06-10T21:41:39.249382+0000","last_undegraded":"2023-06-10T21:41:39.249382+0000","last_fullsized":"2023-06-10T21:41:39.249382+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":325,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248107+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:41:39.248107+0000","last_peered":"2023-06-10T21:41:39.248107+0000","last_clean":"2023-06-10T21:41:39.248107+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:41:39.248107+0000","last_undegraded":"2023-06-10T21:41:39.248107+0000","last_fullsized":"2023-06-10T21:41:39.248107+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":249,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249027+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:41:39.249027+0000","last_peered":"2023-06-10T21:41:39.249027+0000","last_clean":"2023-06-10T21:41:39.249027+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:41:39.249027+0000","last_undegraded":"2023-06-10T21:41:39.249027+0000","last_fullsized":"2023-06-10T21:41:39.249027+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":80,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252161+0000","last_change":"2023-06-10T21:40:30.280758+0000","last_active":"2023-06-10T21:41:39.252161+0000","last_peered":"2023-06-10T21:41:39.252161+0000","last_clean":"2023-06-10T21:41:39.252161+0000","last_became_active":"2023-06-10T21:40:30.280270+0000","last_became_peered":"2023-06-10T21:40:30.280270+0000","last_unstale":"2023-06-10T21:41:39.252161+0000","last_undegraded":"2023-06-10T21:41:39.252161+0000","last_fullsized":"2023-06-10T21:41:39.252161+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":391,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249000+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:41:39.249000+0000","last_peered":"2023-06-10T21:41:39.249000+0000","last_clean":"2023-06-10T21:41:39.249000+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:41:39.249000+0000","last_undegraded":"2023-06-10T21:41:39.249000+0000","last_fullsized":"2023-06-10T21:41:39.249000+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248621+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:41:39.248621+0000","last_peered":"2023-06-10T21:41:39.248621+0000","last_clean":"2023-06-10T21:41:39.248621+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:41:39.248621+0000","last_undegraded":"2023-06-10T21:41:39.248621+0000","last_fullsized":"2023-06-10T21:41:39.248621+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":422,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251278+0000","last_change":"2023-06-10T21:40:30.476009+0000","last_active":"2023-06-10T21:41:39.251278+0000","last_peered":"2023-06-10T21:41:39.251278+0000","last_clean":"2023-06-10T21:41:39.251278+0000","last_became_active":"2023-06-10T21:40:30.237078+0000","last_became_peered":"2023-06-10T21:40:30.237078+0000","last_unstale":"2023-06-10T21:41:39.251278+0000","last_undegraded":"2023-06-10T21:41:39.251278+0000","last_fullsized":"2023-06-10T21:41:39.251278+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248926+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:41:39.248926+0000","last_peered":"2023-06-10T21:41:39.248926+0000","last_clean":"2023-06-10T21:41:39.248926+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:41:39.248926+0000","last_undegraded":"2023-06-10T21:41:39.248926+0000","last_fullsized":"2023-06-10T21:41:39.248926+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"379'2431","reported_seq":2446,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:55.678119+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:41:55.678119+0000","last_peered":"2023-06-10T21:41:55.678119+0000","last_clean":"2023-06-10T21:41:55.678119+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:41:55.678119+0000","last_undegraded":"2023-06-10T21:41:55.678119+0000","last_fullsized":"2023-06-10T21:41:55.678119+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2431,"log_dups_size":0,"ondisk_log_size":2431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159318016,"num_objects":2431,"num_object_clones":0,"num_object_copies":7293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2431,"num_write_kb":155584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":156,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249384+0000","last_change":"2023-06-10T21:40:30.567200+0000","last_active":"2023-06-10T21:41:39.249384+0000","last_peered":"2023-06-10T21:41:39.249384+0000","last_clean":"2023-06-10T21:41:39.249384+0000","last_became_active":"2023-06-10T21:40:30.285607+0000","last_became_peered":"2023-06-10T21:40:30.285607+0000","last_unstale":"2023-06-10T21:41:39.249384+0000","last_undegraded":"2023-06-10T21:41:39.249384+0000","last_fullsized":"2023-06-10T21:41:39.249384+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251310+0000","last_change":"2023-06-10T21:40:30.232813+0000","last_active":"2023-06-10T21:41:39.251310+0000","last_peered":"2023-06-10T21:41:39.251310+0000","last_clean":"2023-06-10T21:41:39.251310+0000","last_became_active":"2023-06-10T21:40:30.232432+0000","last_became_peered":"2023-06-10T21:40:30.232432+0000","last_unstale":"2023-06-10T21:41:39.251310+0000","last_undegraded":"2023-06-10T21:41:39.251310+0000","last_fullsized":"2023-06-10T21:41:39.251310+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"379'2308","reported_seq":2322,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.110711+0000","last_change":"2023-06-10T21:41:28.181199+0000","last_active":"2023-06-10T21:41:54.110711+0000","last_peered":"2023-06-10T21:41:54.110711+0000","last_clean":"2023-06-10T21:41:54.110711+0000","last_became_active":"2023-06-10T21:41:28.180841+0000","last_became_peered":"2023-06-10T21:41:28.180841+0000","last_unstale":"2023-06-10T21:41:54.110711+0000","last_undegraded":"2023-06-10T21:41:54.110711+0000","last_fullsized":"2023-06-10T21:41:54.110711+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2308,"log_dups_size":0,"ondisk_log_size":2308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151257088,"num_objects":2308,"num_object_clones":0,"num_object_copies":6924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2308,"num_write_kb":147712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":156,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249588+0000","last_change":"2023-06-10T21:40:30.576549+0000","last_active":"2023-06-10T21:41:39.249588+0000","last_peered":"2023-06-10T21:41:39.249588+0000","last_clean":"2023-06-10T21:41:39.249588+0000","last_became_active":"2023-06-10T21:40:30.286869+0000","last_became_peered":"2023-06-10T21:40:30.286869+0000","last_unstale":"2023-06-10T21:41:39.249588+0000","last_undegraded":"2023-06-10T21:41:39.249588+0000","last_fullsized":"2023-06-10T21:41:39.249588+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248898+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:41:39.248898+0000","last_peered":"2023-06-10T21:41:39.248898+0000","last_clean":"2023-06-10T21:41:39.248898+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:41:39.248898+0000","last_undegraded":"2023-06-10T21:41:39.248898+0000","last_fullsized":"2023-06-10T21:41:39.248898+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"379'2168","reported_seq":2183,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:52.443399+0000","last_change":"2023-06-10T21:41:28.177829+0000","last_active":"2023-06-10T21:41:52.443399+0000","last_peered":"2023-06-10T21:41:52.443399+0000","last_clean":"2023-06-10T21:41:52.443399+0000","last_became_active":"2023-06-10T21:41:28.177535+0000","last_became_peered":"2023-06-10T21:41:28.177535+0000","last_unstale":"2023-06-10T21:41:52.443399+0000","last_undegraded":"2023-06-10T21:41:52.443399+0000","last_fullsized":"2023-06-10T21:41:52.443399+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2168,"log_dups_size":0,"ondisk_log_size":2168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:34:42.879929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142082048,"num_objects":2168,"num_object_clones":0,"num_object_copies":6504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2168,"num_write_kb":138752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":443,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249095+0000","last_change":"2023-06-10T21:40:30.521312+0000","last_active":"2023-06-10T21:41:39.249095+0000","last_peered":"2023-06-10T21:41:39.249095+0000","last_clean":"2023-06-10T21:41:39.249095+0000","last_became_active":"2023-06-10T21:40:30.285713+0000","last_became_peered":"2023-06-10T21:40:30.285713+0000","last_unstale":"2023-06-10T21:41:39.249095+0000","last_undegraded":"2023-06-10T21:41:39.249095+0000","last_fullsized":"2023-06-10T21:41:39.249095+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248898+0000","last_change":"2023-06-10T21:40:30.343628+0000","last_active":"2023-06-10T21:41:39.248898+0000","last_peered":"2023-06-10T21:41:39.248898+0000","last_clean":"2023-06-10T21:41:39.248898+0000","last_became_active":"2023-06-10T21:40:30.343292+0000","last_became_peered":"2023-06-10T21:40:30.343292+0000","last_unstale":"2023-06-10T21:41:39.248898+0000","last_undegraded":"2023-06-10T21:41:39.248898+0000","last_fullsized":"2023-06-10T21:41:39.248898+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"379'2321","reported_seq":2335,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.060381+0000","last_change":"2023-06-10T21:41:28.177207+0000","last_active":"2023-06-10T21:41:54.060381+0000","last_peered":"2023-06-10T21:41:54.060381+0000","last_clean":"2023-06-10T21:41:54.060381+0000","last_became_active":"2023-06-10T21:41:28.176814+0000","last_became_peered":"2023-06-10T21:41:28.176814+0000","last_unstale":"2023-06-10T21:41:54.060381+0000","last_undegraded":"2023-06-10T21:41:54.060381+0000","last_fullsized":"2023-06-10T21:41:54.060381+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2321,"log_dups_size":0,"ondisk_log_size":2321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2321,"num_write_kb":148544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":81,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251959+0000","last_change":"2023-06-10T21:40:30.280834+0000","last_active":"2023-06-10T21:41:39.251959+0000","last_peered":"2023-06-10T21:41:39.251959+0000","last_clean":"2023-06-10T21:41:39.251959+0000","last_became_active":"2023-06-10T21:40:30.280353+0000","last_became_peered":"2023-06-10T21:40:30.280353+0000","last_unstale":"2023-06-10T21:41:39.251959+0000","last_undegraded":"2023-06-10T21:41:39.251959+0000","last_fullsized":"2023-06-10T21:41:39.251959+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249552+0000","last_change":"2023-06-10T21:40:30.272439+0000","last_active":"2023-06-10T21:41:39.249552+0000","last_peered":"2023-06-10T21:41:39.249552+0000","last_clean":"2023-06-10T21:41:39.249552+0000","last_became_active":"2023-06-10T21:40:30.272004+0000","last_became_peered":"2023-06-10T21:40:30.272004+0000","last_unstale":"2023-06-10T21:41:39.249552+0000","last_undegraded":"2023-06-10T21:41:39.249552+0000","last_fullsized":"2023-06-10T21:41:39.249552+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:30.425736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"379'2291","reported_seq":2305,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.055687+0000","last_change":"2023-06-10T21:41:28.178371+0000","last_active":"2023-06-10T21:41:54.055687+0000","last_peered":"2023-06-10T21:41:54.055687+0000","last_clean":"2023-06-10T21:41:54.055687+0000","last_became_active":"2023-06-10T21:41:28.178058+0000","last_became_peered":"2023-06-10T21:41:28.178058+0000","last_unstale":"2023-06-10T21:41:54.055687+0000","last_undegraded":"2023-06-10T21:41:54.055687+0000","last_fullsized":"2023-06-10T21:41:54.055687+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2291,"log_dups_size":0,"ondisk_log_size":2291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150142976,"num_objects":2291,"num_object_clones":0,"num_object_copies":6873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2291,"num_write_kb":146624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":35,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249463+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:41:39.249463+0000","last_peered":"2023-06-10T21:41:39.249463+0000","last_clean":"2023-06-10T21:41:39.249463+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:41:39.249463+0000","last_undegraded":"2023-06-10T21:41:39.249463+0000","last_fullsized":"2023-06-10T21:41:39.249463+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247877+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:41:39.247877+0000","last_peered":"2023-06-10T21:41:39.247877+0000","last_clean":"2023-06-10T21:41:39.247877+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:41:39.247877+0000","last_undegraded":"2023-06-10T21:41:39.247877+0000","last_fullsized":"2023-06-10T21:41:39.247877+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"379'2330","reported_seq":2345,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:55.677837+0000","last_change":"2023-06-10T21:41:28.182345+0000","last_active":"2023-06-10T21:41:55.677837+0000","last_peered":"2023-06-10T21:41:55.677837+0000","last_clean":"2023-06-10T21:41:55.677837+0000","last_became_active":"2023-06-10T21:41:28.181851+0000","last_became_peered":"2023-06-10T21:41:28.181851+0000","last_unstale":"2023-06-10T21:41:55.677837+0000","last_undegraded":"2023-06-10T21:41:55.677837+0000","last_fullsized":"2023-06-10T21:41:55.677837+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2330,"log_dups_size":0,"ondisk_log_size":2330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:07:28.251208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152698880,"num_objects":2330,"num_object_clones":0,"num_object_copies":6990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2330,"num_write_kb":149120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":183,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249623+0000","last_change":"2023-06-10T21:39:50.465026+0000","last_active":"2023-06-10T21:41:39.249623+0000","last_peered":"2023-06-10T21:41:39.249623+0000","last_clean":"2023-06-10T21:41:39.249623+0000","last_became_active":"2023-06-10T21:39:50.464762+0000","last_became_peered":"2023-06-10T21:39:50.464762+0000","last_unstale":"2023-06-10T21:41:39.249623+0000","last_undegraded":"2023-06-10T21:41:39.249623+0000","last_fullsized":"2023-06-10T21:41:39.249623+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:55:00.850195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":401,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:50.193151+0000","last_change":"2023-06-10T21:39:50.259203+0000","last_active":"2023-06-10T21:41:50.193151+0000","last_peered":"2023-06-10T21:41:50.193151+0000","last_clean":"2023-06-10T21:41:50.193151+0000","last_became_active":"2023-06-10T21:39:50.258873+0000","last_became_peered":"2023-06-10T21:39:50.258873+0000","last_unstale":"2023-06-10T21:41:50.193151+0000","last_undegraded":"2023-06-10T21:41:50.193151+0000","last_fullsized":"2023-06-10T21:41:50.193151+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:56:41.490096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"379'2196","reported_seq":2210,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.102643+0000","last_change":"2023-06-10T21:41:28.184243+0000","last_active":"2023-06-10T21:41:54.102643+0000","last_peered":"2023-06-10T21:41:54.102643+0000","last_clean":"2023-06-10T21:41:54.102643+0000","last_became_active":"2023-06-10T21:41:28.183870+0000","last_became_peered":"2023-06-10T21:41:28.183870+0000","last_unstale":"2023-06-10T21:41:54.102643+0000","last_undegraded":"2023-06-10T21:41:54.102643+0000","last_fullsized":"2023-06-10T21:41:54.102643+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2196,"log_dups_size":0,"ondisk_log_size":2196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143917056,"num_objects":2196,"num_object_clones":0,"num_object_copies":6588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2196,"num_write_kb":140544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":378,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.244056+0000","last_change":"2023-06-10T21:39:50.432352+0000","last_active":"2023-06-10T21:41:39.244056+0000","last_peered":"2023-06-10T21:41:39.244056+0000","last_clean":"2023-06-10T21:41:39.244056+0000","last_became_active":"2023-06-10T21:39:50.429920+0000","last_became_peered":"2023-06-10T21:39:50.429920+0000","last_unstale":"2023-06-10T21:41:39.244056+0000","last_undegraded":"2023-06-10T21:41:39.244056+0000","last_fullsized":"2023-06-10T21:41:39.244056+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:14.733503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248637+0000","last_change":"2023-06-10T21:40:30.351796+0000","last_active":"2023-06-10T21:41:39.248637+0000","last_peered":"2023-06-10T21:41:39.248637+0000","last_clean":"2023-06-10T21:41:39.248637+0000","last_became_active":"2023-06-10T21:40:30.350450+0000","last_became_peered":"2023-06-10T21:40:30.350450+0000","last_unstale":"2023-06-10T21:41:39.248637+0000","last_undegraded":"2023-06-10T21:41:39.248637+0000","last_fullsized":"2023-06-10T21:41:39.248637+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:38.306536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"379'2244","reported_seq":2258,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.055552+0000","last_change":"2023-06-10T21:41:28.177644+0000","last_active":"2023-06-10T21:41:54.055552+0000","last_peered":"2023-06-10T21:41:54.055552+0000","last_clean":"2023-06-10T21:41:54.055552+0000","last_became_active":"2023-06-10T21:41:28.177318+0000","last_became_peered":"2023-06-10T21:41:28.177318+0000","last_unstale":"2023-06-10T21:41:54.055552+0000","last_undegraded":"2023-06-10T21:41:54.055552+0000","last_fullsized":"2023-06-10T21:41:54.055552+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2244,"log_dups_size":0,"ondisk_log_size":2244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2244,"num_write_kb":143616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247597+0000","last_change":"2023-06-10T21:40:30.279010+0000","last_active":"2023-06-10T21:41:39.247597+0000","last_peered":"2023-06-10T21:41:39.247597+0000","last_clean":"2023-06-10T21:41:39.247597+0000","last_became_active":"2023-06-10T21:40:30.277943+0000","last_became_peered":"2023-06-10T21:40:30.277943+0000","last_unstale":"2023-06-10T21:41:39.247597+0000","last_undegraded":"2023-06-10T21:41:39.247597+0000","last_fullsized":"2023-06-10T21:41:39.247597+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:38:15.175343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":310,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249145+0000","last_change":"2023-06-10T21:39:50.233694+0000","last_active":"2023-06-10T21:41:39.249145+0000","last_peered":"2023-06-10T21:41:39.249145+0000","last_clean":"2023-06-10T21:41:39.249145+0000","last_became_active":"2023-06-10T21:39:50.233429+0000","last_became_peered":"2023-06-10T21:39:50.233429+0000","last_unstale":"2023-06-10T21:41:39.249145+0000","last_undegraded":"2023-06-10T21:41:39.249145+0000","last_fullsized":"2023-06-10T21:41:39.249145+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:15:42.432299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"379'2248","reported_seq":2260,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.045478+0000","last_change":"2023-06-10T21:41:28.176942+0000","last_active":"2023-06-10T21:41:54.045478+0000","last_peered":"2023-06-10T21:41:54.045478+0000","last_clean":"2023-06-10T21:41:54.045478+0000","last_became_active":"2023-06-10T21:41:28.176613+0000","last_became_peered":"2023-06-10T21:41:28.176613+0000","last_unstale":"2023-06-10T21:41:54.045478+0000","last_undegraded":"2023-06-10T21:41:54.045478+0000","last_fullsized":"2023-06-10T21:41:54.045478+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2248,"log_dups_size":0,"ondisk_log_size":2248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2248,"num_write_kb":143872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":162,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249513+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:41:39.249513+0000","last_peered":"2023-06-10T21:41:39.249513+0000","last_clean":"2023-06-10T21:41:39.249513+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:41:39.249513+0000","last_undegraded":"2023-06-10T21:41:39.249513+0000","last_fullsized":"2023-06-10T21:41:39.249513+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":62,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251868+0000","last_change":"2023-06-10T21:40:30.281222+0000","last_active":"2023-06-10T21:41:39.251868+0000","last_peered":"2023-06-10T21:41:39.251868+0000","last_clean":"2023-06-10T21:41:39.251868+0000","last_became_active":"2023-06-10T21:40:30.280846+0000","last_became_peered":"2023-06-10T21:40:30.280846+0000","last_unstale":"2023-06-10T21:41:39.251868+0000","last_undegraded":"2023-06-10T21:41:39.251868+0000","last_fullsized":"2023-06-10T21:41:39.251868+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251732+0000","last_change":"2023-06-10T21:39:50.466302+0000","last_active":"2023-06-10T21:41:39.251732+0000","last_peered":"2023-06-10T21:41:39.251732+0000","last_clean":"2023-06-10T21:41:39.251732+0000","last_became_active":"2023-06-10T21:39:50.465409+0000","last_became_peered":"2023-06-10T21:39:50.465409+0000","last_unstale":"2023-06-10T21:41:39.251732+0000","last_undegraded":"2023-06-10T21:41:39.251732+0000","last_fullsized":"2023-06-10T21:41:39.251732+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"379'2244","reported_seq":2258,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.116361+0000","last_change":"2023-06-10T21:41:28.182320+0000","last_active":"2023-06-10T21:41:54.116361+0000","last_peered":"2023-06-10T21:41:54.116361+0000","last_clean":"2023-06-10T21:41:54.116361+0000","last_became_active":"2023-06-10T21:41:28.181951+0000","last_became_peered":"2023-06-10T21:41:28.181951+0000","last_unstale":"2023-06-10T21:41:54.116361+0000","last_undegraded":"2023-06-10T21:41:54.116361+0000","last_fullsized":"2023-06-10T21:41:54.116361+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2244,"log_dups_size":0,"ondisk_log_size":2244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2244,"num_write_kb":143616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":81,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251592+0000","last_change":"2023-06-10T21:40:30.236317+0000","last_active":"2023-06-10T21:41:39.251592+0000","last_peered":"2023-06-10T21:41:39.251592+0000","last_clean":"2023-06-10T21:41:39.251592+0000","last_became_active":"2023-06-10T21:40:30.235998+0000","last_became_peered":"2023-06-10T21:40:30.235998+0000","last_unstale":"2023-06-10T21:41:39.251592+0000","last_undegraded":"2023-06-10T21:41:39.251592+0000","last_fullsized":"2023-06-10T21:41:39.251592+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":359,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252160+0000","last_change":"2023-06-10T21:40:30.281401+0000","last_active":"2023-06-10T21:41:39.252160+0000","last_peered":"2023-06-10T21:41:39.252160+0000","last_clean":"2023-06-10T21:41:39.252160+0000","last_became_active":"2023-06-10T21:40:30.280966+0000","last_became_peered":"2023-06-10T21:40:30.280966+0000","last_unstale":"2023-06-10T21:41:39.252160+0000","last_undegraded":"2023-06-10T21:41:39.252160+0000","last_fullsized":"2023-06-10T21:41:39.252160+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":242,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.245482+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:41:39.245482+0000","last_peered":"2023-06-10T21:41:39.245482+0000","last_clean":"2023-06-10T21:41:39.245482+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:41:39.245482+0000","last_undegraded":"2023-06-10T21:41:39.245482+0000","last_fullsized":"2023-06-10T21:41:39.245482+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"379'2268","reported_seq":2282,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.102421+0000","last_change":"2023-06-10T21:41:28.182336+0000","last_active":"2023-06-10T21:41:54.102421+0000","last_peered":"2023-06-10T21:41:54.102421+0000","last_clean":"2023-06-10T21:41:54.102421+0000","last_became_active":"2023-06-10T21:41:28.181961+0000","last_became_peered":"2023-06-10T21:41:28.181961+0000","last_unstale":"2023-06-10T21:41:54.102421+0000","last_undegraded":"2023-06-10T21:41:54.102421+0000","last_fullsized":"2023-06-10T21:41:54.102421+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2268,"log_dups_size":0,"ondisk_log_size":2268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2268,"num_write_kb":145152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":262,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.244053+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:41:39.244053+0000","last_peered":"2023-06-10T21:41:39.244053+0000","last_clean":"2023-06-10T21:41:39.244053+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:41:39.244053+0000","last_undegraded":"2023-06-10T21:41:39.244053+0000","last_fullsized":"2023-06-10T21:41:39.244053+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":252,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:40.518377+0000","last_change":"2023-06-10T21:40:30.229244+0000","last_active":"2023-06-10T21:41:40.518377+0000","last_peered":"2023-06-10T21:41:40.518377+0000","last_clean":"2023-06-10T21:41:40.518377+0000","last_became_active":"2023-06-10T21:40:30.228964+0000","last_became_peered":"2023-06-10T21:40:30.228964+0000","last_unstale":"2023-06-10T21:41:40.518377+0000","last_undegraded":"2023-06-10T21:41:40.518377+0000","last_fullsized":"2023-06-10T21:41:40.518377+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:41.629358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":80,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252994+0000","last_change":"2023-06-10T21:40:30.236712+0000","last_active":"2023-06-10T21:41:39.252994+0000","last_peered":"2023-06-10T21:41:39.252994+0000","last_clean":"2023-06-10T21:41:39.252994+0000","last_became_active":"2023-06-10T21:40:30.236404+0000","last_became_peered":"2023-06-10T21:40:30.236404+0000","last_unstale":"2023-06-10T21:41:39.252994+0000","last_undegraded":"2023-06-10T21:41:39.252994+0000","last_fullsized":"2023-06-10T21:41:39.252994+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"379'2187","reported_seq":2200,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.070017+0000","last_change":"2023-06-10T21:41:28.181267+0000","last_active":"2023-06-10T21:41:54.070017+0000","last_peered":"2023-06-10T21:41:54.070017+0000","last_clean":"2023-06-10T21:41:54.070017+0000","last_became_active":"2023-06-10T21:41:28.180932+0000","last_became_peered":"2023-06-10T21:41:28.180932+0000","last_unstale":"2023-06-10T21:41:54.070017+0000","last_undegraded":"2023-06-10T21:41:54.070017+0000","last_fullsized":"2023-06-10T21:41:54.070017+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2187,"log_dups_size":0,"ondisk_log_size":2187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2187,"num_write_kb":139968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":53,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251366+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:41:39.251366+0000","last_peered":"2023-06-10T21:41:39.251366+0000","last_clean":"2023-06-10T21:41:39.251366+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:41:39.251366+0000","last_undegraded":"2023-06-10T21:41:39.251366+0000","last_fullsized":"2023-06-10T21:41:39.251366+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":579,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248684+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:41:39.248684+0000","last_peered":"2023-06-10T21:41:39.248684+0000","last_clean":"2023-06-10T21:41:39.248684+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:41:39.248684+0000","last_undegraded":"2023-06-10T21:41:39.248684+0000","last_fullsized":"2023-06-10T21:41:39.248684+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249397+0000","last_change":"2023-06-10T21:40:30.260288+0000","last_active":"2023-06-10T21:41:39.249397+0000","last_peered":"2023-06-10T21:41:39.249397+0000","last_clean":"2023-06-10T21:41:39.249397+0000","last_became_active":"2023-06-10T21:40:30.259958+0000","last_became_peered":"2023-06-10T21:40:30.259958+0000","last_unstale":"2023-06-10T21:41:39.249397+0000","last_undegraded":"2023-06-10T21:41:39.249397+0000","last_fullsized":"2023-06-10T21:41:39.249397+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:41.077016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":247,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249183+0000","last_change":"2023-06-10T21:40:30.286026+0000","last_active":"2023-06-10T21:41:39.249183+0000","last_peered":"2023-06-10T21:41:39.249183+0000","last_clean":"2023-06-10T21:41:39.249183+0000","last_became_active":"2023-06-10T21:40:30.285670+0000","last_became_peered":"2023-06-10T21:40:30.285670+0000","last_unstale":"2023-06-10T21:41:39.249183+0000","last_undegraded":"2023-06-10T21:41:39.249183+0000","last_fullsized":"2023-06-10T21:41:39.249183+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"379'2248","reported_seq":2262,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.093863+0000","last_change":"2023-06-10T21:41:28.183802+0000","last_active":"2023-06-10T21:41:54.093863+0000","last_peered":"2023-06-10T21:41:54.093863+0000","last_clean":"2023-06-10T21:41:54.093863+0000","last_became_active":"2023-06-10T21:41:28.183511+0000","last_became_peered":"2023-06-10T21:41:28.183511+0000","last_unstale":"2023-06-10T21:41:54.093863+0000","last_undegraded":"2023-06-10T21:41:54.093863+0000","last_fullsized":"2023-06-10T21:41:54.093863+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2248,"log_dups_size":0,"ondisk_log_size":2248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147324928,"num_objects":2248,"num_object_clones":0,"num_object_copies":6744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2248,"num_write_kb":143872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":416,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248743+0000","last_change":"2023-06-10T21:39:50.237017+0000","last_active":"2023-06-10T21:41:39.248743+0000","last_peered":"2023-06-10T21:41:39.248743+0000","last_clean":"2023-06-10T21:41:39.248743+0000","last_became_active":"2023-06-10T21:39:50.232270+0000","last_became_peered":"2023-06-10T21:39:50.232270+0000","last_unstale":"2023-06-10T21:41:39.248743+0000","last_undegraded":"2023-06-10T21:41:39.248743+0000","last_fullsized":"2023-06-10T21:41:39.248743+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249441+0000","last_change":"2023-06-10T21:40:30.316582+0000","last_active":"2023-06-10T21:41:39.249441+0000","last_peered":"2023-06-10T21:41:39.249441+0000","last_clean":"2023-06-10T21:41:39.249441+0000","last_became_active":"2023-06-10T21:40:30.315657+0000","last_became_peered":"2023-06-10T21:40:30.315657+0000","last_unstale":"2023-06-10T21:41:39.249441+0000","last_undegraded":"2023-06-10T21:41:39.249441+0000","last_fullsized":"2023-06-10T21:41:39.249441+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:08:04.756800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":114,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:40.518695+0000","last_change":"2023-06-10T21:39:50.559093+0000","last_active":"2023-06-10T21:41:40.518695+0000","last_peered":"2023-06-10T21:41:40.518695+0000","last_clean":"2023-06-10T21:41:40.518695+0000","last_became_active":"2023-06-10T21:39:50.558289+0000","last_became_peered":"2023-06-10T21:39:50.558289+0000","last_unstale":"2023-06-10T21:41:40.518695+0000","last_undegraded":"2023-06-10T21:41:40.518695+0000","last_fullsized":"2023-06-10T21:41:40.518695+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:37.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"379'2389","reported_seq":2404,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:55.667754+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:41:55.667754+0000","last_peered":"2023-06-10T21:41:55.667754+0000","last_clean":"2023-06-10T21:41:55.667754+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:41:55.667754+0000","last_undegraded":"2023-06-10T21:41:55.667754+0000","last_fullsized":"2023-06-10T21:41:55.667754+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2389,"log_dups_size":0,"ondisk_log_size":2389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156565504,"num_objects":2389,"num_object_clones":0,"num_object_copies":7167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2389,"num_write_kb":152896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":410,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249121+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:41:39.249121+0000","last_peered":"2023-06-10T21:41:39.249121+0000","last_clean":"2023-06-10T21:41:39.249121+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:41:39.249121+0000","last_undegraded":"2023-06-10T21:41:39.249121+0000","last_fullsized":"2023-06-10T21:41:39.249121+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":242,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.245563+0000","last_change":"2023-06-10T21:39:50.296322+0000","last_active":"2023-06-10T21:41:39.245563+0000","last_peered":"2023-06-10T21:41:39.245563+0000","last_clean":"2023-06-10T21:41:39.245563+0000","last_became_active":"2023-06-10T21:39:50.295201+0000","last_became_peered":"2023-06-10T21:39:50.295201+0000","last_unstale":"2023-06-10T21:41:39.245563+0000","last_undegraded":"2023-06-10T21:41:39.245563+0000","last_fullsized":"2023-06-10T21:41:39.245563+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:28.403434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":208,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252230+0000","last_change":"2023-06-10T21:39:50.271175+0000","last_active":"2023-06-10T21:41:39.252230+0000","last_peered":"2023-06-10T21:41:39.252230+0000","last_clean":"2023-06-10T21:41:39.252230+0000","last_became_active":"2023-06-10T21:39:50.270302+0000","last_became_peered":"2023-06-10T21:39:50.270302+0000","last_unstale":"2023-06-10T21:41:39.252230+0000","last_undegraded":"2023-06-10T21:41:39.252230+0000","last_fullsized":"2023-06-10T21:41:39.252230+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"379'2294","reported_seq":2308,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:54.053150+0000","last_change":"2023-06-10T21:41:28.179779+0000","last_active":"2023-06-10T21:41:54.053150+0000","last_peered":"2023-06-10T21:41:54.053150+0000","last_clean":"2023-06-10T21:41:54.053150+0000","last_became_active":"2023-06-10T21:41:28.179455+0000","last_became_peered":"2023-06-10T21:41:28.179455+0000","last_unstale":"2023-06-10T21:41:54.053150+0000","last_undegraded":"2023-06-10T21:41:54.053150+0000","last_fullsized":"2023-06-10T21:41:54.053150+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2294,"log_dups_size":0,"ondisk_log_size":2294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150339584,"num_objects":2294,"num_object_clones":0,"num_object_copies":6882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2294,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":372,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249010+0000","last_change":"2023-06-10T21:39:30.266087+0000","last_active":"2023-06-10T21:41:39.249010+0000","last_peered":"2023-06-10T21:41:39.249010+0000","last_clean":"2023-06-10T21:41:39.249010+0000","last_became_active":"2023-06-10T21:39:30.231858+0000","last_became_peered":"2023-06-10T21:39:30.231858+0000","last_unstale":"2023-06-10T21:41:39.249010+0000","last_undegraded":"2023-06-10T21:41:39.249010+0000","last_fullsized":"2023-06-10T21:41:39.249010+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:08:09.982313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":183,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248802+0000","last_change":"2023-06-10T21:39:50.227162+0000","last_active":"2023-06-10T21:41:39.248802+0000","last_peered":"2023-06-10T21:41:39.248802+0000","last_clean":"2023-06-10T21:41:39.248802+0000","last_became_active":"2023-06-10T21:39:50.226891+0000","last_became_peered":"2023-06-10T21:39:50.226891+0000","last_unstale":"2023-06-10T21:41:39.248802+0000","last_undegraded":"2023-06-10T21:41:39.248802+0000","last_fullsized":"2023-06-10T21:41:39.248802+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:48:59.680224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247514+0000","last_change":"2023-06-10T21:40:30.279185+0000","last_active":"2023-06-10T21:41:39.247514+0000","last_peered":"2023-06-10T21:41:39.247514+0000","last_clean":"2023-06-10T21:41:39.247514+0000","last_became_active":"2023-06-10T21:40:30.278330+0000","last_became_peered":"2023-06-10T21:40:30.278330+0000","last_unstale":"2023-06-10T21:41:39.247514+0000","last_undegraded":"2023-06-10T21:41:39.247514+0000","last_fullsized":"2023-06-10T21:41:39.247514+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:15.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"379'2440","reported_seq":2455,"reported_epoch":379,"state":"active+clean","last_fresh":"2023-06-10T21:41:55.666611+0000","last_change":"2023-06-10T21:41:28.176991+0000","last_active":"2023-06-10T21:41:55.666611+0000","last_peered":"2023-06-10T21:41:55.666611+0000","last_clean":"2023-06-10T21:41:55.666611+0000","last_became_active":"2023-06-10T21:41:28.176650+0000","last_became_peered":"2023-06-10T21:41:28.176650+0000","last_unstale":"2023-06-10T21:41:55.666611+0000","last_undegraded":"2023-06-10T21:41:55.666611+0000","last_fullsized":"2023-06-10T21:41:55.666611+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":2440,"log_dups_size":0,"ondisk_log_size":2440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:14.567179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159907840,"num_objects":2440,"num_object_clones":0,"num_object_copies":7320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2440,"num_write_kb":156160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":127,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249483+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:41:39.249483+0000","last_peered":"2023-06-10T21:41:39.249483+0000","last_clean":"2023-06-10T21:41:39.249483+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:41:39.249483+0000","last_undegraded":"2023-06-10T21:41:39.249483+0000","last_fullsized":"2023-06-10T21:41:39.249483+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":226,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249255+0000","last_change":"2023-06-10T21:40:30.580293+0000","last_active":"2023-06-10T21:41:39.249255+0000","last_peered":"2023-06-10T21:41:39.249255+0000","last_clean":"2023-06-10T21:41:39.249255+0000","last_became_active":"2023-06-10T21:40:30.391049+0000","last_became_peered":"2023-06-10T21:40:30.391049+0000","last_unstale":"2023-06-10T21:41:39.249255+0000","last_undegraded":"2023-06-10T21:41:39.249255+0000","last_fullsized":"2023-06-10T21:41:39.249255+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":62,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.251346+0000","last_change":"2023-06-10T21:39:30.208137+0000","last_active":"2023-06-10T21:41:39.251346+0000","last_peered":"2023-06-10T21:41:39.251346+0000","last_clean":"2023-06-10T21:41:39.251346+0000","last_became_active":"2023-06-10T21:39:30.207848+0000","last_became_peered":"2023-06-10T21:39:30.207848+0000","last_unstale":"2023-06-10T21:41:39.251346+0000","last_undegraded":"2023-06-10T21:41:39.251346+0000","last_fullsized":"2023-06-10T21:41:39.251346+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":78,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249540+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:41:39.249540+0000","last_peered":"2023-06-10T21:41:39.249540+0000","last_clean":"2023-06-10T21:41:39.249540+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:41:39.249540+0000","last_undegraded":"2023-06-10T21:41:39.249540+0000","last_fullsized":"2023-06-10T21:41:39.249540+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":259,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.245640+0000","last_change":"2023-06-10T21:40:30.227732+0000","last_active":"2023-06-10T21:41:39.245640+0000","last_peered":"2023-06-10T21:41:39.245640+0000","last_clean":"2023-06-10T21:41:39.245640+0000","last_became_active":"2023-06-10T21:40:30.227380+0000","last_became_peered":"2023-06-10T21:40:30.227380+0000","last_unstale":"2023-06-10T21:41:39.245640+0000","last_undegraded":"2023-06-10T21:41:39.245640+0000","last_fullsized":"2023-06-10T21:41:39.245640+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:33.673485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":66,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252061+0000","last_change":"2023-06-10T21:39:50.304413+0000","last_active":"2023-06-10T21:41:39.252061+0000","last_peered":"2023-06-10T21:41:39.252061+0000","last_clean":"2023-06-10T21:41:39.252061+0000","last_became_active":"2023-06-10T21:39:50.303414+0000","last_became_peered":"2023-06-10T21:39:50.303414+0000","last_unstale":"2023-06-10T21:41:39.252061+0000","last_undegraded":"2023-06-10T21:41:39.252061+0000","last_fullsized":"2023-06-10T21:41:39.252061+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":282,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249731+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:41:39.249731+0000","last_peered":"2023-06-10T21:41:39.249731+0000","last_clean":"2023-06-10T21:41:39.249731+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:41:39.249731+0000","last_undegraded":"2023-06-10T21:41:39.249731+0000","last_fullsized":"2023-06-10T21:41:39.249731+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":83,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252249+0000","last_change":"2023-06-10T21:40:30.565123+0000","last_active":"2023-06-10T21:41:39.252249+0000","last_peered":"2023-06-10T21:41:39.252249+0000","last_clean":"2023-06-10T21:41:39.252249+0000","last_became_active":"2023-06-10T21:40:30.280290+0000","last_became_peered":"2023-06-10T21:40:30.280290+0000","last_unstale":"2023-06-10T21:41:39.252249+0000","last_undegraded":"2023-06-10T21:41:39.252249+0000","last_fullsized":"2023-06-10T21:41:39.252249+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":338,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.249698+0000","last_change":"2023-06-10T21:39:30.245702+0000","last_active":"2023-06-10T21:41:39.249698+0000","last_peered":"2023-06-10T21:41:39.249698+0000","last_clean":"2023-06-10T21:41:39.249698+0000","last_became_active":"2023-06-10T21:39:30.245426+0000","last_became_peered":"2023-06-10T21:39:30.245426+0000","last_unstale":"2023-06-10T21:41:39.249698+0000","last_undegraded":"2023-06-10T21:41:39.249698+0000","last_fullsized":"2023-06-10T21:41:39.249698+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:52:27.316156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248715+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:41:39.248715+0000","last_peered":"2023-06-10T21:41:39.248715+0000","last_clean":"2023-06-10T21:41:39.248715+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:41:39.248715+0000","last_undegraded":"2023-06-10T21:41:39.248715+0000","last_fullsized":"2023-06-10T21:41:39.248715+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.248353+0000","last_change":"2023-06-10T21:40:30.253755+0000","last_active":"2023-06-10T21:41:39.248353+0000","last_peered":"2023-06-10T21:41:39.248353+0000","last_clean":"2023-06-10T21:41:39.248353+0000","last_became_active":"2023-06-10T21:40:30.253530+0000","last_became_peered":"2023-06-10T21:40:30.253530+0000","last_unstale":"2023-06-10T21:41:39.248353+0000","last_undegraded":"2023-06-10T21:41:39.248353+0000","last_fullsized":"2023-06-10T21:41:39.248353+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:14.839319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":319,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.252785+0000","last_change":"2023-06-10T21:40:30.574471+0000","last_active":"2023-06-10T21:41:39.252785+0000","last_peered":"2023-06-10T21:41:39.252785+0000","last_clean":"2023-06-10T21:41:39.252785+0000","last_became_active":"2023-06-10T21:40:30.281351+0000","last_became_peered":"2023-06-10T21:40:30.281351+0000","last_unstale":"2023-06-10T21:41:39.252785+0000","last_undegraded":"2023-06-10T21:41:39.252785+0000","last_fullsized":"2023-06-10T21:41:39.252785+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-06-10T21:41:39.247680+0000","last_change":"2023-06-10T21:40:30.245261+0000","last_active":"2023-06-10T21:41:39.247680+0000","last_peered":"2023-06-10T21:41:39.247680+0000","last_clean":"2023-06-10T21:41:39.247680+0000","last_became_active":"2023-06-10T21:40:30.244695+0000","last_became_peered":"2023-06-10T21:40:30.244695+0000","last_unstale":"2023-06-10T21:41:39.247680+0000","last_undegraded":"2023-06-10T21:41:39.247680+0000","last_fullsized":"2023-06-10T21:41:39.247680+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:54.784460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2399076352,"num_objects":36607,"num_object_clones":0,"num_object_copies":109821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36607,"num_write_kb":2342848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7116095488,"data_stored":7116095488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36607,"ondisk_log_size":36607,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":18486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430738,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2403524,"kb_used_data":1562172,"kb_used_omap":0,"kb_used_meta":841344,"kb_avail":91341628,"statfs":{"total":95995035648,"available":93533827072,"internally_reserved":0,"allocated":1599664128,"data_stored":1598967267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":861536256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,5,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.0470000000000002,"15min":4.282},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.1110000000000002,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.093,"5min":2.4199999999999999,"15min":4.5679999999999996},"min":{"1min":0.28899999999999998,"5min":0.25,"15min":0.223},"max":{"1min":4.8380000000000001,"5min":56.628,"15min":185.58000000000001},"last":0.753}]},{"osd":1,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.9660000000000002,"15min":2.976},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":38.078000000000003,"5min":60.863,"15min":93.093999999999994},"last":0.96499999999999997},{"interface":"front","average":{"1min":3.476,"5min":3.2919999999999998,"15min":3.1000000000000001},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":38.341000000000001,"5min":61.148000000000003,"15min":93.119},"last":0.71799999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:41:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.867,"15min":4.9009999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":24.106000000000002,"5min":137.64099999999999,"15min":172.33600000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.4089999999999998,"5min":4.54,"15min":4.8369999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29499999999999998},"max":{"1min":24.132999999999999,"5min":136.96600000000001,"15min":172.185},"last":0.82099999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":4.3460000000000001,"15min":4.2649999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":24.375,"5min":102.25700000000001,"15min":147.00700000000001},"last":0.93700000000000006},{"interface":"front","average":{"1min":2.347,"5min":4.1070000000000002,"15min":3.923},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":26.460000000000001,"5min":103.401,"15min":146.93899999999999},"last":0.93300000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.52100000000000002,"15min":0.52300000000000002},"min":{"1min":0.29799999999999999,"5min":0.253,"15min":0.249},"max":{"1min":1.0289999999999999,"5min":5.859,"15min":5.859},"last":1.175},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.81399999999999995,"15min":0.66700000000000004},"min":{"1min":0.32000000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.3290000000000002,"5min":22.77,"15min":22.77},"last":1.732}]},{"osd":5,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.48399999999999999,"15min":0.64200000000000002},"min":{"1min":0.26900000000000002,"5min":0.252,"15min":0.252},"max":{"1min":1.8220000000000001,"5min":2.7930000000000001,"15min":12.048},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.57199999999999995,"15min":0.56499999999999995},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.97499999999999998,"5min":5.9100000000000001,"15min":6.7530000000000001},"last":4.0099999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.68300000000000005,"15min":0.72399999999999998},"min":{"1min":0.34699999999999998,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":1.847,"5min":10.890000000000001,"15min":13.073},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.59299999999999997,"15min":0.56799999999999995},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.0209999999999999,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.90300000000000002}]}]},{"osd":6,"up_from":0,"seq":180388626966,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566408,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020385,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075797,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2573696,"kb_used_data":1788856,"kb_used_omap":7,"kb_used_meta":784824,"kb_avail":91171456,"statfs":{"total":95995035648,"available":93359570944,"internally_reserved":0,"allocated":1831788544,"data_stored":1830515252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":803659857},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":6.0129999999999999,"15min":4.8410000000000002},"min":{"1min":0.23999999999999999,"5min":0.19,"15min":0.19},"max":{"1min":5.6500000000000004,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":5.4619999999999997,"15min":4.8570000000000002},"min":{"1min":0.29099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6280000000000001,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":5.9130000000000003,"5min":8.6440000000000001,"15min":5.3680000000000003},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.19},"max":{"1min":85.709999999999994,"5min":171.286,"15min":171.286},"last":0.318},{"interface":"front","average":{"1min":6.3300000000000001,"5min":4.7469999999999999,"15min":3.827},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":85.501999999999995,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Jun 10 21:41:20 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":5.694,"15min":5.5270000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":70.480999999999995,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":5.4980000000000002,"5min":5.3550000000000004,"15min":5.7530000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":71.144999999999996,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.746}]},{"osd":4,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":3.4969999999999999,"15min":3.5449999999999999},"min":{"1min":0.36699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.093,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.2060000000000004,"15min":4.4390000000000001},"min":{"1min":0.35599999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.423,"5min":171.18100000000001,"15min":223.28399999999999},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.613,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.7759999999999998,"5min":171.303,"15min":171.303},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.5430000000000001,"15min":2.3279999999999998},"min":{"1min":0.38,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1280000000000001,"5min":171.203,"15min":171.203},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":3.585,"15min":2.4420000000000002},"min":{"1min":0.48599999999999999,"5min":0.307,"15min":0.307},"max":{"1min":2.02,"5min":171.441,"15min":171.441},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.86099999999999999,"5min":5.2610000000000001,"15min":3.3090000000000002},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":3.4630000000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":1.1240000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":3.7360000000000002,"15min":4.4139999999999997},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":5.7400000000000002,"5min":171.26499999999999,"15min":219.131},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.85199999999999998,"5min":3.5670000000000002,"15min":4.4219999999999997},"min":{"1min":0.41599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.0489999999999999,"5min":171.52000000000001,"15min":219.19},"last":0.71899999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824156,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2298364,"kb_used_data":1786804,"kb_used_omap":7,"kb_used_meta":511544,"kb_avail":91446788,"statfs":{"total":95995035648,"available":93641510912,"internally_reserved":0,"allocated":1829687296,"data_stored":1828297384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":523821075},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":3.3039999999999998,"15min":4.234},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.744,"5min":30.744,"15min":105.09},"last":0.32900000000000001},{"interface":"front","average":{"1min":3.7559999999999998,"5min":3.3029999999999999,"15min":4.1200000000000001},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":30.805,"5min":30.805,"15min":99.852999999999994},"last":0.27400000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.9830000000000001,"15min":3.819},"min":{"1min":0.23599999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.6509999999999998,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.325,"5min":3.5960000000000001,"15min":3.7690000000000001},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.9649999999999999,"5min":41.18,"15min":94.691000000000003},"last":0.437}]},{"osd":2,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":4.8869999999999996,"15min":6.4669999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.206,"5min":231.07900000000001,"15min":231.07900000000001},"last":0.312},{"interface":"front","average":{"1min":1.8879999999999999,"5min":6.1260000000000003,"15min":7.9299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":7.6909999999999998,"5min":231.58600000000001,"15min":231.58600000000001},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.379,"15min":1.407},"min":{"1min":0.32100000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.0820000000000007,"5min":10.368,"15min":13.417},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.2309999999999999,"15min":2.4990000000000001},"min":{"1min":0.38400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.6150000000000002,"5min":19.244,"15min":19.728000000000002},"last":0.55900000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.847,"15min":1.5489999999999999},"min":{"1min":0.42899999999999999,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":3.8559999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.4830000000000001,"15min":1.5049999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":9.1210000000000004,"5min":10.414999999999999,"15min":13.470000000000001},"last":0.76200000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":2.254,"15min":2.52},"min":{"1min":0.46800000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.5270000000000001,"5min":19.164999999999999,"15min":19.805},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.839,"15min":1.5800000000000001},"min":{"1min":0.49399999999999999,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":3.8180000000000001,"5min":17.327000000000002,"15min":17.327000000000002},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.3330000000000002,"15min":2.778},"min":{"1min":0.50800000000000001,"5min":0.379,"15min":0.33900000000000002},"max":{"1min":7.5789999999999997,"5min":19.193000000000001,"15min":79.984999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.9350000000000001,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":3.6960000000000002,"5min":17.251000000000001,"15min":80.272999999999996},"last":5.1950000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019422,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823194,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2711612,"kb_used_data":2048692,"kb_used_omap":0,"kb_used_meta":662912,"kb_avail":91033540,"statfs":{"total":95995035648,"available":93218344960,"internally_reserved":0,"allocated":2097860608,"data_stored":2097171544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":678821888},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.655,"15min":3.2349999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.4530000000000003,"5min":18.562000000000001,"15min":134.71899999999999},"last":0.318},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.121,"15min":3.4710000000000001},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.2709999999999999,"5min":27.256,"15min":135.386},"last":1.054}]},{"osd":1,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":2.4750000000000001,"15min":2.4060000000000001},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.4830000000000001,"5min":60.399999999999999,"15min":71.349000000000004},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":2.363,"15min":2.3439999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":2.3929999999999998,"5min":59.780999999999999,"15min":71.331999999999994},"last":1.1479999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":5.8949999999999996,"15min":6.0910000000000002},"min":{"1min":0.373,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.3330000000000002,"5min":324.21600000000001,"15min":324.21600000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.177,"5min":5.5650000000000004,"15min":5.8639999999999999},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.1779999999999999,"5min":324.16300000000001,"15min":324.16300000000001},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":4.5609999999999999,"15min":4.3949999999999996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.2510000000000003,"5min":90.725999999999999,"15min":128.089},"last":1.655},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.9390000000000001,"15min":4.3959999999999999},"min":{"1min":0.39000000000000001,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":5.907,"5min":90.180000000000007,"15min":128.24299999999999},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.51000000000000001,"15min":1.135},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":2.7000000000000002,"5min":5.3460000000000001,"15min":165.73699999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.59999999999999998,"15min":1.2330000000000001},"min":{"1min":0.34000000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":3.2829999999999999,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.60599999999999998,"15min":0.68300000000000005},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.22},"max":{"1min":3.4580000000000002,"5min":5.3739999999999997,"15min":29.760000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.751,"5min":0.61399999999999999,"15min":0.68700000000000006},"min":{"1min":0.378,"5min":0.34399999999999997,"15min":0.29199999999999998},"max":{"1min":3.4340000000000002,"5min":4.3380000000000001,"15min":28.995999999999999},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.6359999999999999,"15min":1.46},"min":{"1min":0.33900000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":1.179,"5min":94.102000000000004,"15min":136.904},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6850000000000001,"15min":1.4930000000000001},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":2.754,"5min":95.653999999999996,"15min":137.661},"last":0.43099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1883111424,"data_stored":1883111424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1519517696,"data_stored":1519517696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2074345472,"data_stored":2074345472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1639120896,"data_stored":1639120896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:41:58.933 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:41:58.933 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:41:58.933 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-06-10T21:41:58.934 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:41:58.934 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:41:58.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-06-10T21:41:59.027 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:41:59.040 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:41:59.090 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:59.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:41:59.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:41:59.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:59.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:41:59.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:41:59.355 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:41:59 smithi152 ceph-mon[156819]: pgmap v1397: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-06-10T21:41:59.360 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:59.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 42184 42168 119.768 58.5625 0.0101833 0.00834273 2023-06-10T21:41:59.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:41:59 smithi064 ceph-mon[186706]: pgmap v1397: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-06-10T21:41:59.514 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:41:59 smithi064 ceph-mon[180297]: pgmap v1397: 89 pgs: 89 active+clean; 2.3 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 105 MiB/s wr, 1.67k op/s 2023-06-10T21:41:59.523 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:41:59.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:41:59.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:41:59.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:41:59.652 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:41:59.716 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:41:59.818 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:00.163 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:00.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:00.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:00.203 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:42:00.237 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3536491515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.238 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3536491515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.239 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:00.325 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:00.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 43548 43532 118.267 85.25 0.0143805 0.00844907 2023-06-10T21:42:00.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3536491515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:00.687 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:00.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:00.793 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:00.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:01.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 44847 44831 116.722 81.1875 0.00899404 0.00854326 2023-06-10T21:42:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:01 smithi152 ceph-mon[156819]: pgmap v1398: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-06-10T21:42:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:01 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:01 smithi152 ceph-mon[156819]: osdmap e380: 8 total, 8 up, 4 in 2023-06-10T21:42:01.460 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[186706]: pgmap v1398: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-06-10T21:42:01.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:01.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:01.461 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[186706]: osdmap e380: 8 total, 8 up, 4 in 2023-06-10T21:42:01.461 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[180297]: pgmap v1398: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-06-10T21:42:01.461 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:01.462 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:01.462 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:01 smithi064 ceph-mon[180297]: osdmap e380: 8 total, 8 up, 4 in 2023-06-10T21:42:01.620 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:01.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:01.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:01.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:01.776 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:01.998 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:02.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:02.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:02.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:02.248 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:02.272 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:02.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:02.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:02.284 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:02.316 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:02.359 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:02.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:02.394 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:02.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 46006 45990 114.951 72.4375 0.00902245 0.00869466 2023-06-10T21:42:02.403 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:02 smithi152 ceph-mon[156819]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.403 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:02 smithi152 ceph-mon[156819]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.404 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:02 smithi152 ceph-mon[156819]: pgmap v1400: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[186706]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[186706]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[186706]: pgmap v1400: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[180297]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[180297]: failed to encode map e380 with expected crc 2023-06-10T21:42:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:02 smithi064 ceph-mon[180297]: pgmap v1400: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-10T21:42:02.433 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:42:02.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:02.466 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:02.507 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.525 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.526 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.526 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.526 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:42:02.546 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:02.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:02.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:02.710 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:02] "GET /metrics HTTP/1.1" 200 224168 "" "Prometheus/2.33.4" 2023-06-10T21:42:03.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:03.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:03.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 46953 46937 112.806 59.1875 0.00657635 0.00885946 2023-06-10T21:42:03.413 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:03.430 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:42:03.430 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "" 2023-06-10T21:42:03.430 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:42:03.467 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/267415297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:03.467 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:03.472 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/267415297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:03.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:03.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/267415297' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:03.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:03.724 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:03.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:03.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:03.755 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:42:03.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:03.923 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:04.058 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:04.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:04.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:04.206 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:04.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 48382 48366 111.936 89.3125 0.0116782 0.00892826 2023-06-10T21:42:04.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:04 smithi064 ceph-mon[186706]: pgmap v1401: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-06-10T21:42:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:04 smithi064 ceph-mon[180297]: pgmap v1401: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-06-10T21:42:04.533 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:04.542 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:04.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:04 smithi152 ceph-mon[156819]: pgmap v1401: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-06-10T21:42:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:04.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:04.647 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:04.728 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:04.914 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:04.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:05.019 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:05.092 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:05.169 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:42:05.170 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:42:05.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:42:05.170 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [] 2023-06-10T21:42:05.170 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:42:05.201 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:05.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:05.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:05.310 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:05.357 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:05.395 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 49691 49675 110.86 81.8125 0.00986548 0.00901568 2023-06-10T21:42:05.592 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:05.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:05.699 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:42:05.727 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:05.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:42:05.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.884 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:05.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:06.109 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:42:06.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:06.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:06.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:06.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:06.117 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:06.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:42:06.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:06.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:42:06.279 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:06.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:06.399 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 50676 50660 109.149 61.5625 0.0179544 0.00915608 2023-06-10T21:42:06.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:06.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:06.560 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:06.814 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:06.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:06.892 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:06.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:42:06.984 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:06.989 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:07.079 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:42:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:07.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:42:07.184 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:07.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:07 smithi152 ceph-mon[156819]: pgmap v1402: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:07.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:42:07.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:42:07.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:07.247 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:07.328 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:07.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:07.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:07.398 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 52007 51991 108.283 83.1875 0.0129063 0.00923121 2023-06-10T21:42:07.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:07.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[186706]: pgmap v1402: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:07.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:42:07.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:42:07.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:07.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[180297]: pgmap v1402: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:07.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:42:07.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:42:07.516 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:07.521 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:07.612 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:07.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:07.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:07.729 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:07.786 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:07.878 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:07.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:07.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:08.009 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:08.051 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:08.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:08.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:08.155 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:08.178 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:42:08.182 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:08.236 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:08.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:08.341 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:08.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:08.398 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 52931 52915 106.653 57.75 0.0121114 0.00937099 2023-06-10T21:42:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:09.076 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:53571 13.f 13:f91daa81:::benchmark_data_smithi064_501823_object53570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e381)", 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:42:08.836925+0000", 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.23730380200000001, 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23733411400000001, 2023-06-10T21:42:09.077 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 53571 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:42:09.078 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836925+0000", 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836925+0000", 2023-06-10T21:42:09.079 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836924+0000", 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9142999999999999e-05 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.080 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836983+0000", 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5290000000000003e-06 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:42:09.081 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836991+0000", 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5999999999999997e-06 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.836996+0000", 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002920972 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.082 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.839917+0000", 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000135988 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.840053+0000", 2023-06-10T21:42:09.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016539300000000001 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.840219+0000", 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039762790000000001 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.084 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844195+0000", 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00074182700000000002 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844937+0000", 2023-06-10T21:42:09.085 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080121519999999998 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:53583 13.e 13:70773ada:::benchmark_data_smithi064_501823_object53582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e381)", 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:42:08.844709+0000", 2023-06-10T21:42:09.086 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.229519787, 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22961402, 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 53583 2023-06-10T21:42:09.087 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844709+0000", 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.088 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844709+0000", 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999852 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844694+0000", 2023-06-10T21:42:09.089 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2552999999999996e-05 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844757+0000", 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3152000000000001e-05 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.090 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844780+0000", 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.37e-06 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844786+0000", 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106666 2023-06-10T21:42:09.091 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.844893+0000", 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013258700000000001 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.092 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845025+0000", 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000170132 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845196+0000", 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021910499999999999 2023-06-10T21:42:09.093 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.847387+0000", 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035218179999999999 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:42:09.094 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.850908+0000", 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061997529999999997 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:53584 13.f 13:ffaabb9b:::benchmark_data_smithi064_501823_object53583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e381)", 2023-06-10T21:42:09.095 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:42:08.845651+0000", 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.228577314, 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.22874050700000001, 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:42:09.096 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 53584 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845651+0000", 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:42:09.097 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845651+0000", 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:42:09.098 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845650+0000", 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3398000000000003e-05 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845734+0000", 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4800000000000007e-06 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.099 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845743+0000", 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.131e-06 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.845750+0000", 2023-06-10T21:42:09.100 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0014141550000000001 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.847165+0000", 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106321 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.101 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.847271+0000", 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014552799999999999 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.847416+0000", 2023-06-10T21:42:09.102 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001768695 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.849185+0000", 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044272840000000001 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:42:09.103 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:42:08.853612+0000", 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0079611560000000005 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:42:09.104 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 3 2023-06-10T21:42:09.105 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:42:09.180 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:09 smithi152 ceph-mon[156819]: pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:09.180 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:09 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:09.180 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:09 smithi152 ceph-mon[156819]: failed to encode map e381 with expected crc 2023-06-10T21:42:09.180 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:09.181 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:09 smithi152 ceph-mon[156819]: osdmap e381: 8 total, 8 up, 4 in 2023-06-10T21:42:09.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:09.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:09.398 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 54031 54015 105.469 68.75 0.0124428 0.00947631 2023-06-10T21:42:09.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[180297]: pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:09.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:09.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[180297]: failed to encode map e381 with expected crc 2023-06-10T21:42:09.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:09.435 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[180297]: osdmap e381: 8 total, 8 up, 4 in 2023-06-10T21:42:09.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[186706]: pgmap v1403: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-10T21:42:09.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:09.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[186706]: failed to encode map e381 with expected crc 2023-06-10T21:42:09.436 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1936868160' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:09.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:09 smithi064 ceph-mon[186706]: osdmap e381: 8 total, 8 up, 4 in 2023-06-10T21:42:09.490 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:09.538 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:09.687 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:09.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:09.698 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:42:09.699 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:42:09.699 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-06-10T21:42:09.699 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:42:09.792 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:09.866 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:09.970 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:09.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:10.002 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:10.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:10.076 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:10.077 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:10.120 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:10.147 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:10.162 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:10.167 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:42:10.207 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:42:10.241 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:10.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:10.246 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1479056342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:10.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1479056342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:10.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:10.281 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.291 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.292 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.292 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:42:10.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:10.399 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 54982 54966 104.074 59.4375 0.0144662 0.00960362 2023-06-10T21:42:10.408 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:10.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1479056342' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:10.506 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:10.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:10.616 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:10.679 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:10.754 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:10.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:10.773 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:10.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:42:10.887 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:10.998 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:42:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:11.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:42:11.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:11.160 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:11.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 56072 56056 103.011 68.125 0.0117088 0.00970157 2023-06-10T21:42:11.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:11 smithi064 ceph-mon[180297]: pgmap v1405: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:42:11.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:11 smithi064 ceph-mon[186706]: pgmap v1405: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:42:11.415 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:11.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:11.448 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:11.466 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:42:11.531 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:11.571 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:11 smithi152 ceph-mon[156819]: pgmap v1405: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-10T21:42:11.573 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:11.677 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:11.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:11.784 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:11.814 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:12.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:12.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:12.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:12.166 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:12.288 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:12.289 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:12 smithi064 ceph-mon[186706]: pgmap v1406: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-06-10T21:42:12.290 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:12 smithi064 ceph-mon[180297]: pgmap v1406: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-06-10T21:42:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:12.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:12.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:12.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 57163 57147 102.016 68.1875 0.00625613 0.00979721 2023-06-10T21:42:12.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:12.526 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:12.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:12.631 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:12.637 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:12 smithi152 ceph-mon[156819]: pgmap v1406: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 73 MiB/s wr, 1.16k op/s 2023-06-10T21:42:12.705 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:12.820 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:12.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:12.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:12.919 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:12.930 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:12.956 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:42:12.956 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:42:13.005 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:13.150 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:13.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:13.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:12] "GET /metrics HTTP/1.1" 200 224146 "" "Prometheus/2.33.4" 2023-06-10T21:42:13.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:13.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:13.322 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:13.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 58470 58454 101.451 81.6875 0.0122662 0.00985183 2023-06-10T21:42:13.411 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:13.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:13.431 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:42:13.517 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:13.548 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:13.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:13.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:14.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:42:14.089 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:14.147 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:14.197 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:14.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:14.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:14.272 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:42:14.277 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:42:14.301 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:14.328 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:14.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:14.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:14.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:14.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 59148 59132 99.8548 42.375 0.00717485 0.00994502 2023-06-10T21:42:14.443 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:14.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:14 smithi152 ceph-mon[156819]: pgmap v1407: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:42:14.477 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:14.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:14 smithi064 ceph-mon[180297]: pgmap v1407: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:42:14.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:14 smithi064 ceph-mon[186706]: pgmap v1407: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-10T21:42:14.521 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:15.242 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:15 smithi064 ceph-mon[180297]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:42:15.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:15.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:15.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 60163 60147 98.8962 63.4375 0.0147387 0.0101065 2023-06-10T21:42:15.428 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:15.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:15.618 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:15.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:15.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:15 smithi064 ceph-mon[186706]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:42:15.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:15.670 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1407,"stamp":"2023-06-10T21:42:14.138418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3801094030,"num_objects":58015,"num_object_clones":0,"num_object_copies":174045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58015,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":58252,"num_write_kb":3713870,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":2379958,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58186,"ondisk_log_size":58186,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":271,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":14060284,"kb_used_data":11175132,"kb_used_omap":15,"kb_used_meta":2885040,"kb_avail":360920324,"statfs":{"total":383980142592,"available":369582411776,"internally_reserved":0,"allocated":11443335168,"data_stored":11439282295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16284,"internal_metadata":2954281060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":854065152,"num_objects":13032,"num_object_clones":0,"num_object_copies":39096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13032,"num_write_kb":834048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004213"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":404,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.787602+0000","last_change":"2023-06-10T21:39:50.557548+0000","last_active":"2023-06-10T21:42:00.787602+0000","last_peered":"2023-06-10T21:42:00.787602+0000","last_clean":"2023-06-10T21:42:00.787602+0000","last_became_active":"2023-06-10T21:39:50.557023+0000","last_became_peered":"2023-06-10T21:39:50.557023+0000","last_unstale":"2023-06-10T21:42:00.787602+0000","last_undegraded":"2023-06-10T21:42:00.787602+0000","last_fullsized":"2023-06-10T21:42:00.787602+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:45.328782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784996+0000","last_change":"2023-06-10T21:40:30.243387+0000","last_active":"2023-06-10T21:42:00.784996+0000","last_peered":"2023-06-10T21:42:00.784996+0000","last_clean":"2023-06-10T21:42:00.784996+0000","last_became_active":"2023-06-10T21:40:30.243087+0000","last_became_peered":"2023-06-10T21:40:30.243087+0000","last_unstale":"2023-06-10T21:42:00.784996+0000","last_undegraded":"2023-06-10T21:42:00.784996+0000","last_fullsized":"2023-06-10T21:42:00.784996+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:33:11.139462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":311,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784978+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:42:00.784978+0000","last_peered":"2023-06-10T21:42:00.784978+0000","last_clean":"2023-06-10T21:42:00.784978+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:42:00.784978+0000","last_undegraded":"2023-06-10T21:42:00.784978+0000","last_fullsized":"2023-06-10T21:42:00.784978+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":251,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785445+0000","last_change":"2023-06-10T21:40:30.287586+0000","last_active":"2023-06-10T21:42:00.785445+0000","last_peered":"2023-06-10T21:42:00.785445+0000","last_clean":"2023-06-10T21:42:00.785445+0000","last_became_active":"2023-06-10T21:40:30.286423+0000","last_became_peered":"2023-06-10T21:40:30.286423+0000","last_unstale":"2023-06-10T21:42:00.785445+0000","last_undegraded":"2023-06-10T21:42:00.785445+0000","last_fullsized":"2023-06-10T21:42:00.785445+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.782959+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:42:00.782959+0000","last_peered":"2023-06-10T21:42:00.782959+0000","last_clean":"2023-06-10T21:42:00.782959+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:42:00.782959+0000","last_undegraded":"2023-06-10T21:42:00.782959+0000","last_fullsized":"2023-06-10T21:42:00.782959+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":82,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785441+0000","last_change":"2023-06-10T21:40:30.236647+0000","last_active":"2023-06-10T21:42:00.785441+0000","last_peered":"2023-06-10T21:42:00.785441+0000","last_clean":"2023-06-10T21:42:00.785441+0000","last_became_active":"2023-06-10T21:40:30.236307+0000","last_became_peered":"2023-06-10T21:40:30.236307+0000","last_unstale":"2023-06-10T21:42:00.785441+0000","last_undegraded":"2023-06-10T21:42:00.785441+0000","last_fullsized":"2023-06-10T21:42:00.785441+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":265,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784772+0000","last_change":"2023-06-10T21:40:30.280815+0000","last_active":"2023-06-10T21:42:00.784772+0000","last_peered":"2023-06-10T21:42:00.784772+0000","last_clean":"2023-06-10T21:42:00.784772+0000","last_became_active":"2023-06-10T21:40:30.280427+0000","last_became_peered":"2023-06-10T21:40:30.280427+0000","last_unstale":"2023-06-10T21:42:00.784772+0000","last_undegraded":"2023-06-10T21:42:00.784772+0000","last_fullsized":"2023-06-10T21:42:00.784772+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783196+0000","last_change":"2023-06-10T21:40:30.245395+0000","last_active":"2023-06-10T21:42:00.783196+0000","last_peered":"2023-06-10T21:42:00.783196+0000","last_clean":"2023-06-10T21:42:00.783196+0000","last_became_active":"2023-06-10T21:40:30.244982+0000","last_became_peered":"2023-06-10T21:40:30.244982+0000","last_unstale":"2023-06-10T21:42:00.783196+0000","last_undegraded":"2023-06-10T21:42:00.783196+0000","last_fullsized":"2023-06-10T21:42:00.783196+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:27:28.313757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":73,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.787037+0000","last_change":"2023-06-10T21:40:30.280871+0000","last_active":"2023-06-10T21:42:00.787037+0000","last_peered":"2023-06-10T21:42:00.787037+0000","last_clean":"2023-06-10T21:42:00.787037+0000","last_became_active":"2023-06-10T21:40:30.280332+0000","last_became_peered":"2023-06-10T21:40:30.280332+0000","last_unstale":"2023-06-10T21:42:00.787037+0000","last_undegraded":"2023-06-10T21:42:00.787037+0000","last_fullsized":"2023-06-10T21:42:00.787037+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784887+0000","last_change":"2023-06-10T21:38:37.280711+0000","last_active":"2023-06-10T21:42:00.784887+0000","last_peered":"2023-06-10T21:42:00.784887+0000","last_clean":"2023-06-10T21:42:00.784887+0000","last_became_active":"2023-06-10T21:38:37.280430+0000","last_became_peered":"2023-06-10T21:38:37.280430+0000","last_unstale":"2023-06-10T21:42:00.784887+0000","last_undegraded":"2023-06-10T21:42:00.784887+0000","last_fullsized":"2023-06-10T21:42:00.784887+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:51:19.463525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":138,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785463+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:42:00.785463+0000","last_peered":"2023-06-10T21:42:00.785463+0000","last_clean":"2023-06-10T21:42:00.785463+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:42:00.785463+0000","last_undegraded":"2023-06-10T21:42:00.785463+0000","last_fullsized":"2023-06-10T21:42:00.785463+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":364,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.787654+0000","last_change":"2023-06-10T21:39:50.557388+0000","last_active":"2023-06-10T21:42:00.787654+0000","last_peered":"2023-06-10T21:42:00.787654+0000","last_clean":"2023-06-10T21:42:00.787654+0000","last_became_active":"2023-06-10T21:39:50.556659+0000","last_became_peered":"2023-06-10T21:39:50.556659+0000","last_unstale":"2023-06-10T21:42:00.787654+0000","last_undegraded":"2023-06-10T21:42:00.787654+0000","last_fullsized":"2023-06-10T21:42:00.787654+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:29:32.982830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":289,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785919+0000","last_change":"2023-06-10T21:39:30.845741+0000","last_active":"2023-06-10T21:42:00.785919+0000","last_peered":"2023-06-10T21:42:00.785919+0000","last_clean":"2023-06-10T21:42:00.785919+0000","last_became_active":"2023-06-10T21:39:30.234598+0000","last_became_peered":"2023-06-10T21:39:30.234598+0000","last_unstale":"2023-06-10T21:42:00.785919+0000","last_undegraded":"2023-06-10T21:42:00.785919+0000","last_fullsized":"2023-06-10T21:42:00.785919+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":120,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785104+0000","last_change":"2023-06-10T21:39:50.270986+0000","last_active":"2023-06-10T21:42:00.785104+0000","last_peered":"2023-06-10T21:42:00.785104+0000","last_clean":"2023-06-10T21:42:00.785104+0000","last_became_active":"2023-06-10T21:39:50.269843+0000","last_became_peered":"2023-06-10T21:39:50.269843+0000","last_unstale":"2023-06-10T21:42:00.785104+0000","last_undegraded":"2023-06-10T21:42:00.785104+0000","last_fullsized":"2023-06-10T21:42:00.785104+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":425,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786319+0000","last_change":"2023-06-10T21:39:50.781250+0000","last_active":"2023-06-10T21:42:00.786319+0000","last_peered":"2023-06-10T21:42:00.786319+0000","last_clean":"2023-06-10T21:42:00.786319+0000","last_became_active":"2023-06-10T21:39:50.376925+0000","last_became_peered":"2023-06-10T21:39:50.376925+0000","last_unstale":"2023-06-10T21:42:00.786319+0000","last_undegraded":"2023-06-10T21:42:00.786319+0000","last_fullsized":"2023-06-10T21:42:00.786319+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":327,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784897+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:42:00.784897+0000","last_peered":"2023-06-10T21:42:00.784897+0000","last_clean":"2023-06-10T21:42:00.784897+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:42:00.784897+0000","last_undegraded":"2023-06-10T21:42:00.784897+0000","last_fullsized":"2023-06-10T21:42:00.784897+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":251,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785387+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:42:00.785387+0000","last_peered":"2023-06-10T21:42:00.785387+0000","last_clean":"2023-06-10T21:42:00.785387+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:42:00.785387+0000","last_undegraded":"2023-06-10T21:42:00.785387+0000","last_fullsized":"2023-06-10T21:42:00.785387+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":82,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784978+0000","last_change":"2023-06-10T21:40:30.280758+0000","last_active":"2023-06-10T21:42:00.784978+0000","last_peered":"2023-06-10T21:42:00.784978+0000","last_clean":"2023-06-10T21:42:00.784978+0000","last_became_active":"2023-06-10T21:40:30.280270+0000","last_became_peered":"2023-06-10T21:40:30.280270+0000","last_unstale":"2023-06-10T21:42:00.784978+0000","last_undegraded":"2023-06-10T21:42:00.784978+0000","last_fullsized":"2023-06-10T21:42:00.784978+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":393,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785275+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:42:00.785275+0000","last_peered":"2023-06-10T21:42:00.785275+0000","last_clean":"2023-06-10T21:42:00.785275+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:42:00.785275+0000","last_undegraded":"2023-06-10T21:42:00.785275+0000","last_fullsized":"2023-06-10T21:42:00.785275+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785017+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:42:00.785017+0000","last_peered":"2023-06-10T21:42:00.785017+0000","last_clean":"2023-06-10T21:42:00.785017+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:42:00.785017+0000","last_undegraded":"2023-06-10T21:42:00.785017+0000","last_fullsized":"2023-06-10T21:42:00.785017+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":424,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784237+0000","last_change":"2023-06-10T21:40:30.476009+0000","last_active":"2023-06-10T21:42:00.784237+0000","last_peered":"2023-06-10T21:42:00.784237+0000","last_clean":"2023-06-10T21:42:00.784237+0000","last_became_active":"2023-06-10T21:40:30.237078+0000","last_became_peered":"2023-06-10T21:40:30.237078+0000","last_unstale":"2023-06-10T21:42:00.784237+0000","last_undegraded":"2023-06-10T21:42:00.784237+0000","last_fullsized":"2023-06-10T21:42:00.784237+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784501+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:42:00.784501+0000","last_peered":"2023-06-10T21:42:00.784501+0000","last_clean":"2023-06-10T21:42:00.784501+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:42:00.784501+0000","last_undegraded":"2023-06-10T21:42:00.784501+0000","last_fullsized":"2023-06-10T21:42:00.784501+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"381'3484","reported_seq":3501,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:10.694463+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:42:10.694463+0000","last_peered":"2023-06-10T21:42:10.694463+0000","last_clean":"2023-06-10T21:42:10.694463+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:42:10.694463+0000","last_undegraded":"2023-06-10T21:42:10.694463+0000","last_fullsized":"2023-06-10T21:42:10.694463+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3484,"log_dups_size":0,"ondisk_log_size":3484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228327424,"num_objects":3484,"num_object_clones":0,"num_object_copies":10452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3484,"num_write_kb":222976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":158,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786167+0000","last_change":"2023-06-10T21:40:30.567200+0000","last_active":"2023-06-10T21:42:00.786167+0000","last_peered":"2023-06-10T21:42:00.786167+0000","last_clean":"2023-06-10T21:42:00.786167+0000","last_became_active":"2023-06-10T21:40:30.285607+0000","last_became_peered":"2023-06-10T21:40:30.285607+0000","last_unstale":"2023-06-10T21:42:00.786167+0000","last_undegraded":"2023-06-10T21:42:00.786167+0000","last_fullsized":"2023-06-10T21:42:00.786167+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784350+0000","last_change":"2023-06-10T21:40:30.232813+0000","last_active":"2023-06-10T21:42:00.784350+0000","last_peered":"2023-06-10T21:42:00.784350+0000","last_clean":"2023-06-10T21:42:00.784350+0000","last_became_active":"2023-06-10T21:40:30.232432+0000","last_became_peered":"2023-06-10T21:40:30.232432+0000","last_unstale":"2023-06-10T21:42:00.784350+0000","last_undegraded":"2023-06-10T21:42:00.784350+0000","last_fullsized":"2023-06-10T21:42:00.784350+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"381'3751","reported_seq":3767,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.113284+0000","last_change":"2023-06-10T21:41:28.181199+0000","last_active":"2023-06-10T21:42:14.113284+0000","last_peered":"2023-06-10T21:42:14.113284+0000","last_clean":"2023-06-10T21:42:14.113284+0000","last_became_active":"2023-06-10T21:41:28.180841+0000","last_became_peered":"2023-06-10T21:41:28.180841+0000","last_unstale":"2023-06-10T21:42:14.113284+0000","last_undegraded":"2023-06-10T21:42:14.113284+0000","last_fullsized":"2023-06-10T21:42:14.113284+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3751,"log_dups_size":0,"ondisk_log_size":3751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245825536,"num_objects":3751,"num_object_clones":0,"num_object_copies":11253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3751,"num_write_kb":240064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":158,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785521+0000","last_change":"2023-06-10T21:40:30.576549+0000","last_active":"2023-06-10T21:42:00.785521+0000","last_peered":"2023-06-10T21:42:00.785521+0000","last_clean":"2023-06-10T21:42:00.785521+0000","last_became_active":"2023-06-10T21:40:30.286869+0000","last_became_peered":"2023-06-10T21:40:30.286869+0000","last_unstale":"2023-06-10T21:42:00.785521+0000","last_undegraded":"2023-06-10T21:42:00.785521+0000","last_fullsized":"2023-06-10T21:42:00.785521+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785071+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:42:00.785071+0000","last_peered":"2023-06-10T21:42:00.785071+0000","last_clean":"2023-06-10T21:42:00.785071+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:42:00.785071+0000","last_undegraded":"2023-06-10T21:42:00.785071+0000","last_fullsized":"2023-06-10T21:42:00.785071+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"381'3588","reported_seq":3605,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:12.448012+0000","last_change":"2023-06-10T21:41:28.177829+0000","last_active":"2023-06-10T21:42:12.448012+0000","last_peered":"2023-06-10T21:42:12.448012+0000","last_clean":"2023-06-10T21:42:12.448012+0000","last_became_active":"2023-06-10T21:41:28.177535+0000","last_became_peered":"2023-06-10T21:41:28.177535+0000","last_unstale":"2023-06-10T21:42:12.448012+0000","last_undegraded":"2023-06-10T21:42:12.448012+0000","last_fullsized":"2023-06-10T21:42:12.448012+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3588,"log_dups_size":0,"ondisk_log_size":3588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:34:42.879929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235143168,"num_objects":3588,"num_object_clones":0,"num_object_copies":10764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3588,"num_write_kb":229632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":445,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785564+0000","last_change":"2023-06-10T21:40:30.521312+0000","last_active":"2023-06-10T21:42:00.785564+0000","last_peered":"2023-06-10T21:42:00.785564+0000","last_clean":"2023-06-10T21:42:00.785564+0000","last_became_active":"2023-06-10T21:40:30.285713+0000","last_became_peered":"2023-06-10T21:40:30.285713+0000","last_unstale":"2023-06-10T21:42:00.785564+0000","last_undegraded":"2023-06-10T21:42:00.785564+0000","last_fullsized":"2023-06-10T21:42:00.785564+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":129,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785214+0000","last_change":"2023-06-10T21:40:30.343628+0000","last_active":"2023-06-10T21:42:00.785214+0000","last_peered":"2023-06-10T21:42:00.785214+0000","last_clean":"2023-06-10T21:42:00.785214+0000","last_became_active":"2023-06-10T21:40:30.343292+0000","last_became_peered":"2023-06-10T21:40:30.343292+0000","last_unstale":"2023-06-10T21:42:00.785214+0000","last_undegraded":"2023-06-10T21:42:00.785214+0000","last_fullsized":"2023-06-10T21:42:00.785214+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"381'3743","reported_seq":3759,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.050537+0000","last_change":"2023-06-10T21:41:28.177207+0000","last_active":"2023-06-10T21:42:14.050537+0000","last_peered":"2023-06-10T21:42:14.050537+0000","last_clean":"2023-06-10T21:42:14.050537+0000","last_became_active":"2023-06-10T21:41:28.176814+0000","last_became_peered":"2023-06-10T21:41:28.176814+0000","last_unstale":"2023-06-10T21:42:14.050537+0000","last_undegraded":"2023-06-10T21:42:14.050537+0000","last_fullsized":"2023-06-10T21:42:14.050537+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3743,"log_dups_size":0,"ondisk_log_size":3743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3743,"num_write_kb":239552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":83,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784352+0000","last_change":"2023-06-10T21:40:30.280834+0000","last_active":"2023-06-10T21:42:00.784352+0000","last_peered":"2023-06-10T21:42:00.784352+0000","last_clean":"2023-06-10T21:42:00.784352+0000","last_became_active":"2023-06-10T21:40:30.280353+0000","last_became_peered":"2023-06-10T21:40:30.280353+0000","last_unstale":"2023-06-10T21:42:00.784352+0000","last_undegraded":"2023-06-10T21:42:00.784352+0000","last_fullsized":"2023-06-10T21:42:00.784352+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786572+0000","last_change":"2023-06-10T21:40:30.272439+0000","last_active":"2023-06-10T21:42:00.786572+0000","last_peered":"2023-06-10T21:42:00.786572+0000","last_clean":"2023-06-10T21:42:00.786572+0000","last_became_active":"2023-06-10T21:40:30.272004+0000","last_became_peered":"2023-06-10T21:40:30.272004+0000","last_unstale":"2023-06-10T21:42:00.786572+0000","last_undegraded":"2023-06-10T21:42:00.786572+0000","last_fullsized":"2023-06-10T21:42:00.786572+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:10:30.425736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"381'3686","reported_seq":3702,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.052051+0000","last_change":"2023-06-10T21:41:28.178371+0000","last_active":"2023-06-10T21:42:14.052051+0000","last_peered":"2023-06-10T21:42:14.052051+0000","last_clean":"2023-06-10T21:42:14.052051+0000","last_became_active":"2023-06-10T21:41:28.178058+0000","last_became_peered":"2023-06-10T21:41:28.178058+0000","last_unstale":"2023-06-10T21:42:14.052051+0000","last_undegraded":"2023-06-10T21:42:14.052051+0000","last_fullsized":"2023-06-10T21:42:14.052051+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3686,"log_dups_size":0,"ondisk_log_size":3686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241565696,"num_objects":3686,"num_object_clones":0,"num_object_copies":11058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3686,"num_write_kb":235904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":37,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786522+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:42:00.786522+0000","last_peered":"2023-06-10T21:42:00.786522+0000","last_clean":"2023-06-10T21:42:00.786522+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:42:00.786522+0000","last_undegraded":"2023-06-10T21:42:00.786522+0000","last_fullsized":"2023-06-10T21:42:00.786522+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784769+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:42:00.784769+0000","last_peered":"2023-06-10T21:42:00.784769+0000","last_clean":"2023-06-10T21:42:00.784769+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:42:00.784769+0000","last_undegraded":"2023-06-10T21:42:00.784769+0000","last_fullsized":"2023-06-10T21:42:00.784769+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"381'3437","reported_seq":3454,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:10.662476+0000","last_change":"2023-06-10T21:41:28.182345+0000","last_active":"2023-06-10T21:42:10.662476+0000","last_peered":"2023-06-10T21:42:10.662476+0000","last_clean":"2023-06-10T21:42:10.662476+0000","last_became_active":"2023-06-10T21:41:28.181851+0000","last_became_peered":"2023-06-10T21:41:28.181851+0000","last_unstale":"2023-06-10T21:42:10.662476+0000","last_undegraded":"2023-06-10T21:42:10.662476+0000","last_fullsized":"2023-06-10T21:42:10.662476+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3437,"log_dups_size":0,"ondisk_log_size":3437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:07:28.251208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225247232,"num_objects":3437,"num_object_clones":0,"num_object_copies":10311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3437,"num_write_kb":219968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":186,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783743+0000","last_change":"2023-06-10T21:39:50.465026+0000","last_active":"2023-06-10T21:42:00.783743+0000","last_peered":"2023-06-10T21:42:00.783743+0000","last_clean":"2023-06-10T21:42:00.783743+0000","last_became_active":"2023-06-10T21:39:50.464762+0000","last_became_peered":"2023-06-10T21:39:50.464762+0000","last_unstale":"2023-06-10T21:42:00.783743+0000","last_undegraded":"2023-06-10T21:42:00.783743+0000","last_fullsized":"2023-06-10T21:42:00.783743+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:55:00.850195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":403,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.788540+0000","last_change":"2023-06-10T21:39:50.259203+0000","last_active":"2023-06-10T21:42:00.788540+0000","last_peered":"2023-06-10T21:42:00.788540+0000","last_clean":"2023-06-10T21:42:00.788540+0000","last_became_active":"2023-06-10T21:39:50.258873+0000","last_became_peered":"2023-06-10T21:39:50.258873+0000","last_unstale":"2023-06-10T21:42:00.788540+0000","last_undegraded":"2023-06-10T21:42:00.788540+0000","last_fullsized":"2023-06-10T21:42:00.788540+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:56:41.490096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"381'3667","reported_seq":3683,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.111996+0000","last_change":"2023-06-10T21:41:28.184243+0000","last_active":"2023-06-10T21:42:14.111996+0000","last_peered":"2023-06-10T21:42:14.111996+0000","last_clean":"2023-06-10T21:42:14.111996+0000","last_became_active":"2023-06-10T21:41:28.183870+0000","last_became_peered":"2023-06-10T21:41:28.183870+0000","last_unstale":"2023-06-10T21:42:14.111996+0000","last_undegraded":"2023-06-10T21:42:14.111996+0000","last_fullsized":"2023-06-10T21:42:14.111996+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3667,"log_dups_size":0,"ondisk_log_size":3667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3667,"num_write_kb":234688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":380,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785767+0000","last_change":"2023-06-10T21:39:50.432352+0000","last_active":"2023-06-10T21:42:00.785767+0000","last_peered":"2023-06-10T21:42:00.785767+0000","last_clean":"2023-06-10T21:42:00.785767+0000","last_became_active":"2023-06-10T21:39:50.429920+0000","last_became_peered":"2023-06-10T21:39:50.429920+0000","last_unstale":"2023-06-10T21:42:00.785767+0000","last_undegraded":"2023-06-10T21:42:00.785767+0000","last_fullsized":"2023-06-10T21:42:00.785767+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:14.733503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784939+0000","last_change":"2023-06-10T21:40:30.351796+0000","last_active":"2023-06-10T21:42:00.784939+0000","last_peered":"2023-06-10T21:42:00.784939+0000","last_clean":"2023-06-10T21:42:00.784939+0000","last_became_active":"2023-06-10T21:40:30.350450+0000","last_became_peered":"2023-06-10T21:40:30.350450+0000","last_unstale":"2023-06-10T21:42:00.784939+0000","last_undegraded":"2023-06-10T21:42:00.784939+0000","last_fullsized":"2023-06-10T21:42:00.784939+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:38.306536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"381'3666","reported_seq":3682,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.047641+0000","last_change":"2023-06-10T21:41:28.177644+0000","last_active":"2023-06-10T21:42:14.047641+0000","last_peered":"2023-06-10T21:42:14.047641+0000","last_clean":"2023-06-10T21:42:14.047641+0000","last_became_active":"2023-06-10T21:41:28.177318+0000","last_became_peered":"2023-06-10T21:41:28.177318+0000","last_unstale":"2023-06-10T21:42:14.047641+0000","last_undegraded":"2023-06-10T21:42:14.047641+0000","last_fullsized":"2023-06-10T21:42:14.047641+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3666,"log_dups_size":0,"ondisk_log_size":3666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240254976,"num_objects":3666,"num_object_clones":0,"num_object_copies":10998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3666,"num_write_kb":234624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783784+0000","last_change":"2023-06-10T21:40:30.279010+0000","last_active":"2023-06-10T21:42:00.783784+0000","last_peered":"2023-06-10T21:42:00.783784+0000","last_clean":"2023-06-10T21:42:00.783784+0000","last_became_active":"2023-06-10T21:40:30.277943+0000","last_became_peered":"2023-06-10T21:40:30.277943+0000","last_unstale":"2023-06-10T21:42:00.783784+0000","last_undegraded":"2023-06-10T21:42:00.783784+0000","last_fullsized":"2023-06-10T21:42:00.783784+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:38:15.175343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":312,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784734+0000","last_change":"2023-06-10T21:39:50.233694+0000","last_active":"2023-06-10T21:42:00.784734+0000","last_peered":"2023-06-10T21:42:00.784734+0000","last_clean":"2023-06-10T21:42:00.784734+0000","last_became_active":"2023-06-10T21:39:50.233429+0000","last_became_peered":"2023-06-10T21:39:50.233429+0000","last_unstale":"2023-06-10T21:42:00.784734+0000","last_undegraded":"2023-06-10T21:42:00.784734+0000","last_fullsized":"2023-06-10T21:42:00.784734+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:15:42.432299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"381'3574","reported_seq":3590,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.069660+0000","last_change":"2023-06-10T21:41:28.176942+0000","last_active":"2023-06-10T21:42:14.069660+0000","last_peered":"2023-06-10T21:42:14.069660+0000","last_clean":"2023-06-10T21:42:14.069660+0000","last_became_active":"2023-06-10T21:41:28.176613+0000","last_became_peered":"2023-06-10T21:41:28.176613+0000","last_unstale":"2023-06-10T21:42:14.069660+0000","last_undegraded":"2023-06-10T21:42:14.069660+0000","last_fullsized":"2023-06-10T21:42:14.069660+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3574,"log_dups_size":0,"ondisk_log_size":3574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234225664,"num_objects":3574,"num_object_clones":0,"num_object_copies":10722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3574,"num_write_kb":228736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":164,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786011+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:42:00.786011+0000","last_peered":"2023-06-10T21:42:00.786011+0000","last_clean":"2023-06-10T21:42:00.786011+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:42:00.786011+0000","last_undegraded":"2023-06-10T21:42:00.786011+0000","last_fullsized":"2023-06-10T21:42:00.786011+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":64,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784607+0000","last_change":"2023-06-10T21:40:30.281222+0000","last_active":"2023-06-10T21:42:00.784607+0000","last_peered":"2023-06-10T21:42:00.784607+0000","last_clean":"2023-06-10T21:42:00.784607+0000","last_became_active":"2023-06-10T21:40:30.280846+0000","last_became_peered":"2023-06-10T21:40:30.280846+0000","last_unstale":"2023-06-10T21:42:00.784607+0000","last_undegraded":"2023-06-10T21:42:00.784607+0000","last_fullsized":"2023-06-10T21:42:00.784607+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784734+0000","last_change":"2023-06-10T21:39:50.466302+0000","last_active":"2023-06-10T21:42:00.784734+0000","last_peered":"2023-06-10T21:42:00.784734+0000","last_clean":"2023-06-10T21:42:00.784734+0000","last_became_active":"2023-06-10T21:39:50.465409+0000","last_became_peered":"2023-06-10T21:39:50.465409+0000","last_unstale":"2023-06-10T21:42:00.784734+0000","last_undegraded":"2023-06-10T21:42:00.784734+0000","last_fullsized":"2023-06-10T21:42:00.784734+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"381'3646","reported_seq":3662,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.108474+0000","last_change":"2023-06-10T21:41:28.182320+0000","last_active":"2023-06-10T21:42:14.108474+0000","last_peered":"2023-06-10T21:42:14.108474+0000","last_clean":"2023-06-10T21:42:14.108474+0000","last_became_active":"2023-06-10T21:41:28.181951+0000","last_became_peered":"2023-06-10T21:41:28.181951+0000","last_unstale":"2023-06-10T21:42:14.108474+0000","last_undegraded":"2023-06-10T21:42:14.108474+0000","last_fullsized":"2023-06-10T21:42:14.108474+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3646,"log_dups_size":0,"ondisk_log_size":3646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238944256,"num_objects":3646,"num_object_clones":0,"num_object_copies":10938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3646,"num_write_kb":233344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":83,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784585+0000","last_change":"2023-06-10T21:40:30.236317+0000","last_active":"2023-06-10T21:42:00.784585+0000","last_peered":"2023-06-10T21:42:00.784585+0000","last_clean":"2023-06-10T21:42:00.784585+0000","last_became_active":"2023-06-10T21:40:30.235998+0000","last_became_peered":"2023-06-10T21:40:30.235998+0000","last_unstale":"2023-06-10T21:42:00.784585+0000","last_undegraded":"2023-06-10T21:42:00.784585+0000","last_fullsized":"2023-06-10T21:42:00.784585+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":361,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785204+0000","last_change":"2023-06-10T21:40:30.281401+0000","last_active":"2023-06-10T21:42:00.785204+0000","last_peered":"2023-06-10T21:42:00.785204+0000","last_clean":"2023-06-10T21:42:00.785204+0000","last_became_active":"2023-06-10T21:40:30.280966+0000","last_became_peered":"2023-06-10T21:40:30.280966+0000","last_unstale":"2023-06-10T21:42:00.785204+0000","last_undegraded":"2023-06-10T21:42:00.785204+0000","last_fullsized":"2023-06-10T21:42:00.785204+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":244,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.788788+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:42:00.788788+0000","last_peered":"2023-06-10T21:42:00.788788+0000","last_clean":"2023-06-10T21:42:00.788788+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:42:00.788788+0000","last_undegraded":"2023-06-10T21:42:00.788788+0000","last_fullsized":"2023-06-10T21:42:00.788788+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"381'3678","reported_seq":3694,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.126704+0000","last_change":"2023-06-10T21:41:28.182336+0000","last_active":"2023-06-10T21:42:14.126704+0000","last_peered":"2023-06-10T21:42:14.126704+0000","last_clean":"2023-06-10T21:42:14.126704+0000","last_became_active":"2023-06-10T21:41:28.181961+0000","last_became_peered":"2023-06-10T21:41:28.181961+0000","last_unstale":"2023-06-10T21:42:14.126704+0000","last_undegraded":"2023-06-10T21:42:14.126704+0000","last_fullsized":"2023-06-10T21:42:14.126704+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3678,"log_dups_size":0,"ondisk_log_size":3678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241041408,"num_objects":3678,"num_object_clones":0,"num_object_copies":11034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3678,"num_write_kb":235392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":264,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785704+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:42:00.785704+0000","last_peered":"2023-06-10T21:42:00.785704+0000","last_clean":"2023-06-10T21:42:00.785704+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:42:00.785704+0000","last_undegraded":"2023-06-10T21:42:00.785704+0000","last_fullsized":"2023-06-10T21:42:00.785704+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":254,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786121+0000","last_change":"2023-06-10T21:40:30.229244+0000","last_active":"2023-06-10T21:42:00.786121+0000","last_peered":"2023-06-10T21:42:00.786121+0000","last_clean":"2023-06-10T21:42:00.786121+0000","last_became_active":"2023-06-10T21:40:30.228964+0000","last_became_peered":"2023-06-10T21:40:30.228964+0000","last_unstale":"2023-06-10T21:42:00.786121+0000","last_undegraded":"2023-06-10T21:42:00.786121+0000","last_fullsized":"2023-06-10T21:42:00.786121+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:41.629358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785293+0000","last_change":"2023-06-10T21:40:30.236712+0000","last_active":"2023-06-10T21:42:00.785293+0000","last_peered":"2023-06-10T21:42:00.785293+0000","last_clean":"2023-06-10T21:42:00.785293+0000","last_became_active":"2023-06-10T21:40:30.236404+0000","last_became_peered":"2023-06-10T21:40:30.236404+0000","last_unstale":"2023-06-10T21:42:00.785293+0000","last_undegraded":"2023-06-10T21:42:00.785293+0000","last_fullsized":"2023-06-10T21:42:00.785293+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"381'3685","reported_seq":3700,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.048273+0000","last_change":"2023-06-10T21:41:28.181267+0000","last_active":"2023-06-10T21:42:14.048273+0000","last_peered":"2023-06-10T21:42:14.048273+0000","last_clean":"2023-06-10T21:42:14.048273+0000","last_became_active":"2023-06-10T21:41:28.180932+0000","last_became_peered":"2023-06-10T21:41:28.180932+0000","last_unstale":"2023-06-10T21:42:14.048273+0000","last_undegraded":"2023-06-10T21:42:14.048273+0000","last_fullsized":"2023-06-10T21:42:14.048273+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3685,"log_dups_size":0,"ondisk_log_size":3685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241500160,"num_objects":3685,"num_object_clones":0,"num_object_copies":11055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3685,"num_write_kb":235840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":55,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784887+0000","last_change":"2023-06-10T21:38:37.269392+0000","last_active":"2023-06-10T21:42:00.784887+0000","last_peered":"2023-06-10T21:42:00.784887+0000","last_clean":"2023-06-10T21:42:00.784887+0000","last_became_active":"2023-06-10T21:38:37.269025+0000","last_became_peered":"2023-06-10T21:38:37.269025+0000","last_unstale":"2023-06-10T21:42:00.784887+0000","last_undegraded":"2023-06-10T21:42:00.784887+0000","last_fullsized":"2023-06-10T21:42:00.784887+0000","mapping_epoch":335,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":581,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784124+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:42:00.784124+0000","last_peered":"2023-06-10T21:42:00.784124+0000","last_clean":"2023-06-10T21:42:00.784124+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:42:00.784124+0000","last_undegraded":"2023-06-10T21:42:00.784124+0000","last_fullsized":"2023-06-10T21:42:00.784124+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783538+0000","last_change":"2023-06-10T21:40:30.260288+0000","last_active":"2023-06-10T21:42:00.783538+0000","last_peered":"2023-06-10T21:42:00.783538+0000","last_clean":"2023-06-10T21:42:00.783538+0000","last_became_active":"2023-06-10T21:40:30.259958+0000","last_became_peered":"2023-06-10T21:40:30.259958+0000","last_unstale":"2023-06-10T21:42:00.783538+0000","last_undegraded":"2023-06-10T21:42:00.783538+0000","last_fullsized":"2023-06-10T21:42:00.783538+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:01:41.077016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785739+0000","last_change":"2023-06-10T21:40:30.286026+0000","last_active":"2023-06-10T21:42:00.785739+0000","last_peered":"2023-06-10T21:42:00.785739+0000","last_clean":"2023-06-10T21:42:00.785739+0000","last_became_active":"2023-06-10T21:40:30.285670+0000","last_became_peered":"2023-06-10T21:40:30.285670+0000","last_unstale":"2023-06-10T21:42:00.785739+0000","last_undegraded":"2023-06-10T21:42:00.785739+0000","last_fullsized":"2023-06-10T21:42:00.785739+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"381'3649","reported_seq":3665,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.116913+0000","last_change":"2023-06-10T21:41:28.183802+0000","last_active":"2023-06-10T21:42:14.116913+0000","last_peered":"2023-06-10T21:42:14.116913+0000","last_clean":"2023-06-10T21:42:14.116913+0000","last_became_active":"2023-06-10T21:41:28.183511+0000","last_became_peered":"2023-06-10T21:41:28.183511+0000","last_unstale":"2023-06-10T21:42:14.116913+0000","last_undegraded":"2023-06-10T21:42:14.116913+0000","last_fullsized":"2023-06-10T21:42:14.116913+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3649,"log_dups_size":0,"ondisk_log_size":3649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239140864,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3649,"num_write_kb":233536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":418,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784519+0000","last_change":"2023-06-10T21:39:50.237017+0000","last_active":"2023-06-10T21:42:00.784519+0000","last_peered":"2023-06-10T21:42:00.784519+0000","last_clean":"2023-06-10T21:42:00.784519+0000","last_became_active":"2023-06-10T21:39:50.232270+0000","last_became_peered":"2023-06-10T21:39:50.232270+0000","last_unstale":"2023-06-10T21:42:00.784519+0000","last_undegraded":"2023-06-10T21:42:00.784519+0000","last_fullsized":"2023-06-10T21:42:00.784519+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784427+0000","last_change":"2023-06-10T21:40:30.316582+0000","last_active":"2023-06-10T21:42:00.784427+0000","last_peered":"2023-06-10T21:42:00.784427+0000","last_clean":"2023-06-10T21:42:00.784427+0000","last_became_active":"2023-06-10T21:40:30.315657+0000","last_became_peered":"2023-06-10T21:40:30.315657+0000","last_unstale":"2023-06-10T21:42:00.784427+0000","last_undegraded":"2023-06-10T21:42:00.784427+0000","last_fullsized":"2023-06-10T21:42:00.784427+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:08:04.756800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":116,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.789563+0000","last_change":"2023-06-10T21:39:50.559093+0000","last_active":"2023-06-10T21:42:00.789563+0000","last_peered":"2023-06-10T21:42:00.789563+0000","last_clean":"2023-06-10T21:42:00.789563+0000","last_became_active":"2023-06-10T21:39:50.558289+0000","last_became_peered":"2023-06-10T21:39:50.558289+0000","last_unstale":"2023-06-10T21:42:00.789563+0000","last_undegraded":"2023-06-10T21:42:00.789563+0000","last_fullsized":"2023-06-10T21:42:00.789563+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:37.230755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"381'3455","reported_seq":3470,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:10.694333+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:42:10.694333+0000","last_peered":"2023-06-10T21:42:10.694333+0000","last_clean":"2023-06-10T21:42:10.694333+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:42:10.694333+0000","last_undegraded":"2023-06-10T21:42:10.694333+0000","last_fullsized":"2023-06-10T21:42:10.694333+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3455,"log_dups_size":0,"ondisk_log_size":3455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226426880,"num_objects":3455,"num_object_clones":0,"num_object_copies":10365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3455,"num_write_kb":221120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":412,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785653+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:42:00.785653+0000","last_peered":"2023-06-10T21:42:00.785653+0000","last_clean":"2023-06-10T21:42:00.785653+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:42:00.785653+0000","last_undegraded":"2023-06-10T21:42:00.785653+0000","last_fullsized":"2023-06-10T21:42:00.785653+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":244,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.788878+0000","last_change":"2023-06-10T21:39:50.296322+0000","last_active":"2023-06-10T21:42:00.788878+0000","last_peered":"2023-06-10T21:42:00.788878+0000","last_clean":"2023-06-10T21:42:00.788878+0000","last_became_active":"2023-06-10T21:39:50.295201+0000","last_became_peered":"2023-06-10T21:39:50.295201+0000","last_unstale":"2023-06-10T21:42:00.788878+0000","last_undegraded":"2023-06-10T21:42:00.788878+0000","last_fullsized":"2023-06-10T21:42:00.788878+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:28.403434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":210,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785230+0000","last_change":"2023-06-10T21:39:50.271175+0000","last_active":"2023-06-10T21:42:00.785230+0000","last_peered":"2023-06-10T21:42:00.785230+0000","last_clean":"2023-06-10T21:42:00.785230+0000","last_became_active":"2023-06-10T21:39:50.270302+0000","last_became_peered":"2023-06-10T21:39:50.270302+0000","last_unstale":"2023-06-10T21:42:00.785230+0000","last_undegraded":"2023-06-10T21:42:00.785230+0000","last_fullsized":"2023-06-10T21:42:00.785230+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"381'3775","reported_seq":3791,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:14.064381+0000","last_change":"2023-06-10T21:41:28.179779+0000","last_active":"2023-06-10T21:42:14.064381+0000","last_peered":"2023-06-10T21:42:14.064381+0000","last_clean":"2023-06-10T21:42:14.064381+0000","last_became_active":"2023-06-10T21:41:28.179455+0000","last_became_peered":"2023-06-10T21:41:28.179455+0000","last_unstale":"2023-06-10T21:42:14.064381+0000","last_undegraded":"2023-06-10T21:42:14.064381+0000","last_fullsized":"2023-06-10T21:42:14.064381+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3775,"log_dups_size":0,"ondisk_log_size":3775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247398400,"num_objects":3775,"num_object_clones":0,"num_object_copies":11325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3775,"num_write_kb":241600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":375,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784588+0000","last_change":"2023-06-10T21:39:30.266087+0000","last_active":"2023-06-10T21:42:00.784588+0000","last_peered":"2023-06-10T21:42:00.784588+0000","last_clean":"2023-06-10T21:42:00.784588+0000","last_became_active":"2023-06-10T21:39:30.231858+0000","last_became_peered":"2023-06-10T21:39:30.231858+0000","last_unstale":"2023-06-10T21:42:00.784588+0000","last_undegraded":"2023-06-10T21:42:00.784588+0000","last_fullsized":"2023-06-10T21:42:00.784588+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:08:09.982313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":186,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785259+0000","last_change":"2023-06-10T21:39:50.227162+0000","last_active":"2023-06-10T21:42:00.785259+0000","last_peered":"2023-06-10T21:42:00.785259+0000","last_clean":"2023-06-10T21:42:00.785259+0000","last_became_active":"2023-06-10T21:39:50.226891+0000","last_became_peered":"2023-06-10T21:39:50.226891+0000","last_unstale":"2023-06-10T21:42:00.785259+0000","last_undegraded":"2023-06-10T21:42:00.785259+0000","last_fullsized":"2023-06-10T21:42:00.785259+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:48:59.680224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783699+0000","last_change":"2023-06-10T21:40:30.279185+0000","last_active":"2023-06-10T21:42:00.783699+0000","last_peered":"2023-06-10T21:42:00.783699+0000","last_clean":"2023-06-10T21:42:00.783699+0000","last_became_active":"2023-06-10T21:40:30.278330+0000","last_became_peered":"2023-06-10T21:40:30.278330+0000","last_unstale":"2023-06-10T21:42:00.783699+0000","last_undegraded":"2023-06-10T21:42:00.783699+0000","last_fullsized":"2023-06-10T21:42:00.783699+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:15.964124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"381'3507","reported_seq":3524,"reported_epoch":381,"state":"active+clean","last_fresh":"2023-06-10T21:42:10.694250+0000","last_change":"2023-06-10T21:41:28.176991+0000","last_active":"2023-06-10T21:42:10.694250+0000","last_peered":"2023-06-10T21:42:10.694250+0000","last_clean":"2023-06-10T21:42:10.694250+0000","last_became_active":"2023-06-10T21:41:28.176650+0000","last_became_peered":"2023-06-10T21:41:28.176650+0000","last_unstale":"2023-06-10T21:42:10.694250+0000","last_undegraded":"2023-06-10T21:42:10.694250+0000","last_fullsized":"2023-06-10T21:42:10.694250+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":3507,"log_dups_size":0,"ondisk_log_size":3507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:48:14.567179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229834752,"num_objects":3507,"num_object_clones":0,"num_object_copies":10521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3507,"num_write_kb":224448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":129,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785886+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:42:00.785886+0000","last_peered":"2023-06-10T21:42:00.785886+0000","last_clean":"2023-06-10T21:42:00.785886+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:42:00.785886+0000","last_undegraded":"2023-06-10T21:42:00.785886+0000","last_fullsized":"2023-06-10T21:42:00.785886+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":228,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785360+0000","last_change":"2023-06-10T21:40:30.580293+0000","last_active":"2023-06-10T21:42:00.785360+0000","last_peered":"2023-06-10T21:42:00.785360+0000","last_clean":"2023-06-10T21:42:00.785360+0000","last_became_active":"2023-06-10T21:40:30.391049+0000","last_became_peered":"2023-06-10T21:40:30.391049+0000","last_unstale":"2023-06-10T21:42:00.785360+0000","last_undegraded":"2023-06-10T21:42:00.785360+0000","last_fullsized":"2023-06-10T21:42:00.785360+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784062+0000","last_change":"2023-06-10T21:39:30.208137+0000","last_active":"2023-06-10T21:42:00.784062+0000","last_peered":"2023-06-10T21:42:00.784062+0000","last_clean":"2023-06-10T21:42:00.784062+0000","last_became_active":"2023-06-10T21:39:30.207848+0000","last_became_peered":"2023-06-10T21:39:30.207848+0000","last_unstale":"2023-06-10T21:42:00.784062+0000","last_undegraded":"2023-06-10T21:42:00.784062+0000","last_fullsized":"2023-06-10T21:42:00.784062+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":80,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786402+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:42:00.786402+0000","last_peered":"2023-06-10T21:42:00.786402+0000","last_clean":"2023-06-10T21:42:00.786402+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:42:00.786402+0000","last_undegraded":"2023-06-10T21:42:00.786402+0000","last_fullsized":"2023-06-10T21:42:00.786402+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":261,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.788960+0000","last_change":"2023-06-10T21:40:30.227732+0000","last_active":"2023-06-10T21:42:00.788960+0000","last_peered":"2023-06-10T21:42:00.788960+0000","last_clean":"2023-06-10T21:42:00.788960+0000","last_became_active":"2023-06-10T21:40:30.227380+0000","last_became_peered":"2023-06-10T21:40:30.227380+0000","last_unstale":"2023-06-10T21:42:00.788960+0000","last_undegraded":"2023-06-10T21:42:00.788960+0000","last_fullsized":"2023-06-10T21:42:00.788960+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:47:33.673485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":68,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784942+0000","last_change":"2023-06-10T21:39:50.304413+0000","last_active":"2023-06-10T21:42:00.784942+0000","last_peered":"2023-06-10T21:42:00.784942+0000","last_clean":"2023-06-10T21:42:00.784942+0000","last_became_active":"2023-06-10T21:39:50.303414+0000","last_became_peered":"2023-06-10T21:39:50.303414+0000","last_unstale":"2023-06-10T21:42:00.784942+0000","last_undegraded":"2023-06-10T21:42:00.784942+0000","last_fullsized":"2023-06-10T21:42:00.784942+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":284,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.786738+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:42:00.786738+0000","last_peered":"2023-06-10T21:42:00.786738+0000","last_clean":"2023-06-10T21:42:00.786738+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:42:00.786738+0000","last_undegraded":"2023-06-10T21:42:00.786738+0000","last_fullsized":"2023-06-10T21:42:00.786738+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":85,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785077+0000","last_change":"2023-06-10T21:40:30.565123+0000","last_active":"2023-06-10T21:42:00.785077+0000","last_peered":"2023-06-10T21:42:00.785077+0000","last_clean":"2023-06-10T21:42:00.785077+0000","last_became_active":"2023-06-10T21:40:30.280290+0000","last_became_peered":"2023-06-10T21:40:30.280290+0000","last_unstale":"2023-06-10T21:42:00.785077+0000","last_undegraded":"2023-06-10T21:42:00.785077+0000","last_fullsized":"2023-06-10T21:42:00.785077+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":340,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783821+0000","last_change":"2023-06-10T21:39:30.245702+0000","last_active":"2023-06-10T21:42:00.783821+0000","last_peered":"2023-06-10T21:42:00.783821+0000","last_clean":"2023-06-10T21:42:00.783821+0000","last_became_active":"2023-06-10T21:39:30.245426+0000","last_became_peered":"2023-06-10T21:39:30.245426+0000","last_unstale":"2023-06-10T21:42:00.783821+0000","last_undegraded":"2023-06-10T21:42:00.783821+0000","last_fullsized":"2023-06-10T21:42:00.783821+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:52:27.316156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.785111+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:42:00.785111+0000","last_peered":"2023-06-10T21:42:00.785111+0000","last_clean":"2023-06-10T21:42:00.785111+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:42:00.785111+0000","last_undegraded":"2023-06-10T21:42:00.785111+0000","last_fullsized":"2023-06-10T21:42:00.785111+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784576+0000","last_change":"2023-06-10T21:40:30.253755+0000","last_active":"2023-06-10T21:42:00.784576+0000","last_peered":"2023-06-10T21:42:00.784576+0000","last_clean":"2023-06-10T21:42:00.784576+0000","last_became_active":"2023-06-10T21:40:30.253530+0000","last_became_peered":"2023-06-10T21:40:30.253530+0000","last_unstale":"2023-06-10T21:42:00.784576+0000","last_undegraded":"2023-06-10T21:42:00.784576+0000","last_fullsized":"2023-06-10T21:42:00.784576+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:14.839319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":321,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.784744+0000","last_change":"2023-06-10T21:40:30.574471+0000","last_active":"2023-06-10T21:42:00.784744+0000","last_peered":"2023-06-10T21:42:00.784744+0000","last_clean":"2023-06-10T21:42:00.784744+0000","last_became_active":"2023-06-10T21:40:30.281351+0000","last_became_peered":"2023-06-10T21:40:30.281351+0000","last_unstale":"2023-06-10T21:42:00.784744+0000","last_undegraded":"2023-06-10T21:42:00.784744+0000","last_fullsized":"2023-06-10T21:42:00.784744+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":33,"reported_epoch":380,"state":"active+clean","last_fresh":"2023-06-10T21:42:00.783884+0000","last_change":"2023-06-10T21:40:30.245261+0000","last_active":"2023-06-10T21:42:00.783884+0000","last_peered":"2023-06-10T21:42:00.783884+0000","last_clean":"2023-06-10T21:42:00.783884+0000","last_became_active":"2023-06-10T21:40:30.244695+0000","last_became_peered":"2023-06-10T21:40:30.244695+0000","last_unstale":"2023-06-10T21:42:00.783884+0000","last_undegraded":"2023-06-10T21:42:00.783884+0000","last_fullsized":"2023-06-10T21:42:00.783884+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:54.784460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3800498176,"num_objects":57991,"num_object_clones":0,"num_object_copies":173973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":57991,"num_write_kb":3711424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11350376448,"data_stored":11350376448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57991,"ondisk_log_size":57991,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":18486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30523,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430742,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3445908,"kb_used_data":2591628,"kb_used_omap":0,"kb_used_meta":854272,"kb_avail":90299244,"statfs":{"total":95995035648,"available":92466425856,"internally_reserved":0,"allocated":2653827072,"data_stored":2653129275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":874774528},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":2.0470000000000002,"15min":4.282},"min":{"1min":0.26100000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.1110000000000002,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.62},{"interface":"front","average":{"1min":1.093,"5min":2.4199999999999999,"15min":4.5679999999999996},"min":{"1min":0.28899999999999998,"5min":0.25,"15min":0.223},"max":{"1min":4.8380000000000001,"5min":56.628,"15min":185.58000000000001},"last":0.79000000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":2.9660000000000002,"15min":2.976},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.21299999999999999},"max":{"1min":38.078000000000003,"5min":60.863,"15min":93.093999999999994},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.476,"5min":3.2919999999999998,"15min":3.1000000000000001},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":38.341000000000001,"5min":61.148000000000003,"15min":93.119},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:41:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.867,"15min":4.9009999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":24.106000000000002,"5min":137.64099999999999,"15min":172.33600000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":2.4089999999999998,"5min":4.54,"15min":4.8369999999999997},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.29499999999999998},"max":{"1min":24.132999999999999,"5min":136.96600000000001,"15min":172.185},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:41:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":4.3460000000000001,"15min":4.2649999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":24.375,"5min":102.25700000000001,"15min":147.00700000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":2.347,"5min":4.1070000000000002,"15min":3.923},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":26.460000000000001,"5min":103.401,"15min":146.93899999999999},"last":0.72399999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.57499999999999996,"15min":0.53000000000000003},"min":{"1min":0.38500000000000001,"5min":0.253,"15min":0.249},"max":{"1min":2.6000000000000001,"5min":5.859,"15min":5.859},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.874,"5min":0.90100000000000002,"15min":0.68600000000000005},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.245},"max":{"1min":2.863,"5min":22.77,"15min":22.77},"last":3.0049999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:42:08 2023","interfaces":[{"interface":"back","average":{"1min":11.141999999999999,"5min":2.6240000000000001,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.252,"15min":0.252},"max":{"1min":174.13200000000001,"5min":174.13200000000001,"15min":174.13200000000001},"last":2.3799999999999999},{"interface":"front","average":{"1min":11.669,"5min":2.806,"15min":1.302},"min":{"1min":0.38700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":177.81200000000001,"5min":177.81200000000001,"15min":177.81200000000001},"last":0.745}]},{"osd":6,"last update":"Sat Jun 10 21:42:08 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.74099999999999999,"15min":0.73999999999999999},"min":{"1min":0.40699999999999997,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":2.9510000000000001,"5min":10.890000000000001,"15min":13.073},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.87,"5min":0.66500000000000004,"15min":0.57899999999999996},"min":{"1min":0.38,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.8959999999999999,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":2.29}]}]},{"osd":6,"up_from":0,"seq":180388626970,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566412,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020389,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075800,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3414480,"kb_used_data":2623816,"kb_used_omap":7,"kb_used_meta":790584,"kb_avail":90330672,"statfs":{"total":95995035648,"available":92498608128,"internally_reserved":0,"allocated":2686787584,"data_stored":2685513356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":809558097},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:55 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":6.0129999999999999,"15min":4.8410000000000002},"min":{"1min":0.23999999999999999,"5min":0.19,"15min":0.19},"max":{"1min":5.6500000000000004,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":5.4619999999999997,"15min":4.8570000000000002},"min":{"1min":0.29099999999999998,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6280000000000001,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":5.9130000000000003,"5min":8.6440000000000001,"15min":5.3680000000000003},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.19},"max":{"1min":85.709999999999994,"5min":171.286,"15min":171.286},"last":1.1879999999999999},{"interface":"front","average":{"1min":6.3300000000000001,"5min":4.7469999999999999,"15min":3.827},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":85.501999999999995,"5min":85.501999999999995,"15min":147.81299999999999},"last":1.4299999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:41:20 2023","interfaces":[{"interface":"back","average":{"1min":5.7830000000000004,"5min":5.694,"15min":5.5270000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":70.480999999999995,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":5.4980000000000002,"5min":5.3550000000000004,"15min":5.7530000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":71.144999999999996,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.91700000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":3.4969999999999999,"15min":3.5449999999999999},"min":{"1min":0.36699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":2.093,"5min":171.48699999999999,"15min":224.00999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":5.2060000000000004,"15min":4.4390000000000001},"min":{"1min":0.35599999999999998,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":3.423,"5min":171.18100000000001,"15min":223.28399999999999},"last":1.3620000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.613,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.7759999999999998,"5min":171.303,"15min":171.303},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.5430000000000001,"15min":2.3279999999999998},"min":{"1min":0.38,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1280000000000001,"5min":171.203,"15min":171.203},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":3.585,"15min":2.4420000000000002},"min":{"1min":0.48599999999999999,"5min":0.307,"15min":0.307},"max":{"1min":2.02,"5min":171.441,"15min":171.441},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.86099999999999999,"5min":5.2610000000000001,"15min":3.3090000000000002},"min":{"1min":0.39600000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":3.4630000000000001,"5min":171.24600000000001,"15min":171.24600000000001},"last":4.1669999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":3.7360000000000002,"15min":4.4139999999999997},"min":{"1min":0.47399999999999998,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":5.7400000000000002,"5min":171.26499999999999,"15min":219.131},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.85199999999999998,"5min":3.5670000000000002,"15min":4.4219999999999997},"min":{"1min":0.41599999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.0489999999999999,"5min":171.52000000000001,"15min":219.19},"last":0.873}]}]},{"osd":3,"up_from":250,"seq":1073741824160,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3221196,"kb_used_data":2673540,"kb_used_omap":7,"kb_used_meta":547640,"kb_avail":90523956,"statfs":{"total":95995035648,"available":92696530944,"internally_reserved":0,"allocated":2737704960,"data_stored":2736314112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":560783379},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":3.3039999999999998,"15min":4.234},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":30.744,"5min":30.744,"15min":105.09},"last":0.68300000000000005},{"interface":"front","average":{"1min":3.7559999999999998,"5min":3.3029999999999999,"15min":4.1200000000000001},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":30.805,"5min":30.805,"15min":99.852999999999994},"last":4.8639999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":2.9830000000000001,"15min":3.819},"min":{"1min":0.23599999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":7.6509999999999998,"5min":21.704999999999998,"15min":94.766000000000005},"last":1.8600000000000001},{"interface":"front","average":{"1min":1.325,"5min":3.5960000000000001,"15min":3.7690000000000001},"min":{"1min":0.28899999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.9649999999999999,"5min":41.18,"15min":94.691000000000003},"last":0.61099999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":4.8869999999999996,"15min":6.4669999999999996},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":10.206,"5min":231.07900000000001,"15min":231.07900000000001},"last":4.9059999999999997},{"interface":"front","average":{"1min":1.8879999999999999,"5min":6.1260000000000003,"15min":7.9299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":7.6909999999999998,"5min":231.58600000000001,"15min":231.58600000000001},"last":4.3090000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.379,"15min":1.407},"min":{"1min":0.32100000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.0820000000000007,"5min":10.368,"15min":13.417},"last":4.8310000000000004},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.2309999999999999,"15min":2.4990000000000001},"min":{"1min":0.38400000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":7.6150000000000002,"5min":19.244,"15min":19.728000000000002},"last":4.2679999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.847,"15min":1.5489999999999999},"min":{"1min":0.42899999999999999,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":3.8559999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":1.919},{"interface":"front","average":{"1min":1.409,"5min":1.4830000000000001,"15min":1.5049999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":9.1210000000000004,"5min":10.414999999999999,"15min":13.470000000000001},"last":4.6760000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.431,"5min":2.254,"15min":2.52},"min":{"1min":0.46800000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.5270000000000001,"5min":19.164999999999999,"15min":19.805},"last":4.8849999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.839,"15min":1.5800000000000001},"min":{"1min":0.49399999999999999,"5min":0.40000000000000002,"15min":0.39100000000000001},"max":{"1min":3.8180000000000001,"5min":17.327000000000002,"15min":17.327000000000002},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.3330000000000002,"15min":2.778},"min":{"1min":0.50800000000000001,"5min":0.379,"15min":0.33900000000000002},"max":{"1min":7.5789999999999997,"5min":19.193000000000001,"15min":79.984999999999999},"last":5.1280000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.9350000000000001,"15min":1.8320000000000001},"min":{"1min":0.45500000000000002,"5min":0.378,"15min":0.36699999999999999},"max":{"1min":3.6960000000000002,"5min":17.251000000000001,"15min":80.272999999999996},"last":1.538}]}]},{"osd":4,"up_from":0,"seq":128849019426,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":36,"seq":154618823198,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3978700,"kb_used_data":3286148,"kb_used_omap":0,"kb_used_meta":692544,"kb_avail":89766452,"statfs":{"total":95995035648,"available":91920846848,"internally_reserved":0,"allocated":3365015552,"data_stored":3364325552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":709165056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.7090000000000001,"15min":3.137},"min":{"1min":0.27600000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":2.5190000000000001,"5min":18.562000000000001,"15min":134.71899999999999},"last":0.307},{"interface":"front","average":{"1min":1.22,"5min":2.2069999999999999,"15min":3.407},"min":{"1min":0.313,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.9569999999999999,"5min":27.256,"15min":135.386},"last":2.2029999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":2.4750000000000001,"15min":2.4060000000000001},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.4830000000000001,"5min":60.399999999999999,"15min":71.349000000000004},"last":2.2610000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":2.363,"15min":2.3439999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":2.3929999999999998,"5min":59.780999999999999,"15min":71.331999999999994},"last":2.2309999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":5.8949999999999996,"15min":6.0910000000000002},"min":{"1min":0.373,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.3330000000000002,"5min":324.21600000000001,"15min":324.21600000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.177,"5min":5.5650000000000004,"15min":5.8639999999999999},"min":{"1min":0.35599999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.1779999999999999,"5min":324.16300000000001,"15min":324.16300000000001},"last":4.9269999999999996}]},{"osd":3,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":4.5609999999999999,"15min":4.3949999999999996},"min":{"1min":0.39900000000000002,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.2510000000000003,"5min":90.725999999999999,"15min":128.089},"last":2.4390000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.9390000000000001,"15min":4.3959999999999999},"min":{"1min":0.39000000000000001,"5min":0.33400000000000002,"15min":0.317},"max":{"1min":5.907,"5min":90.180000000000007,"15min":128.24299999999999},"last":0.83999999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.51000000000000001,"15min":1.135},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":2.7000000000000002,"5min":5.3460000000000001,"15min":165.73699999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.59999999999999998,"15min":1.2330000000000001},"min":{"1min":0.34000000000000002,"5min":0.27700000000000002,"15min":0.25},"max":{"1min":3.2829999999999999,"5min":4.0940000000000003,"15min":165.77500000000001},"last":0.48199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:41:58 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.64700000000000002,"15min":0.58299999999999996},"min":{"1min":0.36399999999999999,"5min":0.29899999999999999,"15min":0.22},"max":{"1min":2.7280000000000002,"5min":5.3739999999999997,"15min":6.125},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.72299999999999998,"15min":0.60399999999999998},"min":{"1min":0.41699999999999998,"5min":0.34399999999999997,"15min":0.29199999999999998},"max":{"1min":5.2919999999999998,"5min":5.2919999999999998,"15min":5.2919999999999998},"last":0.433}]},{"osd":7,"last update":"Sat Jun 10 21:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.6359999999999999,"15min":1.46},"min":{"1min":0.33900000000000002,"5min":0.31900000000000001,"15min":0.28199999999999997},"max":{"1min":1.179,"5min":94.102000000000004,"15min":136.904},"last":1.3500000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6850000000000001,"15min":1.4930000000000001},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":2.754,"5min":95.653999999999996,"15min":137.661},"last":1.3109999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2721841152,"data_stored":2721841152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2577465344,"data_stored":2577465344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3383492608,"data_stored":3383492608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2667577344,"data_stored":2667577344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:42:15.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set device_health_metrics pgp_num 21 2023-06-10T21:42:15.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:15 smithi152 ceph-mon[156819]: from='client.46880 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:42:15.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:15.880 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:16.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:16.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:16.263 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:16.333 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:16.402 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 61319 61303 98.2127 72.25 0.0104796 0.0101763 2023-06-10T21:42:16.525 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:42:16.525 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:42:16.579 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:16.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:16.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:16.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:16.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:16.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:16.867 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:16.899 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:16.957 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:16.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:17.060 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:17.089 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:17.266 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:42:17.294 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:17.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:17.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:42:17.400135+0000 min lat: 0.0011044 max lat: 0.322489 avg lat: 0.0103041 2023-06-10T21:42:17.402 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:42:17.402 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 62111 62095 96.9951 49.5 0.0099756 0.0103041 2023-06-10T21:42:17.402 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[180297]: pgmap v1408: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/336221266' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[186706]: pgmap v1408: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:17.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/336221266' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:17 smithi152 ceph-mon[156819]: pgmap v1408: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:17.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/336221266' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:17.454 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:42:17.470 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:17.575 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:17.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:17.684 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:17.745 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:17.811 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:17.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:17.920 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:42:17.982 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:18.060 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:42:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:18.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:42:18.149 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:18.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:42:18.213 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:18.289 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:42:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:18.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:42:18.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:18.401 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 63325 63309 96.4799 75.875 0.00850216 0.0103601 2023-06-10T21:42:18.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[186706]: osdmap e382: 8 total, 8 up, 4 in 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:18.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[180297]: osdmap e382: 8 total, 8 up, 4 in 2023-06-10T21:42:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:18 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:18 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:18 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:18 smithi152 ceph-mon[156819]: osdmap e382: 8 total, 8 up, 4 in 2023-06-10T21:42:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:18 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:18 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:18.467 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:18.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:18.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:18.641 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:18.679 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:18.682 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:18.708 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:18.732 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:18.739 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:18.770 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:18.807 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:42:18.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:18.823 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:18.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:18.847 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:18.888 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.901 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.901 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:42:18.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:19.009 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:19.191 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.403 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 64425 64409 95.8173 68.75 0.0146343 0.010432 2023-06-10T21:42:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: pgmap v1410: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/149958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: pgmap v1410: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:19.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:19.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/149958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:19.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.449 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: failed to encode map e382 with expected crc 2023-06-10T21:42:19.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: pgmap v1410: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:19.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:19.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/149958757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:20.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:42:20.364 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:20.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:20.404 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 65589 65573 95.2767 72.75 0.0160169 0.0104913 2023-06-10T21:42:20.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:20.489 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:20 smithi064 ceph-mon[180297]: pgmap v1411: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:20.489 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:20 smithi064 ceph-mon[186706]: pgmap v1411: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:20.583 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:20 smithi152 ceph-mon[156819]: pgmap v1411: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:21.350 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:21.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:21.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:21.405 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 66623 66607 94.5801 64.625 0.007076 0.0105685 2023-06-10T21:42:21.457 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:21.514 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:21.621 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:21.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:21.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:21.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:21.785 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:21.865 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:21.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:21.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:21.987 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:22.081 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:22.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:22.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:22.271 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:22.405 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 67774 67758 94.0769 71.9375 0.0110169 0.0106248 2023-06-10T21:42:22.437 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-10T21:42:22.546 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:42:22.626 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:42:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:42:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:42:22.626 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:42:22.887 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:22] "GET /metrics HTTP/1.1" 200 224171 "" "Prometheus/2.33.4" 2023-06-10T21:42:23.405 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 68932 68916 93.6051 72.375 0.0167057 0.010678 2023-06-10T21:42:23.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[180297]: pgmap v1412: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:23.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/163995002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:23.427 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[186706]: pgmap v1412: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/163995002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:23.428 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:23 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:23.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:23 smithi152 ceph-mon[156819]: pgmap v1412: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-10T21:42:23.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/163995002' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:23.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:23 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-10T21:42:24.223 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:24.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:24.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:24.242 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:42:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-10T21:42:24.340 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:24.393 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:24.407 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 69933 69917 92.9412 62.5625 0.0178095 0.0107545 2023-06-10T21:42:24.477 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:24.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:24.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:24.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:24.657 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:24.924 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:24.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:25.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:25.084 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:25.105 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:25.199 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:25.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:25.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:25.407 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 71066 71050 92.4795 70.8125 0.0149452 0.0108075 2023-06-10T21:42:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:25 smithi152 ceph-mon[156819]: pgmap v1413: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:42:25.470 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:25.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:25 smithi064 ceph-mon[180297]: pgmap v1413: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:42:25.534 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:25 smithi064 ceph-mon[186706]: pgmap v1413: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 55 MiB/s wr, 883 op/s 2023-06-10T21:42:25.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:25.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:25.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:25.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:25.851 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:25.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:26.214 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:42:26.409 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 72028 72012 91.8165 60.125 0.00276651 0.0108577 2023-06-10T21:42:26.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:26 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:26.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:26 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:26 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:27.409 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 73047 73031 91.254 63.6875 0.00623852 0.0109537 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: pgmap v1414: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: osdmap e383: 8 total, 8 up, 4 in 2023-06-10T21:42:27.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: failed to encode map e383 with expected crc 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[186706]: failed to encode map e383 with expected crc 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: pgmap v1414: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:27.615 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: osdmap e383: 8 total, 8 up, 4 in 2023-06-10T21:42:27.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: failed to encode map e383 with expected crc 2023-06-10T21:42:27.616 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:27 smithi064 ceph-mon[180297]: failed to encode map e383 with expected crc 2023-06-10T21:42:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: pgmap v1414: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1247906093' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: osdmap e383: 8 total, 8 up, 4 in 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: failed to encode map e383 with expected crc 2023-06-10T21:42:27.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:27 smithi152 ceph-mon[156819]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.010 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:28.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:28.117 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:28.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:28.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:28.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[180297]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[180297]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.303 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[180297]: pgmap v1416: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:28.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[186706]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[186706]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.303 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:28 smithi064 ceph-mon[186706]: pgmap v1416: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:28.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:28.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:28.412 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 74282 74266 90.9731 77.1875 0.0139518 0.0109865 2023-06-10T21:42:28.464 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:28.543 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:28.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:28.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:28 smithi152 ceph-mon[156819]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:28 smithi152 ceph-mon[156819]: failed to encode map e383 with expected crc 2023-06-10T21:42:28.561 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:28 smithi152 ceph-mon[156819]: pgmap v1416: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-10T21:42:28.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:28.736 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:28.836 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:28.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:28.881 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:28.912 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:28.944 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:28.945 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:28.952 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:42:28.952 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:42:28.992 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:29.052 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:29.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:29.078 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:42:29.112 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:29.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:29.156 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.157 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.158 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.159 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:42:29.223 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:29.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:29.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:29.266 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:42:29.266 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:42:29.267 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [1, 2, 4, 5, 6, 7, 0, 3] 2023-06-10T21:42:29.267 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-06-10T21:42:29.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5 2023-06-10T21:42:29.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:29.378 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3767682528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:29.379 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3767682528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:29.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:29.414 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 75361 75345 90.5172 67.4375 0.0233524 0.0110419 2023-06-10T21:42:29.683 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3767682528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:29.683 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:29 smithi152 systemd[1]: Stopping Ceph osd.5 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:42:29.943 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:29 smithi152 conmon[109488]: debug 2023-06-10T21:42:29.679+0000 7f6898613700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:42:29.943 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:29 smithi152 conmon[109488]: debug 2023-06-10T21:42:29.679+0000 7f6898613700 -1 osd.5 383 *** Got signal Terminated *** 2023-06-10T21:42:29.944 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:29 smithi152 conmon[109488]: debug 2023-06-10T21:42:29.679+0000 7f6898613700 -1 osd.5 383 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:42:30.160 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:42:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:30 smithi064 ceph-mon[180297]: osd.5 marked itself down and dead 2023-06-10T21:42:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:30 smithi064 ceph-mon[180297]: pgmap v1417: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:30 smithi064 ceph-mon[186706]: osd.5 marked itself down and dead 2023-06-10T21:42:30.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:30 smithi064 ceph-mon[186706]: pgmap v1417: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:30.414 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 76096 76080 89.6763 45.9375 0.00863349 0.0111088 2023-06-10T21:42:30.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:30 smithi152 ceph-mon[156819]: osd.5 marked itself down and dead 2023-06-10T21:42:30.537 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:30 smithi152 ceph-mon[156819]: pgmap v1417: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-10T21:42:30.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:42:30.620 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:30.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:30.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:30.725 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:30.755 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:30.765 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:30.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:30.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:30.918 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:30.948 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:31.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:31.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:31.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:31.135 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:31.288 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:31.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:31.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:31.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[180297]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:42:31.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[180297]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[180297]: osdmap e384: 8 total, 7 up, 4 in 2023-06-10T21:42:31.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[180297]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[180297]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[186706]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:42:31.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[186706]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[186706]: osdmap e384: 8 total, 7 up, 4 in 2023-06-10T21:42:31.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[186706]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:31 smithi064 ceph-mon[186706]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.415 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 76271 76255 88.2159 10.9375 0.0137105 0.0113305 2023-06-10T21:42:31.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:31.580 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:31.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:31.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:31.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:31 smithi152 ceph-mon[156819]: Health check failed: 1 osds down (OSD_DOWN) 2023-06-10T21:42:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:31 smithi152 ceph-mon[156819]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:31 smithi152 ceph-mon[156819]: osdmap e384: 8 total, 7 up, 4 in 2023-06-10T21:42:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:31 smithi152 ceph-mon[156819]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:31 smithi152 ceph-mon[156819]: failed to encode map e384 with expected crc 2023-06-10T21:42:31.750 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:32.027 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:32.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:32.053 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:32.136 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:32.203 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:32.250 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[180297]: failed to encode map e384 with expected crc 2023-06-10T21:42:32.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[180297]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[180297]: osdmap e385: 8 total, 7 up, 4 in 2023-06-10T21:42:32.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[180297]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[180297]: pgmap v1420: 89 pgs: 16 peering, 19 stale+active+clean, 54 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:42:32.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 77517 77501 88.0278 77.875 0.0132147 0.0113547 2023-06-10T21:42:32.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:32.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:32.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:32.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[186706]: failed to encode map e384 with expected crc 2023-06-10T21:42:32.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[186706]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[186706]: osdmap e385: 8 total, 7 up, 4 in 2023-06-10T21:42:32.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[186706]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.594 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:32 smithi064 ceph-mon[186706]: pgmap v1420: 89 pgs: 16 peering, 19 stale+active+clean, 54 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:42:32.603 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:42:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:32 smithi152 ceph-mon[156819]: failed to encode map e384 with expected crc 2023-06-10T21:42:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:32 smithi152 ceph-mon[156819]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:32 smithi152 ceph-mon[156819]: osdmap e385: 8 total, 7 up, 4 in 2023-06-10T21:42:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:32 smithi152 ceph-mon[156819]: failed to encode map e385 with expected crc 2023-06-10T21:42:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:32 smithi152 ceph-mon[156819]: pgmap v1420: 89 pgs: 16 peering, 19 stale+active+clean, 54 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 57 MiB/s wr, 905 op/s 2023-06-10T21:42:32.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:32.725 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:32.779 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:32.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:32.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:32.916 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:32.961 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:32.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:32.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:33.066 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:33.098 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:33.150 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:33.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:33.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:32] "GET /metrics HTTP/1.1" 200 224171 "" "Prometheus/2.33.4" 2023-06-10T21:42:33.257 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:33.275 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:42:33.285 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:33.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 78643 78627 87.7126 70.375 0.0144489 0.0113967 2023-06-10T21:42:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: failed to encode map e385 with expected crc 2023-06-10T21:42:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-06-10T21:42:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3009144807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:33.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:33.534 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:33.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:33.639 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: failed to encode map e385 with expected crc 2023-06-10T21:42:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-06-10T21:42:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3009144807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: failed to encode map e385 with expected crc 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3009144807' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:42:33.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:33.669 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:33.772 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:33.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:33.878 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:33.934 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:33 smithi152 bash[314238]: fa30a06be4b988b1fe960ba1fd5c5ab3952e7c7aed44de4843aaddbd0f7288e7 2023-06-10T21:42:33.942 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:34.013 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:34.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:34.122 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:34.190 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:34.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 79715 79699 87.3492 67 0.0041068 0.0114399 2023-06-10T21:42:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: osdmap e386: 8 total, 7 up, 4 in 2023-06-10T21:42:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[186706]: pgmap v1422: 89 pgs: 8 active+undersized+degraded, 21 active+undersized, 16 peering, 7 stale+active+clean, 37 active+clean; 4.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s; 24194/228846 objects degraded (10.572%) 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: osdmap e386: 8 total, 7 up, 4 in 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:34 smithi064 ceph-mon[180297]: pgmap v1422: 89 pgs: 8 active+undersized+degraded, 21 active+undersized, 16 peering, 7 stale+active+clean, 37 active+clean; 4.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s; 24194/228846 objects degraded (10.572%) 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: osdmap e386: 8 total, 7 up, 4 in 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: failed to encode map e386 with expected crc 2023-06-10T21:42:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:34 smithi152 ceph-mon[156819]: pgmap v1422: 89 pgs: 8 active+undersized+degraded, 21 active+undersized, 16 peering, 7 stale+active+clean, 37 active+clean; 4.7 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 63 MiB/s wr, 1.00k op/s; 24194/228846 objects degraded (10.572%) 2023-06-10T21:42:34.864 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:34.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:34.976 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:35.012 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:35.046 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:35.075 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:35.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:35.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:35.257 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:35.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 80864 80848 87.0814 71.8125 0.0125891 0.0114793 2023-06-10T21:42:35.437 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:35 smithi064 ceph-mon[186706]: Health check failed: Degraded data redundancy: 24194/228846 objects degraded (10.572%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:35.437 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:35 smithi064 ceph-mon[180297]: Health check failed: Degraded data redundancy: 24194/228846 objects degraded (10.572%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:35.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:35 smithi152 ceph-mon[156819]: Health check failed: Degraded data redundancy: 24194/228846 objects degraded (10.572%), 8 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:35.470 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:35.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:35.525 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:42:35.525 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:42:35.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:35.656 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:35.818 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5.service' 2023-06-10T21:42:35.882 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:42:35.882 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-06-10T21:42:36.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:36.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:36.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:36.280 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:36.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:36.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 82046 82030 86.8575 73.875 0.00260076 0.011497 2023-06-10T21:42:36.536 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:36.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:36.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:36.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:36.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:42:37.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:42:37.414838+0000 min lat: 0.0011044 max lat: 1.16358 avg lat: 0.0116004 2023-06-10T21:42:37.417 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:42:37.417 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 82771 82755 86.1652 45.3125 0.01053 0.0116004 2023-06-10T21:42:37.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:37 smithi152 ceph-mon[156819]: pgmap v1423: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 58 MiB/s wr, 922 op/s; 69408/238734 objects degraded (29.073%); 53 KiB/s, 0 objects/s recovering 2023-06-10T21:42:37.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:37 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-06-10T21:42:37.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:37 smithi064 ceph-mon[186706]: pgmap v1423: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 58 MiB/s wr, 922 op/s; 69408/238734 objects degraded (29.073%); 53 KiB/s, 0 objects/s recovering 2023-06-10T21:42:37.452 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:37 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-06-10T21:42:37.453 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:42:37.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:37 smithi064 ceph-mon[180297]: pgmap v1423: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 4.9 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 58 MiB/s wr, 922 op/s; 69408/238734 objects degraded (29.073%); 53 KiB/s, 0 objects/s recovering 2023-06-10T21:42:37.456 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:37 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-06-10T21:42:37.466 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:37.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:37.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:37.652 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:37.819 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:37.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:37.926 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:37.959 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:38.015 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:38.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:38.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:38.035 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:38.125 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:38.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:38.207 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:38.243 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:38.279 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:38.331 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:38.367 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:38.367 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:38.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:38.412 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:38.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 83760 83744 85.766 61.8125 0.0186998 0.011654 2023-06-10T21:42:38.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3187678546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:38.439 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3187678546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:38.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3187678546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:38.447 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:38.447 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.447 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.448 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:42:38.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:38.503 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:38.555 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:38.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:38.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:38.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:38.729 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:39.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 85045 85029 85.678 80.3125 0.00900952 0.0116676 2023-06-10T21:42:39.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:39 smithi064 ceph-mon[186706]: pgmap v1424: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.0 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 57 MiB/s wr, 907 op/s; 71057/243681 objects degraded (29.160%); 40 KiB/s, 0 objects/s recovering 2023-06-10T21:42:39.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:39 smithi064 ceph-mon[180297]: pgmap v1424: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.0 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 57 MiB/s wr, 907 op/s; 71057/243681 objects degraded (29.160%); 40 KiB/s, 0 objects/s recovering 2023-06-10T21:42:39.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:39 smithi152 ceph-mon[156819]: pgmap v1424: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.0 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 57 MiB/s wr, 907 op/s; 71057/243681 objects degraded (29.160%); 40 KiB/s, 0 objects/s recovering 2023-06-10T21:42:39.733 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:39.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:39.928 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:39.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:40.030 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:40.063 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:42:40.103 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:42:40.113 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:40.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:40.222 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:40.298 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:40.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 86203 86187 85.4668 72.375 0.00359126 0.0116941 2023-06-10T21:42:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:40 smithi152 ceph-mon[156819]: pgmap v1425: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 52 MiB/s wr, 837 op/s; 72665/248505 objects degraded (29.241%); 36 KiB/s, 0 objects/s recovering 2023-06-10T21:42:40.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:40 smithi064 ceph-mon[180297]: pgmap v1425: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 52 MiB/s wr, 837 op/s; 72665/248505 objects degraded (29.241%); 36 KiB/s, 0 objects/s recovering 2023-06-10T21:42:40.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:40 smithi064 ceph-mon[186706]: pgmap v1425: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.1 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 52 MiB/s wr, 837 op/s; 72665/248505 objects degraded (29.241%); 36 KiB/s, 0 objects/s recovering 2023-06-10T21:42:40.883 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [5] live_osds: [1, 2, 4, 6, 7, 0, 3] 2023-06-10T21:42:40.883 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-06-10T21:42:40.883 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-06-10T21:42:40.884 DEBUG:teuthology.orchestra.run.smithi152:osd.5> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5.service 2023-06-10T21:42:40.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5 2023-06-10T21:42:40.915 INFO:journalctl@ceph.osd.5.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T21:42:41.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:41.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:41.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:41.157 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:41.193 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:40 smithi152 systemd[1]: Starting Ceph osd.5 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:42:41.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:41.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:41.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:41.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 87227 87211 85.1314 64 0.00664144 0.011742 2023-06-10T21:42:41.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:41.454 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:41.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 72665/248505 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:41.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 72665/248505 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:41.509 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 72665/248505 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:41.616 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:41.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:41.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:41.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:41.762 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:41.824 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:41.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:41.929 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:41.973 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:42.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:42:42.241 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:42 smithi064 ceph-mon[180297]: pgmap v1426: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.2 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 59 MiB/s wr, 944 op/s; 73988/254442 objects degraded (29.079%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:42.350 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-06-10T21:42:42.350 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-06-10T21:42:42.350 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-06-10T21:42:42.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-06-10T21:42:42.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2023-06-10T21:42:42.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-06-10T21:42:42.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:42 smithi152 bash[315453]: --> ceph-volume raw activate successful for osd ID: 5 2023-06-10T21:42:42.351 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:42 smithi152 ceph-mon[156819]: pgmap v1426: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.2 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 59 MiB/s wr, 944 op/s; 73988/254442 objects degraded (29.079%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:42.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:42.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:42.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:42.416 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 88293 88277 84.8467 66.625 0.0108916 0.0117812 2023-06-10T21:42:42.504 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:42.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:42 smithi064 ceph-mon[186706]: pgmap v1426: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.2 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 59 MiB/s wr, 944 op/s; 73988/254442 objects degraded (29.079%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:42.576 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:42.710 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:42.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:42.822 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:42.853 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:42.908 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:42.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:43.012 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:43.055 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:43.090 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:42] "GET /metrics HTTP/1.1" 200 224147 "" "Prometheus/2.33.4" 2023-06-10T21:42:43.220 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:42:43.220 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:42:43.220 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:42:43.220 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:42:43.222 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:43.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:43.280 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:42:43.328 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:43.363 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:43.419 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 89326 89310 84.5392 64.5625 0.00614477 0.0118242 2023-06-10T21:42:43.496 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:43.498 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:43.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:43.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:43.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:43.639 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:43.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:43.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:42:43.703 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:43.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:43.809 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:43.872 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:44.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:42:44.417 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 90476 90460 84.3502 71.875 0.0101731 0.0118512 2023-06-10T21:42:44.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:44 smithi152 conmon[316210]: debug 2023-06-10T21:42:44.162+0000 7ff6a1c42700 1 -- 172.21.15.152:0/4113008683 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e7632084e0 con 0x55e7631cf800 2023-06-10T21:42:44.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:44 smithi152 bash[316169]: c1f73cf499b97d49f4b54499cba9bdef02f80f918d9c8d8aa337b172e42cef78 2023-06-10T21:42:44.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:44 smithi152 systemd[1]: Started Ceph osd.5 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:42:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:44 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:44 smithi152 ceph-mon[156819]: osdmap e387: 8 total, 7 up, 4 in 2023-06-10T21:42:44.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:44 smithi152 ceph-mon[156819]: failed to encode map e387 with expected crc 2023-06-10T21:42:44.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:44 smithi152 ceph-mon[156819]: pgmap v1428: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.3 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 75679/259515 objects degraded (29.162%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:44.490 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:44.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:44.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:44.594 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:44.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:44.619 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[186706]: osdmap e387: 8 total, 7 up, 4 in 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[186706]: failed to encode map e387 with expected crc 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[186706]: pgmap v1428: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.3 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 75679/259515 objects degraded (29.162%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1443219692' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:42:44.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[180297]: osdmap e387: 8 total, 7 up, 4 in 2023-06-10T21:42:44.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[180297]: failed to encode map e387 with expected crc 2023-06-10T21:42:44.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:44 smithi064 ceph-mon[180297]: pgmap v1428: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.3 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s; 75679/259515 objects degraded (29.162%); 32 KiB/s, 0 objects/s recovering 2023-06-10T21:42:44.622 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:44.652 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:44.943 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:44 smithi152 conmon[316210]: debug 2023-06-10T21:42:44.680+0000 7ff6a96be200 -1 Falling back to public interface 2023-06-10T21:42:45.264 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:45.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:45.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:45.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:45 smithi064 ceph-mon[180297]: failed to encode map e387 with expected crc 2023-06-10T21:42:45.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:45 smithi064 ceph-mon[186706]: failed to encode map e387 with expected crc 2023-06-10T21:42:45.417 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 91479 91463 84.0316 62.6875 0.00744727 0.0118953 2023-06-10T21:42:45.420 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:42:45.456 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:45.574 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:45.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:45.690 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:45 smithi152 ceph-mon[156819]: failed to encode map e387 with expected crc 2023-06-10T21:42:45.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:45.890 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:45.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:46.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:46.048 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:46.128 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:46.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:46.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:46.298 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:46.413 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:46.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 92621 92605 83.8434 71.375 0.0137883 0.0119223 2023-06-10T21:42:46.451 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:46.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:46.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:0 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:1 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:2 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2023-06-10T21:42:46.469 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2023-06-10T21:42:46.470 INFO:teuthology.orchestra.run.smithi152.stderr:config diff get 2023-06-10T21:42:46.470 INFO:teuthology.orchestra.run.smithi152.stderr:config get 2023-06-10T21:42:46.470 INFO:teuthology.orchestra.run.smithi152.stderr:config help [] 2023-06-10T21:42:46.470 INFO:teuthology.orchestra.run.smithi152.stderr:config set ... 2023-06-10T21:42:46.470 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2023-06-10T21:42:46.488 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:46 smithi152 ceph-mon[156819]: pgmap v1429: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.5 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 78797/270747 objects degraded (29.104%) 2023-06-10T21:42:46.488 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 75679/259515 objects degraded (29.162%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:46.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:46.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:46 smithi064 ceph-mon[186706]: pgmap v1429: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.5 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 78797/270747 objects degraded (29.104%) 2023-06-10T21:42:46.617 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 75679/259515 objects degraded (29.162%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:46.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:46 smithi064 ceph-mon[180297]: pgmap v1429: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.5 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 78797/270747 objects degraded (29.104%) 2023-06-10T21:42:46.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 75679/259515 objects degraded (29.162%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:46.634 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:46.789 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:46.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:46.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:46.895 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:46.926 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:46.938 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:46.954 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:46.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:47.028 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:47.063 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:47.098 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:47.107 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:47.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:47.136 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.137 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-06-10T21:42:47.138 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.139 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:42:47.165 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:42:47.215 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:47.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:42:47.302 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-06-10T21:42:47.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:47.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/70281203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:47.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/70281203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:47.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 93539 93523 83.4653 57.375 0.00880445 0.0119768 2023-06-10T21:42:47.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:47.459 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:47.510 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:47.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:47.517 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/70281203' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:47.617 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:47.650 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:47.705 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:47.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:47.813 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:47.840 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:48.307 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:48.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:48.417 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:48.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 94186 94170 82.8595 40.4375 0.019067 0.012049 2023-06-10T21:42:48.467 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:48.840 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:48.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:48.931 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:42:48.932 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:42:48.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:48.978 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:49.045 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:49.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:49.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:42:49.214 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:49.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:49.395 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:42:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:49.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:42:49.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 95370 95354 82.7364 74 0.00991175 0.0120817 2023-06-10T21:42:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:49 smithi152 ceph-mon[156819]: pgmap v1430: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.6 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 80474/275778 objects degraded (29.181%) 2023-06-10T21:42:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:49.503 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:49.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[186706]: pgmap v1430: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.6 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 80474/275778 objects degraded (29.181%) 2023-06-10T21:42:49.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:49.577 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:49.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[180297]: pgmap v1430: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.6 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.07k op/s; 80474/275778 objects degraded (29.181%) 2023-06-10T21:42:49.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:42:49.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:42:49.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:49.673 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:49.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:49.756 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:42:49.757 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:42:49.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:49.854 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:50.422 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 96569 96553 82.6295 74.9375 0.019859 0.0120977 2023-06-10T21:42:50.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:50 smithi064 ceph-mon[180297]: pgmap v1431: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.7 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s; 81992/280332 objects degraded (29.248%) 2023-06-10T21:42:50.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:50 smithi064 ceph-mon[186706]: pgmap v1431: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.7 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s; 81992/280332 objects degraded (29.248%) 2023-06-10T21:42:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:50 smithi152 ceph-mon[156819]: pgmap v1431: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.7 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s; 81992/280332 objects degraded (29.248%) 2023-06-10T21:42:50.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:42:51.230 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 81992/280332 objects degraded (29.248%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:51.275 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:51.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:51.383 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:51.410 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:51.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 97386 97370 82.203 51.0625 0.00437695 0.0121444 2023-06-10T21:42:51.455 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:51.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:51.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:51.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:51.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 81992/280332 objects degraded (29.248%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:51.628 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 81992/280332 objects degraded (29.248%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:51.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:42:52.016 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:52.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:52.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:52.128 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:52.159 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:52.190 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:52.211 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:52.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:52.264 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:52 smithi064 ceph-mon[180297]: pgmap v1432: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 83243/286002 objects degraded (29.106%) 2023-06-10T21:42:52.269 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:52 smithi064 ceph-mon[186706]: pgmap v1432: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 83243/286002 objects degraded (29.106%) 2023-06-10T21:42:52.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:42:52.320 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:42:52.401 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:52.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 98416 98400 81.9654 64.375 0.023246 0.012196 2023-06-10T21:42:52.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:52 smithi152 ceph-mon[156819]: pgmap v1432: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.8 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 83243/286002 objects degraded (29.106%) 2023-06-10T21:42:52.653 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:42:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:52.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:42:52.758 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:42:52.759 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:52.834 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:53.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:42:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:42:52] "GET /metrics HTTP/1.1" 200 224152 "" "Prometheus/2.33.4" 2023-06-10T21:42:53.286 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:42:53.415 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2603671994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.415 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.421 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 99349 99333 81.6541 58.3125 0.00341622 0.0122317 2023-06-10T21:42:53.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:53.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2603671994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.631 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2603671994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.631 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:42:53.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:53.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:53.733 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:53.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:54.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:54 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:54.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:54.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:54 smithi152 ceph-mon[156819]: osdmap e388: 8 total, 7 up, 4 in 2023-06-10T21:42:54.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:54 smithi152 ceph-mon[156819]: failed to encode map e388 with expected crc 2023-06-10T21:42:54.314 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:54 smithi152 ceph-mon[156819]: pgmap v1434: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 84794/290655 objects degraded (29.173%) 2023-06-10T21:42:54.422 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 100171 100155 81.261 51.375 0.0100296 0.0123016 2023-06-10T21:42:54.532 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:54.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:54.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:54.536 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[186706]: osdmap e388: 8 total, 7 up, 4 in 2023-06-10T21:42:54.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[186706]: failed to encode map e388 with expected crc 2023-06-10T21:42:54.537 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[186706]: pgmap v1434: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 84794/290655 objects degraded (29.173%) 2023-06-10T21:42:54.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:42:54.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:42:54.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[180297]: osdmap e388: 8 total, 7 up, 4 in 2023-06-10T21:42:54.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[180297]: failed to encode map e388 with expected crc 2023-06-10T21:42:54.538 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:54 smithi064 ceph-mon[180297]: pgmap v1434: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 5.9 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 84794/290655 objects degraded (29.173%) 2023-06-10T21:42:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:54.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:54.539 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2023-06-10T21:42:54.539 INFO:teuthology.orchestra.run.smithi152.stderr:0 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:1 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:2 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2023-06-10T21:42:54.540 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2023-06-10T21:42:54.541 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator dump block 2023-06-10T21:42:54.541 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator fragmentation block 2023-06-10T21:42:54.541 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2023-06-10T21:42:54.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:54.667 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:54.723 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:54.813 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:54.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:54.917 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:54.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:55.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:55.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:55.262 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:55.343 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:42:55.363 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-06-10T21:42:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:55 smithi064 ceph-mon[186706]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:55 smithi064 ceph-mon[186706]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:55 smithi064 ceph-mon[180297]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:55 smithi064 ceph-mon[180297]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.424 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 101269 101253 81.0969 68.625 0.0182781 0.0123259 2023-06-10T21:42:55.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:55.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:55.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:55.595 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:42:55.606 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:55.637 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:42:55.672 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:55.677 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:42:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:55 smithi152 ceph-mon[156819]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:55 smithi152 ceph-mon[156819]: failed to encode map e388 with expected crc 2023-06-10T21:42:55.726 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:42:55.750 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:42:55.784 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:42:55.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:55.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:55.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:55.818 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:42:55.818 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-06-10T21:42:55.819 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.820 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:42:55.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:55.955 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:56.015 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:56.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:56.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:56.120 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:56.183 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:56.424 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 102210 102194 80.8149 58.8125 0.00338311 0.0123369 2023-06-10T21:42:56.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:42:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2660833463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[180297]: pgmap v1435: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 989 op/s; 87449/300435 objects degraded (29.107%) 2023-06-10T21:42:56.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 84794/290655 objects degraded (29.173%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2660833463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[186706]: pgmap v1435: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 989 op/s; 87449/300435 objects degraded (29.107%) 2023-06-10T21:42:56.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 84794/290655 objects degraded (29.173%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2660833463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:42:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:56 smithi152 ceph-mon[156819]: pgmap v1435: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.1 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 989 op/s; 87449/300435 objects degraded (29.107%) 2023-06-10T21:42:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 84794/290655 objects degraded (29.173%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:42:57.005 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:57.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:57.116 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:57.153 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:57.214 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:57.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:57.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:57.356 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:57.415 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:57.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:57.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:42:57.423383+0000 min lat: 0.0011044 max lat: 1.16358 avg lat: 0.0123766 2023-06-10T21:42:57.426 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:42:57.426 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 103448 103432 80.771 77.375 0.027449 0.0123766 2023-06-10T21:42:57.520 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:57.557 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:57.610 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:57.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:57.643 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:42:57.643 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:42:57.719 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:57.785 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:57.897 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:57.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:42:58.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:42:58.021 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:42:58.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:58.174 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(12.b 338/325 [PushOp(12:dd5a42db:::benchmark_data_smithi064_442287_object28223:head, version: 316'1679, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:dd5a42db:::benchmark_data_smithi064_442287_object28223:head@316'1679, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:46.814524+0000", 2023-06-10T21:42:58.175 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 251.34588473400001, 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23317073099999999, 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814524+0000", 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814524+0000", 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.177 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814523+0000", 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.178 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814552+0000", 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9189000000000001e-05 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814571+0000", 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8151999999999998e-05 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814583+0000", 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2085e-05 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814650+0000", 2023-06-10T21:42:58.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7124000000000003e-05 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:46.814658+0000", 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4849999999999992e-06 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:47.047695+0000", 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.233036734 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(12.2 342/331 [PushOp(12:4f328d29:::benchmark_data_smithi064_442287_object30473:head, version: 316'1945, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:4f328d29:::benchmark_data_smithi064_442287_object30473:head@316'1945, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.203751+0000", 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 233.95665801800001, 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25779338600000001, 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.183 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203751+0000", 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203751+0000", 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203749+0000", 2023-06-10T21:42:58.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203819+0000", 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0129000000000006e-05 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203845+0000", 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.584e-05 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.203866+0000", 2023-06-10T21:42:58.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0701999999999999e-05 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.461368+0000", 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25750240099999999 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.461544+0000", 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017618500000000001 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117687 12.a e343/331 12:50ee7d02:::benchmark_data_smithi064_442287_object29637:head v 343'7458, mlcod=316'1040)", 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.204721+0000", 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 233.95568838, 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25672173500000001, 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204721+0000", 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204721+0000", 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204719+0000", 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:42:58.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204732+0000", 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3217999999999999e-05 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204747+0000", 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4182000000000001e-05 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.204757+0000", 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0156000000000001e-05 2023-06-10T21:42:58.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.236510+0000", 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031753627 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.236542+0000", 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1921000000000002e-05 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.461396+0000", 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.22485385999999999 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.461442+0000", 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6205000000000002e-05 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136642 12.b e354/353 12:d6d9984d:::benchmark_data_smithi064_442287_object48592:head v 354'8532, mlcod=321'4217)", 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.852073+0000", 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.30833577000001, 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.53438403300000004, 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.198 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852073+0000", 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852073+0000", 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852072+0000", 2023-06-10T21:42:58.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852083+0000", 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0634e-05 2023-06-10T21:42:58.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852098+0000", 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5369000000000001e-05 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852108+0000", 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4590000000000004e-06 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852159+0000", 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1180999999999997e-05 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.852193+0000", 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4288000000000001e-05 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.204 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.386420+0000", 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.53422675900000005 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.386457+0000", 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7496999999999997e-05 2023-06-10T21:42:58.205 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136653 12.b e354/353 12:d5043e89:::benchmark_data_smithi064_442287_object48603:head v 354'8533, mlcod=321'4217)", 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.913396+0000", 2023-06-10T21:42:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.247012587, 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47760669100000003, 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913396+0000", 2023-06-10T21:42:58.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913396+0000", 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913396+0000", 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913404+0000", 2023-06-10T21:42:58.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6060000000000003e-06 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913416+0000", 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1589e-05 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913424+0000", 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.96e-06 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.120857+0000", 2023-06-10T21:42:58.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.20743300100000001 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.120883+0000", 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6704999999999999e-05 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.390971+0000", 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27008809299999997 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391003+0000", 2023-06-10T21:42:58.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1720999999999997e-05 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136658 12.b e354/353 12:d9355ce3:::benchmark_data_smithi064_442287_object48608:head v 354'8534, mlcod=321'4217)", 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.913430+0000", 2023-06-10T21:42:58.214 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.24697862799999, 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47759796100000002, 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913430+0000", 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913430+0000", 2023-06-10T21:42:58.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913430+0000", 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913434+0000", 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3830000000000001e-06 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913439+0000", 2023-06-10T21:42:58.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.198e-06 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.913444+0000", 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7400000000000004e-06 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.299015+0000", 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.38557137800000002 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.299038+0000", 2023-06-10T21:42:58.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3238e-05 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391006+0000", 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.091967627999999996 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391028+0000", 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2629e-05 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136659 12.b e354/353 12:dcd34507:::benchmark_data_smithi064_442287_object48609:head v 354'8535, mlcod=321'4217)", 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.918829+0000", 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.24157982200001, 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.480354061, 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918829+0000", 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918829+0000", 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918828+0000", 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918837+0000", 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5639999999999997e-06 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918854+0000", 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6701000000000002e-05 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.918862+0000", 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.157e-06 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.386549+0000", 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46768739599999998 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.386574+0000", 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5181000000000002e-05 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.399137+0000", 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012562672 2023-06-10T21:42:58.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.399183+0000", 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6306e-05 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136675 12.16 e354/352 12:6eedbdc4:::benchmark_data_smithi064_442287_object48625:head v 354'7810, mlcod=354'7808)", 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.961023+0000", 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.199385909, 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.42397395700000001, 2023-06-10T21:42:58.231 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961023+0000", 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961023+0000", 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961022+0000", 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961031+0000", 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2439999999999999e-06 2023-06-10T21:42:58.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961050+0000", 2023-06-10T21:42:58.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.925e-05 2023-06-10T21:42:58.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.961059+0000", 2023-06-10T21:42:58.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6230000000000003e-06 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.962497+0000", 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001438525 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.962526+0000", 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8334999999999999e-05 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.384953+0000", 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.42242680100000002 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.384997+0000", 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4526e-05 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.239 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136673 12.19 e354/352 12:9f737749:::benchmark_data_smithi064_442287_object48623:head v 354'7965, mlcod=320'3072)", 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:51.965244+0000", 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.19516528899999, 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.232432785, 2023-06-10T21:42:58.240 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965244+0000", 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965244+0000", 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965243+0000", 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965250+0000", 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4100000000000002e-06 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965267+0000", 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6991999999999999e-05 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965275+0000", 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0779999999999996e-06 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965345+0000", 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9493999999999996e-05 2023-06-10T21:42:58.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.965381+0000", 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6173000000000001e-05 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.197625+0000", 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23224394800000001 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.197677+0000", 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1473e-05 2023-06-10T21:42:58.247 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136681 12.6 e354/352 12:64598cb5:::benchmark_data_smithi064_442287_object48631:head v 354'7825, mlcod=354'7823)", 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:52.043002+0000", 2023-06-10T21:42:58.248 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.117406867, 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34534440599999999, 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043002+0000", 2023-06-10T21:42:58.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043002+0000", 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043001+0000", 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043009+0000", 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0180000000000001e-06 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043026+0000", 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7034e-05 2023-06-10T21:42:58.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.043034+0000", 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0499999999999992e-06 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.299122+0000", 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.256088013 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.299148+0000", 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5165e-05 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.388309+0000", 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.089161377 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.388347+0000", 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7648000000000003e-05 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.256 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136674 12.f e354/336 12:f0ab7462:::benchmark_data_smithi064_442287_object48624:head v 354'8547, mlcod=316'1826)", 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:52.055485+0000", 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.104924117, 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33516390499999998, 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055485+0000", 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055485+0000", 2023-06-10T21:42:58.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055485+0000", 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055489+0000", 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8999999999999999e-06 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055494+0000", 2023-06-10T21:42:58.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.434e-06 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.055498+0000", 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1479999999999999e-06 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.198189+0000", 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14269077999999999 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.198216+0000", 2023-06-10T21:42:58.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7382999999999999e-05 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.390612+0000", 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.19239557700000001 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.390649+0000", 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6950999999999999e-05 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:136687 12.b e354/353 12:dcf05046:::benchmark_data_smithi064_442287_object48637:head v 354'8536, mlcod=321'4217)", 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:52.122351+0000", 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 186.038057608, 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29116568999999998, 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122351+0000", 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122351+0000", 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122351+0000", 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122365+0000", 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4837e-05 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122377+0000", 2023-06-10T21:42:58.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1294000000000001e-05 2023-06-10T21:42:58.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.122384+0000", 2023-06-10T21:42:58.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.396e-06 2023-06-10T21:42:58.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391063+0000", 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.268679051 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.391093+0000", 2023-06-10T21:42:58.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9714999999999999e-05 2023-06-10T21:42:58.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:42:58.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.413479+0000", 2023-06-10T21:42:58.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022386006 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:52.413517+0000", 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8275000000000002e-05 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.273 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(12.a 363/358 [PushOp(12:53f3f12a:::benchmark_data_smithi064_442287_object74883:head, version: 322'4732, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:53f3f12a:::benchmark_data_smithi064_442287_object74883:head@322'4732, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 141.85823119200001, 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25852421799999997, 2023-06-10T21:42:58.274 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302176+0000", 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302218+0000", 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1696000000000002e-05 2023-06-10T21:42:58.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558593+0000", 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25637484599999999 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558629+0000", 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5955999999999999e-05 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558742+0000", 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113353 2023-06-10T21:42:58.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558758+0000", 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5642e-05 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:42:58.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.560702+0000", 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019444530000000001 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:42:58.281 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:42:58.406 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:42:58.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:58.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:42:58.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 104584 104568 80.6503 71 0.0079392 0.012395 2023-06-10T21:42:58.512 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:58.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:58 smithi152 ceph-mon[156819]: pgmap v1436: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.2 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s; 89036/305196 objects degraded (29.173%) 2023-06-10T21:42:58.550 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:58.617 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:58.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:58.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:58 smithi064 ceph-mon[180297]: pgmap v1436: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.2 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s; 89036/305196 objects degraded (29.173%) 2023-06-10T21:42:58.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:58 smithi064 ceph-mon[186706]: pgmap v1436: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.2 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s; 89036/305196 objects degraded (29.173%) 2023-06-10T21:42:58.727 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:42:58.774 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:58.831 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:42:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:58.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:42:58.939 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:58.965 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:59.016 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:59.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:59.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:59.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:42:59.162 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:59.221 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:42:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:59.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:42:59.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:42:59.364 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:59.422 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:42:59.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 105693 105677 80.512 69.3125 0.00805782 0.0124161 2023-06-10T21:42:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:59.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:42:59.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:42:59.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:42:59.638 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:42:59.645 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:42:59.693 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:59 smithi152 conmon[316210]: debug 2023-06-10T21:42:59.447+0000 7ff6a96be200 -1 osd.5 383 log_to_monitors {default=true} 2023-06-10T21:42:59.693 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:42:59 smithi152 conmon[316210]: debug 2023-06-10T21:42:59.547+0000 7ff698bb4700 -1 osd.5 383 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:42:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:59 smithi152 ceph-mon[156819]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:42:59 smithi152 ceph-mon[156819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.718 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:59 smithi064 ceph-mon[186706]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.719 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:42:59 smithi064 ceph-mon[186706]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:59 smithi064 ceph-mon[180297]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.719 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:42:59 smithi064 ceph-mon[180297]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-10T21:42:59.929 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:42:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:42:59.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:00.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:00.132 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:00.364 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:43:00.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 106568 106552 80.2009 54.6875 0.0105141 0.0124641 2023-06-10T21:43:00.665 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:00.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: osdmap e389: 8 total, 7 up, 4 in 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:00 smithi152 ceph-mon[156819]: pgmap v1438: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 63 MiB/s wr, 1.01k op/s; 90729/310275 objects degraded (29.241%) 2023-06-10T21:43:00.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:00.776 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:00.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T21:43:00.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: osdmap e389: 8 total, 7 up, 4 in 2023-06-10T21:43:00.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.807 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.808 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[186706]: pgmap v1438: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 63 MiB/s wr, 1.01k op/s; 90729/310275 objects degraded (29.241%) 2023-06-10T21:43:00.808 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: osdmap e389: 8 total, 7 up, 4 in 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: from='osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: failed to encode map e389 with expected crc 2023-06-10T21:43:00.809 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:00 smithi064 ceph-mon[180297]: pgmap v1438: 89 pgs: 25 active+undersized+degraded, 42 active+undersized, 22 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 63 MiB/s wr, 1.01k op/s; 90729/310275 objects degraded (29.241%) 2023-06-10T21:43:00.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:00.868 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:00.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:00.887 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:00.983 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:01.013 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:01.058 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:01.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:01.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:01.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:01.205 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:01.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:01.330 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:01.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:01.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 106703 106687 79.3468 8.4375 0.0262575 0.0124651 2023-06-10T21:43:01.434 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:01.462 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:01.503 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:01.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:01.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:01.632 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:01.674 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:01.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:01.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:01.778 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:01.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:43:01.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071] boot 2023-06-10T21:43:01.803 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: osdmap e390: 8 total, 8 up, 4 in 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 90729/310275 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:01.804 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: osdmap e391: 8 total, 8 up, 4 in 2023-06-10T21:43:01.805 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:01 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:01.805 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:01.847 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:01.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:43:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071] boot 2023-06-10T21:43:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: osdmap e390: 8 total, 8 up, 4 in 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 90729/310275 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: osdmap e391: 8 total, 8 up, 4 in 2023-06-10T21:43:01.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: osd.5 [v2:172.21.15.152:6808/41874071,v1:172.21.15.152:6809/41874071] boot 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: osdmap e390: 8 total, 8 up, 4 in 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-10T21:43:01.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: failed to encode map e390 with expected crc 2023-06-10T21:43:01.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 90729/310275 objects degraded (29.241%), 25 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:01.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: osdmap e391: 8 total, 8 up, 4 in 2023-06-10T21:43:01.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:01 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:01.950 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:01.982 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:02.204 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:43:02.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:02.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:02.425 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 106703 106687 78.4137 0 - 0.0124651 2023-06-10T21:43:02.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:02.533 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:02.885 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e384 with expected crc 2023-06-10T21:43:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e390 with expected crc 2023-06-10T21:43:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.886 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.887 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: pgmap v1441: 89 pgs: 10 peering, 23 active+undersized+degraded, 34 active+undersized, 22 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 48 MiB/s wr, 760 op/s; 78007/314127 objects degraded (24.833%) 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:02.888 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: osdmap e392: 8 total, 8 up, 4 in 2023-06-10T21:43:02.889 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[186706]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.889 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:02] "GET /metrics HTTP/1.1" 200 224152 "" "Prometheus/2.33.4" 2023-06-10T21:43:02.889 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e384 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e390 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: pgmap v1441: 89 pgs: 10 peering, 23 active+undersized+degraded, 34 active+undersized, 22 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 48 MiB/s wr, 760 op/s; 78007/314127 objects degraded (24.833%) 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:02.891 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: osdmap e392: 8 total, 8 up, 4 in 2023-06-10T21:43:02.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:02 smithi064 ceph-mon[180297]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.937 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e384 with expected crc 2023-06-10T21:43:02.937 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e390 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.938 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e391 with expected crc 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: pgmap v1441: 89 pgs: 10 peering, 23 active+undersized+degraded, 34 active+undersized, 22 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 48 MiB/s wr, 760 op/s; 78007/314127 objects degraded (24.833%) 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e392 with expected crc 2023-06-10T21:43:02.939 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/484699561' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:02.940 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: osdmap e392: 8 total, 8 up, 4 in 2023-06-10T21:43:02.940 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:02 smithi152 ceph-mon[156819]: failed to encode map e392 with expected crc 2023-06-10T21:43:03.173 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:43:03.174 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:43:03.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.174 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107021 13.c e392/391 13:311f4f33:::benchmark_data_smithi064_501823_object107020:head v 392'6735, mlcod=383'4742)", 2023-06-10T21:43:03.174 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.147654+0000", 2023-06-10T21:43:03.174 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.012145388999999999, 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012179697, 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147654+0000", 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147654+0000", 2023-06-10T21:43:03.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147654+0000", 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9560000000000002e-05 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147683+0000", 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2086000000000001e-05 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.178 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147695+0000", 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0610000000000001e-06 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147701+0000", 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1662000000000001e-05 2023-06-10T21:43:03.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147753+0000", 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5055999999999999e-05 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147778+0000", 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012363900000000001 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.181 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107027 13.4 e392/391 13:23f6d18e:::benchmark_data_smithi064_501823_object107026:head v 392'6687, mlcod=385'4761)", 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.147717+0000", 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.01208267, 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012190469000000001, 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147717+0000", 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147717+0000", 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147717+0000", 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5580999999999997e-05 2023-06-10T21:43:03.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147752+0000", 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5990000000000002e-06 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147761+0000", 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2939999999999996e-06 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147766+0000", 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4768999999999999e-05 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147801+0000", 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2586e-05 2023-06-10T21:43:03.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.147823+0000", 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106395 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.188 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107014 13.9 e392/391 13:9c75d99a:::benchmark_data_smithi064_501823_object107013:head v 392'6851, mlcod=383'4852)", 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.152218+0000", 2023-06-10T21:43:03.189 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0075820649999999998, 2023-06-10T21:43:03.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077680839999999998, 2023-06-10T21:43:03.190 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.190 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.190 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.191 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.191 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.191 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152218+0000", 2023-06-10T21:43:03.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152218+0000", 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:03.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152217+0000", 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6714000000000001e-05 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152253+0000", 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4192000000000001e-05 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152267+0000", 2023-06-10T21:43:03.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2079999999999997e-06 2023-06-10T21:43:03.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152274+0000", 2023-06-10T21:43:03.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012887900000000001 2023-06-10T21:43:03.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152403+0000", 2023-06-10T21:43:03.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1376999999999999e-05 2023-06-10T21:43:03.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152424+0000", 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000206329 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.198 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.199 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.199 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107028 13.c e392/391 13:3c6f5128:::benchmark_data_smithi064_501823_object107027:head v 392'6736, mlcod=383'4742)", 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.152285+0000", 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0075142450000000001, 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077658989999999997, 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.200 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152285+0000", 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152285+0000", 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152285+0000", 2023-06-10T21:43:03.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9507999999999999e-05 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152315+0000", 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3600000000000001e-06 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152321+0000", 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9589999999999997e-06 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152325+0000", 2023-06-10T21:43:03.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026325099999999999 2023-06-10T21:43:03.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152588+0000", 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9347999999999998e-05 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.152607+0000", 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00032204 2023-06-10T21:43:03.206 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107022 13.9 e392/391)", 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.153630+0000", 2023-06-10T21:43:03.207 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0061695939999999996, 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064833190000000004, 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153630+0000", 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153630+0000", 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:03.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153629+0000", 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4360999999999997e-05 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153683+0000", 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4651000000000001e-05 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153698+0000", 2023-06-10T21:43:03.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3500000000000002e-06 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153704+0000", 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4307000000000005e-05 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.212 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107030 13.c e392/391 13:38e8ff41:::benchmark_data_smithi064_501823_object107029:head v 392'6737, mlcod=383'4742)", 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.153714+0000", 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0060861739999999998, 2023-06-10T21:43:03.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0064516019999999999, 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153714+0000", 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153714+0000", 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153713+0000", 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5030999999999997e-05 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153748+0000", 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4239999999999998e-06 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153755+0000", 2023-06-10T21:43:03.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1019999999999999e-06 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.153759+0000", 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029629629999999999 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156722+0000", 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7484000000000001e-05 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156739+0000", 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030255519999999999 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.220 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107033 13.8 e392/391 13:1503c8f4:::benchmark_data_smithi064_501823_object107032:head v 392'6725, mlcod=383'4779)", 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.155068+0000", 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.004731463, 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0051597680000000003, 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155068+0000", 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155068+0000", 2023-06-10T21:43:03.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155067+0000", 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9687e-05 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155117+0000", 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2478999999999999e-05 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155150+0000", 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0539999999999997e-06 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155157+0000", 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00033333700000000001 2023-06-10T21:43:03.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155490+0000", 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2359000000000001e-05 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.155512+0000", 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000444058 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.228 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:107035 13.2 e392/391 13:40898f32:::benchmark_data_smithi064_501823_object107034:head v 392'6726, mlcod=383'4809)", 2023-06-10T21:43:03.228 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:03.156073+0000", 2023-06-10T21:43:03.228 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0037270509999999999, 2023-06-10T21:43:03.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042181429999999997, 2023-06-10T21:43:03.229 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:03.229 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:03.229 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:03.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:03.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156073+0000", 2023-06-10T21:43:03.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:03.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:03.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156073+0000", 2023-06-10T21:43:03.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999933 2023-06-10T21:43:03.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156066+0000", 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5661000000000004e-05 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:03.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156162+0000", 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1012999999999999e-05 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156173+0000", 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0020000000000003e-06 2023-06-10T21:43:03.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156179+0000", 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6708e-05 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156235+0000", 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0162000000000001e-05 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:03.156256+0000", 2023-06-10T21:43:03.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018293699999999999 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 8 2023-06-10T21:43:03.236 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:43:03.431 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 107267 107251 77.9091 17.625 0.0176207 0.0128304 2023-06-10T21:43:03.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:03 smithi152 ceph-mon[156819]: failed to encode map e392 with expected crc 2023-06-10T21:43:03.566 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:03.566 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[180297]: failed to encode map e392 with expected crc 2023-06-10T21:43:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[186706]: failed to encode map e392 with expected crc 2023-06-10T21:43:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:04.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:04.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:04.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:04.340 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:04.428 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 108066 108050 77.5876 49.9375 0.0293858 0.0128845 2023-06-10T21:43:04.496 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:04.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:43:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:04.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:04.586 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:04 smithi152 ceph-mon[156819]: pgmap v1443: 89 pgs: 10 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 38 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 22 MiB/s wr, 359 op/s; 68792/315279 objects degraded (21.819%); 14 KiB/s, 0 objects/s recovering 2023-06-10T21:43:04.617 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:04.675 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:04.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:04.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:04.778 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:04.825 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:04.877 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:04.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:04 smithi064 ceph-mon[186706]: pgmap v1443: 89 pgs: 10 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 38 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 22 MiB/s wr, 359 op/s; 68792/315279 objects degraded (21.819%); 14 KiB/s, 0 objects/s recovering 2023-06-10T21:43:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:04 smithi064 ceph-mon[180297]: pgmap v1443: 89 pgs: 10 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 20 active+undersized+degraded, 19 active+undersized, 38 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 22 MiB/s wr, 359 op/s; 68792/315279 objects degraded (21.819%); 14 KiB/s, 0 objects/s recovering 2023-06-10T21:43:04.924 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:04.950 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:04.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:04.974 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:05.010 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.042 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.043 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.043 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:43:05.051 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:43:05.051 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:43:05.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:05.119 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:05.227 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:05.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:05.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:05.340 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:05.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:05.428 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 108842 108826 77.2571 48.5 0.0090527 0.0129397 2023-06-10T21:43:05.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3443592322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3443592322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:05.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3443592322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:06.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:43:06.264 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:06.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:06.376 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:06.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 109610 109594 76.9285 48 0.0178951 0.0129946 2023-06-10T21:43:06.443 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:06.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:06 smithi064 ceph-mon[180297]: pgmap v1444: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s; 26675/325461 objects degraded (8.196%); 1.6 MiB/s, 21 objects/s recovering 2023-06-10T21:43:06.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 68792/315279 objects degraded (21.819%), 22 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:06.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:06 smithi064 ceph-mon[186706]: pgmap v1444: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s; 26675/325461 objects degraded (8.196%); 1.6 MiB/s, 21 objects/s recovering 2023-06-10T21:43:06.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 68792/315279 objects degraded (21.819%), 22 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:06.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:06 smithi152 ceph-mon[156819]: pgmap v1444: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 53 MiB/s wr, 843 op/s; 26675/325461 objects degraded (8.196%); 1.6 MiB/s, 21 objects/s recovering 2023-06-10T21:43:06.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 68792/315279 objects degraded (21.819%), 22 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:07.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:07.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:07.385 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:07.421 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.0201 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 109999 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 76.3711 2023-06-10T21:43:07.429 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 37.1906 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 199.625 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1221 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 595.05 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3194 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-06-10T21:43:07.430 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0130895 2023-06-10T21:43:07.431 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0333698 2023-06-10T21:43:07.431 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.11513 2023-06-10T21:43:07.431 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.0011044 2023-06-10T21:43:07.434 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:43:07.436 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:07.436 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:43:07.549 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:07.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:07.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:07 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:43:07.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:07 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:43:07.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:07.657 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:07.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:07 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:43:07.794 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:07.893 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:07.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:08.003 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:08.084 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:08.428 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:08.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:08.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:08.532 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:08.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:08.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:08 smithi152 ceph-mon[156819]: pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s; 26669/326592 objects degraded (8.166%); 1.3 MiB/s, 18 objects/s recovering 2023-06-10T21:43:08.903 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:08 smithi064 ceph-mon[180297]: pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s; 26669/326592 objects degraded (8.166%); 1.3 MiB/s, 18 objects/s recovering 2023-06-10T21:43:08.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:08 smithi064 ceph-mon[186706]: pgmap v1445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 45 MiB/s wr, 714 op/s; 26669/326592 objects degraded (8.166%); 1.3 MiB/s, 18 objects/s recovering 2023-06-10T21:43:09.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:43:09.827 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:09.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:09.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:09.993 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:10.273 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:10 smithi152 ceph-mon[156819]: pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 34 MiB/s wr, 585 op/s; 26669/327345 objects degraded (8.147%); 1.1 MiB/s, 15 objects/s recovering 2023-06-10T21:43:10.292 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:10.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:10.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:10.404 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:10.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:10 smithi064 ceph-mon[186706]: pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 34 MiB/s wr, 585 op/s; 26669/327345 objects degraded (8.147%); 1.1 MiB/s, 15 objects/s recovering 2023-06-10T21:43:10.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:10 smithi064 ceph-mon[180297]: pgmap v1446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 114 B/s rd, 34 MiB/s wr, 585 op/s; 26669/327345 objects degraded (8.147%); 1.1 MiB/s, 15 objects/s recovering 2023-06-10T21:43:10.422 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:10.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:10.721 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:10.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:10.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:10.901 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:11.012 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:11.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:11.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:11.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:11.194 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:11.263 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:11.303 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:11.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:11.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:11.412 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:11.504 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:12.155 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 26669/327345 objects degraded (8.147%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2631939071' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3766505527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.165 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:12.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:12.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:12.215 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:43:12.219 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:43:12.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:12.316 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 26669/327345 objects degraded (8.147%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2631939071' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3766505527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.431 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:12.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 26669/327345 objects degraded (8.147%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:12.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2631939071' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:43:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3766505527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:12.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:12.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:12.575 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:12.618 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:12.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:12.723 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:12.787 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:12.897 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:12.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:13.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:13.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:12] "GET /metrics HTTP/1.1" 200 223799 "" "Prometheus/2.33.4" 2023-06-10T21:43:13.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 33 MiB/s wr, 719 op/s; 26378/324417 objects degraded (8.131%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:13.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:13.174 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.175 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:43:13.175 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:13.175 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: osdmap e393: 8 total, 8 up, 4 in 2023-06-10T21:43:13.175 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:13 smithi152 ceph-mon[156819]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:13.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:13.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:13.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 33 MiB/s wr, 719 op/s; 26378/324417 objects degraded (8.131%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:13.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:13.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.486 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: osdmap e393: 8 total, 8 up, 4 in 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[180297]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: pgmap v1447: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 33 MiB/s wr, 719 op/s; 26378/324417 objects degraded (8.131%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:13.487 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.488 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:43:13.488 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:13.488 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: osdmap e393: 8 total, 8 up, 4 in 2023-06-10T21:43:13.488 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:13 smithi064 ceph-mon[186706]: failed to encode map e393 with expected crc 2023-06-10T21:43:13.586 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:13.658 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:13.770 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:13.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:13.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:13.967 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:14.093 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:14.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:14.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:14.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:14.248 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:14 smithi152 ceph-mon[156819]: failed to encode map e393 with expected crc 2023-06-10T21:43:14.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:14.338 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:14.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:14.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:14.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:14.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:14 smithi064 ceph-mon[180297]: failed to encode map e393 with expected crc 2023-06-10T21:43:14.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:14 smithi064 ceph-mon[186706]: failed to encode map e393 with expected crc 2023-06-10T21:43:14.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:15.438 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:15.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:15.453 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:15 smithi064 ceph-mon[180297]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 31 MiB/s wr, 730 op/s; 26378/322941 objects degraded (8.168%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:15.454 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:15 smithi064 ceph-mon[186706]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 31 MiB/s wr, 730 op/s; 26378/322941 objects degraded (8.168%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:15.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:43:15.554 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:15.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:15 smithi152 ceph-mon[156819]: pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.6 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 31 MiB/s wr, 730 op/s; 26378/322941 objects degraded (8.168%); 2.8 MiB/s, 42 objects/s recovering 2023-06-10T21:43:15.604 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:15.676 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:15.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:15.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:15.789 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:15.894 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:16.062 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:16.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:16.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:16.098 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:43:16.098 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:43:16.182 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:16.224 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:16.246 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:16 smithi064 ceph-mon[186706]: pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 9.6 MiB/s wr, 741 op/s; 26056/312426 objects degraded (8.340%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:43:16.247 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 26378/322941 objects degraded (8.168%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:16.308 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:16.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:16.370 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:16.399 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:16.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:16.429 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:16.473 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:16.477 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:16.511 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:16.550 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:16 smithi064 ceph-mon[180297]: pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 9.6 MiB/s wr, 741 op/s; 26056/312426 objects degraded (8.340%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:43:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 26378/322941 objects degraded (8.168%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:16.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:16 smithi152 ceph-mon[156819]: pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.4 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 9.6 MiB/s wr, 741 op/s; 26056/312426 objects degraded (8.340%); 3.9 MiB/s, 61 objects/s recovering 2023-06-10T21:43:16.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 26378/322941 objects degraded (8.168%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:16.813 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:16.813 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.814 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:43:16.815 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:43:17.266 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:17.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:17.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:17.440 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:17.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:43:17.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2153196669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:17.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2153196669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:17.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:43:17.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2153196669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:18.169 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:18.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:18.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:18.307 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:18.395 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:18.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:18.466 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:18.517 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:18.537 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:18.558 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:18.661 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:18.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:18.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:18.819 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:18.849 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:18.931 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:18.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:18.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:19.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:19.082 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:19.192 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:19.192 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:19 smithi152 ceph-mon[156819]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 754 op/s; 26056/310857 objects degraded (8.382%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:19.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:19.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:19.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:19.296 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[180297]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 754 op/s; 26056/310857 objects degraded (8.382%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:19.296 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:19.296 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:19.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[186706]: pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 7.2 MiB/s wr, 754 op/s; 26056/310857 objects degraded (8.382%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:19.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:19.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:19.300 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:19.350 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:19.462 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:19.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:19.570 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:19.646 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:20.328 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:43:20.819 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:20.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:20.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:20.934 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:20.987 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:21.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:21 smithi064 ceph-mon[180297]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 3.0 MiB/s wr, 765 op/s; 26056/307263 objects degraded (8.480%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:21.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 26056/307263 objects degraded (8.480%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:21.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:21 smithi064 ceph-mon[186706]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 3.0 MiB/s wr, 765 op/s; 26056/307263 objects degraded (8.480%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:21.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 26056/307263 objects degraded (8.480%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:21.669 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:21.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:21.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:21 smithi152 ceph-mon[156819]: pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.3 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 3.0 MiB/s wr, 765 op/s; 26056/307263 objects degraded (8.480%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:43:21.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 26056/307263 objects degraded (8.480%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:21.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:21.830 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:22.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:22.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:22.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:22.842 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:22.941 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:22.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:22.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:22.969 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:22] "GET /metrics HTTP/1.1" 200 224029 "" "Prometheus/2.33.4" 2023-06-10T21:43:23.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:23.126 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:23.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:23 smithi064 ceph-mon[180297]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 25726/299919 objects degraded (8.578%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:43:23.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:23 smithi064 ceph-mon[186706]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 25726/299919 objects degraded (8.578%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:43:23.334 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:23.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:23.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:23.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:23.462 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:23.493 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:23 smithi152 ceph-mon[156819]: pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 25726/299919 objects degraded (8.578%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:43:23.502 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:23.566 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:23.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:23.674 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:23.733 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104494 12.9 12:9ca311c5:::benchmark_data_smithi064_442287_object16444:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:43:23.734 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.54749935900003, 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6361784579999998, 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.735 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104494 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177995+0000", 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178006+0000", 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1514e-05 2023-06-10T21:43:23.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178024+0000", 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7651000000000001e-05 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178035+0000", 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0857e-05 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.188934+0000", 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010898873 2023-06-10T21:43:23.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189075+0000", 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014062699999999999 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:43:23.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189180+0000", 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105855 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.196756+0000", 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075760269999999999 2023-06-10T21:43:23.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207957+0000", 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011200676999999999 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245880+0000", 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037923327999999999 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245895+0000", 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4575e-05 2023-06-10T21:43:23.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200093+0000", 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95419813399999998 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200108+0000", 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5024e-05 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754247+0000", 2023-06-10T21:43:23.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5541385060000001 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754441+0000", 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019410900000000001 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814067+0000", 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059626237999999998 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814176+0000", 2023-06-10T21:43:23.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010860400000000001 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104503 12.a 12:53f91a42:::benchmark_data_smithi064_442287_object16453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:43:23.749 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.519598917, 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.527545849, 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.750 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104503 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205896+0000", 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205904+0000", 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9699999999999999e-06 2023-06-10T21:43:23.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205916+0000", 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2194999999999999e-05 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205924+0000", 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218750+0000", 2023-06-10T21:43:23.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012825721999999999 2023-06-10T21:43:23.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218889+0000", 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138617 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.219004+0000", 2023-06-10T21:43:23.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011551 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.224321+0000", 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053164609999999998 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246026+0000", 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021705486 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246036+0000", 2023-06-10T21:43:23.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0294e-05 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202839+0000", 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95680261799999999 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202852+0000", 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459999999999999e-05 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636477+0000", 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.433625159 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636649+0000", 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017257199999999999 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733342+0000", 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096692527 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733443+0000", 2023-06-10T21:43:23.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101641 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104492 12.8 12:145682b0:::benchmark_data_smithi064_442287_object16442:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:43:23.764 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.49480347899998, 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1766732329999998, 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.765 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104492 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230691+0000", 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230699+0000", 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.237985+0000", 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072853450000000004 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.238000+0000", 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5784999999999999e-05 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258465+0000", 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020465041999999999 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258473+0000", 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9489999999999996e-06 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257243+0000", 2023-06-10T21:43:23.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99876928499999995 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257255+0000", 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2431e-05 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.405789+0000", 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.148533523 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407366+0000", 2023-06-10T21:43:23.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577572 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104510 12.9 12:91eab998:::benchmark_data_smithi064_442287_object16460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:43:23.774 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.46072297799998, 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5536998, 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104510 2023-06-10T21:43:23.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264772+0000", 2023-06-10T21:43:23.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264780+0000", 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264792+0000", 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1987999999999999e-05 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264800+0000", 2023-06-10T21:43:23.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8189999999999998e-06 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264846+0000", 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5782000000000003e-05 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264860+0000", 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4267000000000001e-05 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200139+0000", 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93527848099999999 2023-06-10T21:43:23.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200148+0000", 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758081+0000", 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5579332859999999 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758234+0000", 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015339999999999999 2023-06-10T21:43:23.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758918+0000", 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068356899999999997 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.784273+0000", 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025354811000000001 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.785804+0000", 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00153134 2023-06-10T21:43:23.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814218+0000", 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028414053000000002 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814269+0000", 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.075e-05 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818473+0000", 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042044639999999998 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.787 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104512 12.8 12:1b4cbe21:::benchmark_data_smithi064_442287_object16462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.45415389800002, 2023-06-10T21:43:23.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1733947009999999, 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104512 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.789 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271341+0000", 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271350+0000", 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8789999999999998e-06 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271394+0000", 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.384e-05 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271404+0000", 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6309999999999998e-06 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271472+0000", 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7986999999999998e-05 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271488+0000", 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5911000000000001e-05 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257279+0000", 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98579087099999996 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257287+0000", 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9410000000000007e-06 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407591+0000", 2023-06-10T21:43:23.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15030481 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408064+0000", 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047220399999999999 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408146+0000", 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2446000000000003e-05 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420391+0000", 2023-06-10T21:43:23.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012245285999999999 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423446+0000", 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030550500000000001 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429481+0000", 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060345260000000001 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444541+0000", 2023-06-10T21:43:23.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015059771 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444604+0000", 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3081000000000001e-05 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444737+0000", 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133468 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.802 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104514 12.8 12:1ddd6575:::benchmark_data_smithi064_442287_object16464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 302.44659033300002, 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1660305000000002, 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.803 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104514 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:43:23.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278905+0000", 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278910+0000", 2023-06-10T21:43:23.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278921+0000", 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278927+0000", 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5470000000000004e-06 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278994+0000", 2023-06-10T21:43:23.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6595999999999997e-05 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.279006+0000", 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2479e-05 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257307+0000", 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97830109899999995 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257315+0000", 2023-06-10T21:43:23.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3119999999999994e-06 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410688+0000", 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15337278 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410832+0000", 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014341000000000001 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410967+0000", 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135816 2023-06-10T21:43:23.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420402+0000", 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094344049999999999 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423489+0000", 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030871060000000001 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429518+0000", 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060294930000000004 2023-06-10T21:43:23.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444782+0000", 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015263465 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444832+0000", 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9935999999999998e-05 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444937+0000", 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010481599999999999 2023-06-10T21:43:23.816 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117672 12.10 12:08d95f39:::benchmark_data_smithi064_442287_object29622:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.51430220499998, 2023-06-10T21:43:23.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9927129360000002, 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117672 2023-06-10T21:43:23.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:43:23.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211197+0000", 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8619999999999999e-06 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211203+0000", 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0959999999999997e-06 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211208+0000", 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.412e-06 2023-06-10T21:43:23.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560435+0000", 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922703799999999 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560446+0000", 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0944999999999999e-05 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075800+0000", 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515354077 2023-06-10T21:43:23.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075806+0000", 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0939999999999996e-06 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203794+0000", 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1279870999999999 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203907+0000", 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113702 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.826 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117682 12.18 12:18fc4302:::benchmark_data_smithi064_442287_object29632:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.514247205, 2023-06-10T21:43:23.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.083346562, 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117682 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.828 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211252+0000", 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7800000000000001e-06 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211258+0000", 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.404e-06 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211263+0000", 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1460000000000004e-06 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560489+0000", 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922595299999998 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560495+0000", 2023-06-10T21:43:23.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2299999999999996e-06 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075893+0000", 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515397806 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075905+0000", 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1467e-05 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183546+0000", 2023-06-10T21:43:23.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1076415719999999 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183688+0000", 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141898 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183979+0000", 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029059499999999999 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.186643+0000", 2023-06-10T21:43:23.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026644519999999999 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.231646+0000", 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045002563000000002 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294378+0000", 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062732531999999994 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294444+0000", 2023-06-10T21:43:23.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5165000000000004e-05 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294496+0000", 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2339999999999997e-05 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294596+0000", 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9959999999999998e-05 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.841 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117683 12.10 12:0c4ca780:::benchmark_data_smithi064_442287_object29633:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.51422831500003, 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.12118121, 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.842 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117683 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:43:23.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211270+0000", 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2350000000000002e-06 2023-06-10T21:43:23.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211276+0000", 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.587e-06 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211281+0000", 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2000000000000002e-06 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560465+0000", 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34918397899999998 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560471+0000", 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7729999999999996e-06 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075818+0000", 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5153477930000001 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075823+0000", 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8350000000000003e-06 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294814+0000", 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218991033 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.295436+0000", 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062166299999999995 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313438+0000", 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018001952000000002 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313553+0000", 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115364 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313880+0000", 2023-06-10T21:43:23.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032708500000000002 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.322782+0000", 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089019919999999992 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331101+0000", 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083185140000000008 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331137+0000", 2023-06-10T21:43:23.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6034000000000001e-05 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331263+0000", 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012637400000000001 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331298+0000", 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5088e-05 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.332449+0000", 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151032 2023-06-10T21:43:23.856 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117693 12.19 12:9f069f6e:::benchmark_data_smithi064_442287_object29643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 259.05780849899998, 2023-06-10T21:43:23.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5931760800000001, 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117693 2023-06-10T21:43:23.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667686+0000", 2023-06-10T21:43:23.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667699+0000", 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2989e-05 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667723+0000", 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3802e-05 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667736+0000", 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.375e-05 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667797+0000", 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0807000000000002e-05 2023-06-10T21:43:23.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667811+0000", 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3402e-05 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075633+0000", 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4078227939999999 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075643+0000", 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5529999999999998e-06 2023-06-10T21:43:23.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205257+0000", 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1296138060000001 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205964+0000", 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070749699999999997 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253657+0000", 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047692997000000001 2023-06-10T21:43:23.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253748+0000", 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0437000000000002e-05 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5", 2023-06-10T21:43:23.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253982+0000", 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023403700000000001 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.258419+0000", 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044375309999999998 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259238+0000", 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081873500000000001 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259276+0000", 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8238999999999998e-05 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259376+0000", 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9351000000000005e-05 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259436+0000", 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0044000000000003e-05 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.260864+0000", 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014284499999999999 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136391 12.a 12:50cbddcd:::benchmark_data_smithi064_442287_object48341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:43:23.873 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.54803144900001, 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.351712574, 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.874 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136391 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177464+0000", 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177470+0000", 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8900000000000004e-06 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177481+0000", 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1082e-05 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177488+0000", 2023-06-10T21:43:23.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.605e-06 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177545+0000", 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6372999999999997e-05 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.182867+0000", 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053217430000000003 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210800+0000", 2023-06-10T21:43:23.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027933428 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210903+0000", 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103509 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.211048+0000", 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144902 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.215466+0000", 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044176579999999997 2023-06-10T21:43:23.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221482+0000", 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060159760000000001 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257726+0000", 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036244365000000001 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257739+0000", 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2807e-05 2023-06-10T21:43:23.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.221998+0000", 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96425910500000001 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222010+0000", 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1863e-05 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454898+0000", 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2328882999999999 2023-06-10T21:43:23.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454985+0000", 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6615999999999999e-05 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529107+0000", 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074122399000000005 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529178+0000", 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0131999999999999e-05 2023-06-10T21:43:23.888 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136405 12.a 12:54ed30e6:::benchmark_data_smithi064_442287_object48355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.508909907, 2023-06-10T21:43:23.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3845864269999999, 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136405 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.890 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216586+0000", 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216589+0000", 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5150000000000001e-06 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216594+0000", 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1039999999999998e-06 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216598+0000", 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4460000000000003e-06 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221515+0000", 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049165859999999997 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222958+0000", 2023-06-10T21:43:23.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014432430000000001 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261169+0000", 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038210604000000002 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261175+0000", 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2589999999999997e-06 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222103+0000", 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092808100000005 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222109+0000", 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3199999999999996e-06 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529563+0000", 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307454071 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529785+0000", 2023-06-10T21:43:23.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022143400000000001 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549764+0000", 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01997898 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549868+0000", 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103929 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.550267+0000", 2023-06-10T21:43:23.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000398811 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.568050+0000", 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017783812999999999 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575392+0000", 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007341106 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575441+0000", 2023-06-10T21:43:23.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9833000000000002e-05 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583056+0000", 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076141569999999999 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598427+0000", 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015371246 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598489+0000", 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2738999999999997e-05 2023-06-10T21:43:23.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.601173+0000", 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002683506 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.906 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136406 12.a 12:5c33d118:::benchmark_data_smithi064_442287_object48356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.50888901299999, 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4148715210000002, 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.907 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136406 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216609+0000", 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1780000000000002e-06 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216614+0000", 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.549e-06 2023-06-10T21:43:23.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216618+0000", 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222999+0000", 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063813000000000003 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224213+0000", 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001213312 2023-06-10T21:43:23.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261137+0000", 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036923926000000003 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261146+0000", 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2450000000000004e-06 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222072+0000", 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092633900000002 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222078+0000", 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0990000000000004e-06 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529195+0000", 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307116508 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529519+0000", 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032444499999999997 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583294+0000", 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.053774523999999997 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583424+0000", 2023-06-10T21:43:23.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130443 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583605+0000", 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018088100000000001 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.588407+0000", 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048019949999999999 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593807+0000", 2023-06-10T21:43:23.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053999369999999996 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593843+0000", 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5809999999999998e-05 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.597698+0000", 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038552539999999998 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629308+0000", 2023-06-10T21:43:23.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031610232000000002 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629379+0000", 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0947999999999999e-05 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.631479+0000", 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020995969999999999 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.924 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136411 12.a 12:52d92197:::benchmark_data_smithi064_442287_object48361:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 214.50121781300001, 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.391707663, 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.925 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136411 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:43:23.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224277+0000", 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224285+0000", 2023-06-10T21:43:23.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9610000000000005e-06 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224297+0000", 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.153e-05 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224306+0000", 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6539999999999994e-06 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224396+0000", 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9741999999999996e-05 2023-06-10T21:43:23.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224406+0000", 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6120000000000005e-06 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261193+0000", 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036787411999999999 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261198+0000", 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2730000000000002e-06 2023-06-10T21:43:23.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222130+0000", 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96093205800000003 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222135+0000", 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6940000000000003e-06 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529810+0000", 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307674896 2023-06-10T21:43:23.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.530375+0000", 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056451500000000003 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565747+0000", 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.035372210000000001 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565873+0000", 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125976 2023-06-10T21:43:23.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.566075+0000", 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000202573 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582894+0000", 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016818204999999999 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582953+0000", 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9228000000000003e-05 2023-06-10T21:43:23.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583006+0000", 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3650000000000003e-05 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583198+0000", 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019192600000000001 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613246+0000", 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030047554000000001 2023-06-10T21:43:23.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613310+0000", 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3886999999999993e-05 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.615986+0000", 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026765270000000002 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156289 12.19 12:9ee93671:::benchmark_data_smithi064_442287_object68239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:43:23.942 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.53002133300001, 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2463835950000002, 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.943 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156289 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195473+0000", 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195481+0000", 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8739999999999995e-06 2023-06-10T21:43:23.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195499+0000", 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7822e-05 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195507+0000", 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4540000000000007e-06 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.197882+0000", 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023747109999999998 2023-06-10T21:43:23.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.199076+0000", 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011941499999999999 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212744+0000", 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013667643 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212868+0000", 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124313 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.213164+0000", 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029563400000000002 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231891+0000", 2023-06-10T21:43:23.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018727414000000001 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258125+0000", 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026233570000000001 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258140+0000", 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5158e-05 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211376+0000", 2023-06-10T21:43:23.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95323585399999999 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211388+0000", 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2655e-05 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382314+0000", 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1709252590000001 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382471+0000", 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157326 2023-06-10T21:43:23.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441738+0000", 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059266970000000002 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441859+0000", 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120934 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.957 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156297 12.19 12:9878c605:::benchmark_data_smithi064_442287_object68247:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.514751475, 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2780029549999998, 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.958 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156297 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:43:23.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210743+0000", 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210751+0000", 2023-06-10T21:43:23.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8199999999999997e-06 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210765+0000", 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3745e-05 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210773+0000", 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1030000000000002e-06 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.230620+0000", 2023-06-10T21:43:23.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019846534999999998 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231381+0000", 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076190000000000003 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260069+0000", 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028687363 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260080+0000", 2023-06-10T21:43:23.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0786e-05 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211426+0000", 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134606499999996 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211436+0000", 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0145e-05 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441910+0000", 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2304743680000001 2023-06-10T21:43:23.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.442245+0000", 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000334514 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454189+0000", 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011943949000000001 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454304+0000", 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115359 2023-06-10T21:43:23.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454674+0000", 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000370176 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459572+0000", 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048976749999999998 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459582+0000", 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105999999999999e-05 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464585+0000", 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050026599999999999 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487733+0000", 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023147893999999999 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487794+0000", 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1424e-05 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488748+0000", 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095399799999999996 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156298 12.19 12:98e24496:::benchmark_data_smithi064_442287_object68248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 174.51471297099999, 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2788083060000002, 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:23.975 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156298 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210783+0000", 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210786+0000", 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5849999999999998e-06 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210792+0000", 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9750000000000004e-06 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210797+0000", 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9509999999999997e-06 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260110+0000", 2023-06-10T21:43:23.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049313312999999998 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260119+0000", 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9889999999999998e-06 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211462+0000", 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134341099999997 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211471+0000", 2023-06-10T21:43:23.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7490000000000004e-06 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.446320+0000", 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234848945 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.447323+0000", 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001003122 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457450+0000", 2023-06-10T21:43:23.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010126634000000001 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457569+0000", 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119261 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457915+0000", 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034610199999999999 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459652+0000", 2023-06-10T21:43:23.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001736502 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464637+0000", 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049851749999999997 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.467552+0000", 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002914954 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488815+0000", 2023-06-10T21:43:23.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021263659000000001 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488870+0000", 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4682000000000003e-05 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.489592+0000", 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072164799999999997 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.990 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106688 13.4 13:21a54017:::benchmark_data_smithi064_501823_object106687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.202975475999999, 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1085864490000001, 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:23.991 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106688 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:43:23.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522548+0000", 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7114000000000002e-05 2023-06-10T21:43:23.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522555+0000", 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5259999999999996e-06 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:23.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522561+0000", 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7930000000000003e-06 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.528701+0000", 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061403400000000002 2023-06-10T21:43:23.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.528877+0000", 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017563999999999999 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3", 2023-06-10T21:43:23.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.529052+0000", 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000174902 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.540055+0000", 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011003051999999999 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545371+0000", 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053163510000000004 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545384+0000", 2023-06-10T21:43:23.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266000000000001e-05 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.195964+0000", 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058062100000003 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.195974+0000", 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9119999999999993e-06 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615197+0000", 2023-06-10T21:43:24.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4192233459999999 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615857+0000", 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065989 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.630989+0000", 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015131350999999999 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631107+0000", 2023-06-10T21:43:24.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118786 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106692 13.4 13:277bf932:::benchmark_data_smithi064_501823_object106691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:43:24.004 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.202831464999999, 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1091667009999999, 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:43:24.005 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106692 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:43:24.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522689+0000", 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4600000000000002e-05 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522697+0000", 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7389999999999999e-06 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522703+0000", 2023-06-10T21:43:24.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6459999999999998e-06 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545434+0000", 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022731411 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545443+0000", 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3910000000000004e-06 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196033+0000", 2023-06-10T21:43:24.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058933500000005 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196043+0000", 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0361e-05 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615912+0000", 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4198693060000001 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.616007+0000", 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5063999999999996e-05 2023-06-10T21:43:24.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.616124+0000", 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117063 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:24.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.628240+0000", 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012115310000000001 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.629520+0000", 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001279896 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631158+0000", 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016387319999999999 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:24.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631224+0000", 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5912000000000002e-05 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631832+0000", 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060740999999999996 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.017 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106701 13.4 13:2a3b2934:::benchmark_data_smithi064_501823_object106700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.188037418, 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0991637939999999, 2023-06-10T21:43:24.018 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106701 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:43:24.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:43:24.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537458+0000", 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537484+0000", 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5945e-05 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537492+0000", 2023-06-10T21:43:24.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9789999999999993e-06 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537499+0000", 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2550000000000003e-06 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545468+0000", 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079695500000000006 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545477+0000", 2023-06-10T21:43:24.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6009999999999995e-06 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196065+0000", 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058839099999999 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196073+0000", 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7109999999999995e-06 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618573+0000", 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.422500458 2023-06-10T21:43:24.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618724+0000", 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015112 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:43:24.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618864+0000", 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139529 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631882+0000", 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013018237 2023-06-10T21:43:24.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.632220+0000", 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033821300000000002 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.635418+0000", 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031970430000000001 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.635476+0000", 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8950000000000003e-05 2023-06-10T21:43:24.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.636623+0000", 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001146389 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:24.031 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.032 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:43:24.032 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:43:24.032 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:43:24.032 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:24.067 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:24.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:24.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:24.261 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:43:24.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:24.292 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:43:24.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/84667488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:24.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:24.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:24.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/84667488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:24.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:24.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/84667488' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:24 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:43:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:24.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 25726/298362 objects degraded (8.622%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:43:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:43:25.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: osdmap e394: 8 total, 8 up, 4 in 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: failed to encode map e394 with expected crc 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[186706]: failed to encode map e394 with expected crc 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 25726/298362 objects degraded (8.622%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:25.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:43:25.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:25.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: osdmap e394: 8 total, 8 up, 4 in 2023-06-10T21:43:25.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: failed to encode map e394 with expected crc 2023-06-10T21:43:25.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:25 smithi064 ceph-mon[180297]: failed to encode map e394 with expected crc 2023-06-10T21:43:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: pgmap v1454: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 6.1 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 25726/298362 objects degraded (8.622%); 3.4 MiB/s, 54 objects/s recovering 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/911940412' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: osdmap e394: 8 total, 8 up, 4 in 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: failed to encode map e394 with expected crc 2023-06-10T21:43:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:25 smithi152 ceph-mon[156819]: failed to encode map e394 with expected crc 2023-06-10T21:43:26.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:26.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:26.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: failed to encode map e394 with expected crc 2023-06-10T21:43:26.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:43:26.375 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:43:26.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:26.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 25352/287283 objects degraded (8.825%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:26.376 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25726/298362 objects degraded (8.622%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: failed to encode map e394 with expected crc 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 25352/287283 objects degraded (8.825%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25726/298362 objects degraded (8.622%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:26.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:26.467 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:26.563 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:26.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:26.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:26.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: failed to encode map e394 with expected crc 2023-06-10T21:43:26.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:43:26.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:43:26.580 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:26.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 25352/287283 objects degraded (8.825%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:26.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25726/298362 objects degraded (8.622%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:26.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:26.724 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:27.688 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:27.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:27.731 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:43:27.801 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:27.853 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:27.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:43:28.007 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:28.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:28.116 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:28.198 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:28.318 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:28.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:28.416 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:28.425 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:28.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:28.566 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:28.599 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:28.643 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:28.653 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:28 smithi064 ceph-mon[186706]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 839 op/s; 25352/285663 objects degraded (8.875%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:28.664 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:28 smithi064 ceph-mon[180297]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 839 op/s; 25352/285663 objects degraded (8.875%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:28.692 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:28 smithi152 ceph-mon[156819]: pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.8 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 839 op/s; 25352/285663 objects degraded (8.875%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:28.731 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:28.768 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:28.796 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:28.804 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:28.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:28.834 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.839 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.840 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.840 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:43:28.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:28.978 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:29.107 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:43:29.277 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:29.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:29.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:29.455 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:29.549 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:29.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1777497625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:29.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:29.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1777497625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1777497625' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:29.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:29.717 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:30 smithi064 ceph-mon[186706]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 852 op/s; 25352/281694 objects degraded (9.000%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:30.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:30 smithi064 ceph-mon[180297]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 852 op/s; 25352/281694 objects degraded (9.000%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:30.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:30 smithi152 ceph-mon[156819]: pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.7 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 852 op/s; 25352/281694 objects degraded (9.000%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:43:30.780 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:30.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:30.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:30.954 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:31.054 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:31.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:31.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:31.079 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:31.171 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:31.234 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:31.317 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:31.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:31.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:31.421 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:31.488 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25352/281694 objects degraded (9.000%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:31.489 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25352/281694 objects degraded (9.000%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:31.493 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:31.623 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:31.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25352/281694 objects degraded (9.000%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:31.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:31.800 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:32.023 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:32.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:32.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:32.183 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:32.253 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:32.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:32.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:43:32.302 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:32 smithi064 ceph-mon[180297]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 25010/275280 objects degraded (9.085%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:43:32.303 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:43:32.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:32.386 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:32.431 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:32.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:43:32.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:32 smithi152 ceph-mon[156819]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 25010/275280 objects degraded (9.085%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:43:32.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:32.592 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:32.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:43:32.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:32 smithi064 ceph-mon[186706]: pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 25010/275280 objects degraded (9.085%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:43:32.802 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:32.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:32.875 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:43:32.907 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:32.947 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:33.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:32] "GET /metrics HTTP/1.1" 200 224029 "" "Prometheus/2.33.4" 2023-06-10T21:43:33.392 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:43:33.392 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:43:33.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:33 smithi152 ceph-mon[156819]: failed to encode map e395 with expected crc 2023-06-10T21:43:33.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:43:33.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:33 smithi152 ceph-mon[156819]: osdmap e395: 8 total, 8 up, 4 in 2023-06-10T21:43:33.427 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:33.468 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:33.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[180297]: failed to encode map e395 with expected crc 2023-06-10T21:43:33.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:43:33.473 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[180297]: osdmap e395: 8 total, 8 up, 4 in 2023-06-10T21:43:33.474 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:33.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:33.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[186706]: failed to encode map e395 with expected crc 2023-06-10T21:43:33.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3327409213' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:43:33.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[186706]: osdmap e395: 8 total, 8 up, 4 in 2023-06-10T21:43:33.475 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:33.575 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:33.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 5 1 2023-06-10T21:43:33.639 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:33.980 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:34.000 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:34.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:34.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:34.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:34.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:34.322 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:34.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:34.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:34.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:34.454 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:34.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[180297]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[180297]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[180297]: osdmap e396: 8 total, 8 up, 4 in 2023-06-10T21:43:34.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[180297]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[180297]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 25010/273738 objects degraded (9.136%); 2.7 MiB/s, 42 objects/s recovering 2023-06-10T21:43:34.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[186706]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[186706]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[186706]: osdmap e396: 8 total, 8 up, 4 in 2023-06-10T21:43:34.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[186706]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:34 smithi064 ceph-mon[186706]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 25010/273738 objects degraded (9.136%); 2.7 MiB/s, 42 objects/s recovering 2023-06-10T21:43:34.548 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:34.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:34.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:34.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:34 smithi152 ceph-mon[156819]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:34 smithi152 ceph-mon[156819]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:34 smithi152 ceph-mon[156819]: osdmap e396: 8 total, 8 up, 4 in 2023-06-10T21:43:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:34 smithi152 ceph-mon[156819]: failed to encode map e396 with expected crc 2023-06-10T21:43:34.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:34 smithi152 ceph-mon[156819]: pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 5.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 25010/273738 objects degraded (9.136%); 2.7 MiB/s, 42 objects/s recovering 2023-06-10T21:43:34.719 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:34.801 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:34.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:34.908 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:34.936 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:34.982 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:34.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:35.000 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:43:35.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:35.180 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:35.339 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:43:35.348 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:35.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:35.468 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:35.524 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:35.609 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[186706]: failed to encode map e396 with expected crc 2023-06-10T21:43:35.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/801015942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.610 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[180297]: failed to encode map e396 with expected crc 2023-06-10T21:43:35.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/801015942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:35 smithi152 ceph-mon[156819]: failed to encode map e396 with expected crc 2023-06-10T21:43:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/801015942' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:43:35.720 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:36.210 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:36.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:36.273 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:36.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:36.399 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:36.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:36.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: osdmap e397: 8 total, 8 up, 4 in 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.505 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 925 op/s; 24662/265038 objects degraded (9.305%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 25010/273738 objects degraded (9.136%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: osdmap e397: 8 total, 8 up, 4 in 2023-06-10T21:43:36.506 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 925 op/s; 24662/265038 objects degraded (9.305%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:43:36.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 25010/273738 objects degraded (9.136%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:36.514 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:36.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:36.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:36.622 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:36.686 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: osdmap e397: 8 total, 8 up, 4 in 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: failed to encode map e397 with expected crc 2023-06-10T21:43:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: pgmap v1464: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 925 op/s; 24662/265038 objects degraded (9.305%); 7.2 MiB/s, 114 objects/s recovering 2023-06-10T21:43:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 25010/273738 objects degraded (9.136%), 14 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:36.857 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:36.881 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:36.884 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:36.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:36.904 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:36.923 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:36.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:36.960 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:36.980 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:36.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:37.017 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:37.061 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:37.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:37.094 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:37.100 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.100 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.101 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:43:37.164 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:37.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:37.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:37.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:37.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:37.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2793230469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:37.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2793230469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:37.439 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:37.678 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2793230469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:38.139 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:43:38.139 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:43:38.139 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:43:38.139 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:43:38.139 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:43:38.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:43:38.450 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:38 smithi152 ceph-mon[156819]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 24662/263811 objects degraded (9.348%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:43:38.472 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:38.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:38.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:38.580 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:38.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:38 smithi064 ceph-mon[180297]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 24662/263811 objects degraded (9.348%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:43:38.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:38 smithi064 ceph-mon[186706]: pgmap v1465: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.4 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 24662/263811 objects degraded (9.348%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:43:38.651 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:38.697 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:38.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:38.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:38.827 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:38.933 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:38.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:39.042 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:39.107 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:39.153 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:39.332 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:39.336 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:43:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:39.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:39.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:39.551 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:39.568 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-06-10T21:43:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/66621990' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:39 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.706 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:39.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:39.821 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:39.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/66621990' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:39 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/66621990' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.861 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:39 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:43:39.894 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:40.038 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:40.154 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:40.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:40.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:40.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:40.294 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:40.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:40.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:40.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:40.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:40.516 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:40.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:43:40.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[180297]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:43:40.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[180297]: osdmap e398: 8 total, 8 up, 4 in 2023-06-10T21:43:40.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[180297]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[180297]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 634 op/s; 24284/260661 objects degraded (9.316%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:43:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[186706]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:43:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[186706]: osdmap e398: 8 total, 8 up, 4 in 2023-06-10T21:43:40.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[186706]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:40 smithi064 ceph-mon[186706]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 634 op/s; 24284/260661 objects degraded (9.316%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:43:40.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:40 smithi152 ceph-mon[156819]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:40 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:43:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:40 smithi152 ceph-mon[156819]: osdmap e398: 8 total, 8 up, 4 in 2023-06-10T21:43:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:40 smithi152 ceph-mon[156819]: failed to encode map e398 with expected crc 2023-06-10T21:43:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:40 smithi152 ceph-mon[156819]: pgmap v1467: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 634 op/s; 24284/260661 objects degraded (9.316%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:43:41.141 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:41.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:41.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:41.265 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:43:41.266 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:43:41.283 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:41.388 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:41.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:41.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:41.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:41.682 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:41.829 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:41.847 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:41 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 24284/260661 objects degraded (9.316%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:41.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:41.914 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:41 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 24284/260661 objects degraded (9.316%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:41.914 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:41 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 24284/260661 objects degraded (9.316%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:41.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:42.000 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:42.219 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:42.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:42.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:42.239 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:42.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:42.391 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:42.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:42.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:42.505 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:112960 11.7 e298/269 11:e5139eac:::benchmark_data_smithi064_385725_object28687:head v 298'7073, mlcod=298'7069)", 2023-06-10T21:43:42.506 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 593.78835700299999, 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010463623999999999, 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.507 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711769+0000", 2023-06-10T21:43:42.508 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711768+0000", 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.305e-06 2023-06-10T21:43:42.509 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711774+0000", 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5260000000000004e-06 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.510 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711778+0000", 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8289999999999999e-06 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.711781+0000", 2023-06-10T21:43:42.511 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026061560000000001 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.714387+0000", 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4856999999999999e-05 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.512 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.714412+0000", 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077718650000000002 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.513 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.722184+0000", 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8520999999999998e-05 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:33:48.722232+0000", 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010463623999999999 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.514 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:141854 11.c e302/269 11:377c3135:::benchmark_data_smithi064_385725_object57581:head v 302'8837, mlcod=302'8834)", 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 537.34722598300004, 2023-06-10T21:43:42.515 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012712886, 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.516 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152900+0000", 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.517 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152899+0000", 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6380000000000003e-06 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152904+0000", 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.19e-06 2023-06-10T21:43:42.518 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152910+0000", 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8850000000000001e-06 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.519 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.152914+0000", 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032875410000000002 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.156202+0000", 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5582000000000001e-05 2023-06-10T21:43:42.520 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.156217+0000", 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0093518679999999993 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.521 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.165569+0000", 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3519999999999997e-05 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:45.165613+0000", 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012712886 2023-06-10T21:43:42.522 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:143254 11.d e302/269 11:b20b535e:::benchmark_data_smithi064_385725_object58981:head v 302'8903, mlcod=302'8900)", 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 534.66771157400001, 2023-06-10T21:43:42.523 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010447279, 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:43:42.524 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.525 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832414+0000", 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.152e-06 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832419+0000", 2023-06-10T21:43:42.526 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9190000000000004e-06 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832425+0000", 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8449999999999999e-06 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.527 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.832428+0000", 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0041750249999999997 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.836603+0000", 2023-06-10T21:43:42.528 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8873e-05 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.836622+0000", 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0061937030000000001 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.529 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.842815+0000", 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6097000000000003e-05 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:47.842861+0000", 2023-06-10T21:43:42.530 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010447279 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:144255 11.c e302/269 11:36e1695b:::benchmark_data_smithi064_385725_object59982:head v 302'8986, mlcod=302'8983)", 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:43:42.531 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 532.48447648700005, 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013049181999999999, 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.532 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.533 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015649+0000", 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.5439999999999999e-06 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.534 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015657+0000", 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1619999999999996e-06 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015664+0000", 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9319999999999998e-06 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.535 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.015667+0000", 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0044353730000000003 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.536 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.020103+0000", 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4808999999999999e-05 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.020128+0000", 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0085162720000000001 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.537 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.028644+0000", 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4398000000000003e-05 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.538 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:34:50.028698+0000", 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013049181999999999 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:151200 11.c e303/269 11:333792e6:::benchmark_data_smithi064_385725_object66927:head v 303'9446, mlcod=303'9444)", 2023-06-10T21:43:42.539 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 518.37520589400003, 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013186708, 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.540 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124920+0000", 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.541 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124919+0000", 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1994000000000001e-05 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.542 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124931+0000", 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1949e-05 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124943+0000", 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8090000000000004e-06 2023-06-10T21:43:42.543 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.124949+0000", 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2096999999999999e-05 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.544 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.125001+0000", 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0669000000000001e-05 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.125022+0000", 2023-06-10T21:43:42.545 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013036641999999999 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138059+0000", 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7722000000000002e-05 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.546 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138106+0000", 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013186708 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.547 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.36159.0:151201 11.c e303/269 11:3f124850:::benchmark_data_smithi064_385725_object66928:head v 303'9447, mlcod=303'9444)", 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 518.37397046399997, 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011978391999999999, 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.548 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.549 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126155+0000", 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126154+0000", 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2140000000000001e-05 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126166+0000", 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4511e-05 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.551 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126181+0000", 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3780000000000003e-06 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.126188+0000", 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048361719999999997 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.552 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.131024+0000", 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9647e-05 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.553 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.131044+0000", 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0070684169999999996 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138113+0000", 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.105e-05 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.554 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:35:04.138134+0000", 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011978391999999999 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.555 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:34272 12.b e319/311 12:d1514c3b:::benchmark_data_smithi064_442287_object34271:head v 319'2049, mlcod=319'2047)", 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:35.353437+0000", 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 427.14668844599998, 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083271381000000005, 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.556 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353437+0000", 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.557 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353437+0000", 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353436+0000", 2023-06-10T21:43:42.558 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4300000000000003e-05 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353521+0000", 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2719999999999997e-06 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.559 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353529+0000", 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3100000000000002e-06 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353533+0000", 2023-06-10T21:43:42.560 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9360000000000003e-05 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353573+0000", 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5044e-05 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.561 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353588+0000", 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083039778999999994 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436627+0000", 2023-06-10T21:43:42.562 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1129000000000002e-05 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436709+0000", 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.083271381000000005 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.563 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:34276 12.2 e319/311 12:4127f7e9:::benchmark_data_smithi064_442287_object34275:head v 319'2187, mlcod=319'2184)", 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:35.353544+0000", 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 427.14658113799999, 2023-06-10T21:43:42.564 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.085524120999999995, 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353544+0000", 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.565 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353544+0000", 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.566 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353544+0000", 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011523699999999999 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353659+0000", 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8510000000000001e-06 2023-06-10T21:43:42.567 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353666+0000", 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.292e-06 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.568 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.353669+0000", 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082983206000000004 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436653+0000", 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7426000000000002e-05 2023-06-10T21:43:42.569 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436710+0000", 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002313631 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.570 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.439024+0000", 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4889999999999999e-05 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.439069+0000", 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.085524120999999995 2023-06-10T21:43:42.571 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:34285 12.f e319/311 12:f72c75ff:::benchmark_data_smithi064_442287_object34284:head v 319'2117, mlcod=319'2115)", 2023-06-10T21:43:42.572 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:35.356560+0000", 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 427.14356559100003, 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082563241999999995, 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.573 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.356560+0000", 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.356560+0000", 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.574 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.356559+0000", 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0437000000000003e-05 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.575 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.356630+0000", 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.079955986000000007 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436586+0000", 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9309e-05 2023-06-10T21:43:42.576 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436615+0000", 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116922 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.577 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436732+0000", 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3884000000000002e-05 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436776+0000", 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023049289999999998 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.578 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.439081+0000", 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2530000000000001e-05 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.579 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.439123+0000", 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082563241999999995 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:34280 12.e e319/311 12:70cffdf2:::benchmark_data_smithi064_442287_object34279:head v 319'2164, mlcod=319'2162)", 2023-06-10T21:43:42.580 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:35.357454+0000", 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 427.14267143900003, 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082754241000000006, 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.581 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.357454+0000", 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.357454+0000", 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.582 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.357453+0000", 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0504000000000001e-05 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.583 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.357484+0000", 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.079165369999999999 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436649+0000", 2023-06-10T21:43:42.584 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.177e-05 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436671+0000", 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012685499999999999 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.585 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436798+0000", 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8130000000000001e-05 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436856+0000", 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0033237430000000001 2023-06-10T21:43:42.586 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.440180+0000", 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8575e-05 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.587 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.440208+0000", 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.082754241000000006 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:34287 12.b e319/311 12:dbf5eb1a:::benchmark_data_smithi064_442287_object34286:head v 319'2050, mlcod=319'2047)", 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:36:35.361364+0000", 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 427.13876206499998, 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.080183119999999997, 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.361364+0000", 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.361364+0000", 2023-06-10T21:43:42.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.361363+0000", 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9023999999999997e-05 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.361412+0000", 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.075197189999999997 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436609+0000", 2023-06-10T21:43:42.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9715999999999999e-05 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436629+0000", 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00023685199999999999 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436865+0000", 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6304999999999998e-05 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.436912+0000", 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0046040630000000003 2023-06-10T21:43:42.594 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.441516+0000", 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0945999999999998e-05 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.595 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:36:35.441547+0000", 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.080183119999999997 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.596 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:78137 12.f e322/311 12:f346fa26:::benchmark_data_smithi064_442287_object78136:head v 322'4914, mlcod=322'4910)", 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:37:23.581799+0000", 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 378.918327068, 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091714080000000007, 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581799+0000", 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581799+0000", 2023-06-10T21:43:42.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581798+0000", 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1872000000000007e-05 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581860+0000", 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5611999999999999e-05 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581875+0000", 2023-06-10T21:43:42.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0340000000000002e-06 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581881+0000", 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6604000000000001e-05 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581948+0000", 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1795999999999998e-05 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581980+0000", 2023-06-10T21:43:42.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0089490739999999996 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.590929+0000", 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1270000000000003e-05 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.603 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.590970+0000", 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0091714080000000007 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:78142 12.f e322/311 12:fb48486b:::benchmark_data_smithi064_442287_object78141:head v 322'4915, mlcod=322'4910)", 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:37:23.581899+0000", 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 378.91822663699998, 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090990680000000001, 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.605 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581899+0000", 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581899+0000", 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581899+0000", 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2888000000000001e-05 2023-06-10T21:43:42.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581931+0000", 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4800000000000007e-06 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581941+0000", 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9380000000000002e-06 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.581945+0000", 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002236418 2023-06-10T21:43:42.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.584181+0000", 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7227e-05 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.584199+0000", 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067793810000000001 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.590978+0000", 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0123e-05 2023-06-10T21:43:42.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:23.590998+0000", 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0090990680000000001 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.612 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46562.0:81167 12.0 e323/311 12:096fa29f:::benchmark_data_smithi064_442287_object81166:head v 323'4996, mlcod=323'4994)", 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:37:27.683627+0000", 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 374.81649813600001, 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094514200000000003, 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:42.613 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683627+0000", 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683627+0000", 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683627+0000", 2023-06-10T21:43:42.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3102000000000001e-05 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683680+0000", 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1826e-05 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683692+0000", 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8099999999999997e-06 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683697+0000", 2023-06-10T21:43:42.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5135000000000001e-05 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683742+0000", 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2966e-05 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.683765+0000", 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092694990000000005 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.693034+0000", 2023-06-10T21:43:42.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4801e-05 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:37:27.693079+0000", 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0094514200000000003 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.621 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:42.621 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:42.621 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:43:42.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:42.622 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:43:42.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:42 smithi064 ceph-mon[186706]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.2 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 798 op/s; 24284/254580 objects degraded (9.539%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:43:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:42 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:42.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:42 smithi064 ceph-mon[180297]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.2 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 798 op/s; 24284/254580 objects degraded (9.539%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:43:42.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:42 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:42.669 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:42 smithi152 ceph-mon[156819]: pgmap v1468: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.2 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 798 op/s; 24284/254580 objects degraded (9.539%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:43:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:42 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:43.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:42] "GET /metrics HTTP/1.1" 200 224003 "" "Prometheus/2.33.4" 2023-06-10T21:43:43.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[180297]: failed to encode map e399 with expected crc 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[180297]: osdmap e399: 8 total, 8 up, 4 in 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[180297]: failed to encode map e399 with expected crc 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:43.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[186706]: failed to encode map e399 with expected crc 2023-06-10T21:43:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[186706]: osdmap e399: 8 total, 8 up, 4 in 2023-06-10T21:43:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:43 smithi064 ceph-mon[186706]: failed to encode map e399 with expected crc 2023-06-10T21:43:43.849 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:43:43.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:43 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:43:43.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2390314438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:43:43.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:43 smithi152 ceph-mon[156819]: failed to encode map e399 with expected crc 2023-06-10T21:43:43.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:43 smithi152 ceph-mon[156819]: osdmap e399: 8 total, 8 up, 4 in 2023-06-10T21:43:43.850 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:43 smithi152 ceph-mon[156819]: failed to encode map e399 with expected crc 2023-06-10T21:43:44.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:44.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:44.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:44.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:44.517 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:44.675 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:44.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:44.713 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:44.763 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:44.783 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:44.799 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:44.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:44 smithi064 ceph-mon[186706]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 493 op/s; 24284/253179 objects degraded (9.592%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:44.835 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:44 smithi064 ceph-mon[180297]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 493 op/s; 24284/253179 objects degraded (9.592%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:44.836 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:44.837 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:44.846 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:44 smithi152 ceph-mon[156819]: pgmap v1470: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.1 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 493 op/s; 24284/253179 objects degraded (9.592%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:44.868 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:44.916 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:44.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:44.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:45.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:45.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:45.045 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:45.046 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:45.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.046 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.047 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:43:45.050 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:45.444 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:45.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:45.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:45.619 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:45.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1273721493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:45.735 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:45.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:45.852 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:45.901 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1273721493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:45.903 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1273721493' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:45.904 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:46.094 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:46.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:46.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:46.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:43:46.217 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:46.282 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:46.565 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:46.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:46.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:46 smithi064 ceph-mon[180297]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.0 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 23875/244149 objects degraded (9.779%); 6.1 MiB/s, 98 objects/s recovering 2023-06-10T21:43:46.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 24284/253179 objects degraded (9.592%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:46.674 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:46.744 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:46.845 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:46.858 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:46.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:46.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:46.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:46 smithi152 ceph-mon[156819]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.0 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 23875/244149 objects degraded (9.779%); 6.1 MiB/s, 98 objects/s recovering 2023-06-10T21:43:46.865 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 24284/253179 objects degraded (9.592%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:46.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:46 smithi064 ceph-mon[186706]: pgmap v1471: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 5.0 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 819 op/s; 23875/244149 objects degraded (9.779%); 6.1 MiB/s, 98 objects/s recovering 2023-06-10T21:43:46.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 24284/253179 objects degraded (9.592%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:46.966 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:47.001 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:43:47.001 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:43:47.030 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:47.033 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:43:47.051 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:43:47.273 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:47.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:47.378 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:47.445 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:47.509 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:47.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:47.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:47.642 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:47.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:47 smithi064 ceph-mon[180297]: from='client.36930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:47.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:47 smithi064 ceph-mon[186706]: from='client.36930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:47.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:47 smithi152 ceph-mon[156819]: from='client.36930 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:48.064 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:43:48.129 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:43:48.765 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:48.826 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:48.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:48.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:48.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:48.977 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1471,"stamp":"2023-06-10T21:43:46.171698+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5332473766,"num_objects":81383,"num_object_clones":0,"num_object_copies":244149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23875,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81383,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":138902,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3029,"num_bytes_recovered":195971086,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75722,"ondisk_log_size":75722,"up":267,"acting":254,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":17002724,"kb_used_data":14302084,"kb_used_omap":27,"kb_used_meta":2700452,"kb_avail":357977884,"statfs":{"total":383980142592,"available":366569353216,"internally_reserved":0,"allocated":14645334016,"data_stored":14641163229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2765263450},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,4,3,11,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-429522944,"num_objects":-6554,"num_object_clones":0,"num_object_copies":-19662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-787,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6554,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":787,"num_bytes_recovered":51576832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002336"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":438,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557952+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:43:39.557952+0000","last_peered":"2023-06-10T21:43:39.557952+0000","last_clean":"2023-06-10T21:43:39.557952+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:43:39.557952+0000","last_undegraded":"2023-06-10T21:43:39.557952+0000","last_fullsized":"2023-06-10T21:43:39.557952+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.333130+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:43:35.333130+0000","last_peered":"2023-06-10T21:43:35.333130+0000","last_clean":"2023-06-10T21:43:35.333130+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:43:35.333130+0000","last_undegraded":"2023-06-10T21:43:35.333130+0000","last_fullsized":"2023-06-10T21:43:35.333130+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":329,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330533+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:43:35.330533+0000","last_peered":"2023-06-10T21:43:35.330533+0000","last_clean":"2023-06-10T21:43:35.330533+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:43:35.330533+0000","last_undegraded":"2023-06-10T21:43:35.330533+0000","last_fullsized":"2023-06-10T21:43:35.330533+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":284,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561683+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:43:39.561683+0000","last_peered":"2023-06-10T21:43:39.561683+0000","last_clean":"2023-06-10T21:43:39.561683+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:43:39.561683+0000","last_undegraded":"2023-06-10T21:43:39.561683+0000","last_fullsized":"2023-06-10T21:43:39.561683+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.329982+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:43:35.329982+0000","last_peered":"2023-06-10T21:43:35.329982+0000","last_clean":"2023-06-10T21:43:35.329982+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:43:35.329982+0000","last_undegraded":"2023-06-10T21:43:35.329982+0000","last_fullsized":"2023-06-10T21:43:35.329982+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549687+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:43:39.549687+0000","last_peered":"2023-06-10T21:43:39.549687+0000","last_clean":"2023-06-10T21:43:39.549687+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:43:39.549687+0000","last_undegraded":"2023-06-10T21:43:39.549687+0000","last_fullsized":"2023-06-10T21:43:39.549687+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":264,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549590+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:43:39.549590+0000","last_peered":"2023-06-10T21:43:39.549590+0000","last_clean":"2023-06-10T21:43:39.549590+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:43:39.549590+0000","last_undegraded":"2023-06-10T21:43:39.549590+0000","last_fullsized":"2023-06-10T21:43:39.549590+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330080+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:43:35.330080+0000","last_peered":"2023-06-10T21:43:35.330080+0000","last_clean":"2023-06-10T21:43:35.330080+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:43:35.330080+0000","last_undegraded":"2023-06-10T21:43:35.330080+0000","last_fullsized":"2023-06-10T21:43:35.330080+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548813+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:43:39.548813+0000","last_peered":"2023-06-10T21:43:39.548813+0000","last_clean":"2023-06-10T21:43:39.548813+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:43:39.548813+0000","last_undegraded":"2023-06-10T21:43:39.548813+0000","last_fullsized":"2023-06-10T21:43:39.548813+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":148,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330464+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:43:35.330464+0000","last_peered":"2023-06-10T21:43:35.330464+0000","last_clean":"2023-06-10T21:43:35.330464+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:43:35.330464+0000","last_undegraded":"2023-06-10T21:43:35.330464+0000","last_fullsized":"2023-06-10T21:43:35.330464+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":156,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562949+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:43:39.562949+0000","last_peered":"2023-06-10T21:43:39.562949+0000","last_clean":"2023-06-10T21:43:39.562949+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:43:39.562949+0000","last_undegraded":"2023-06-10T21:43:39.562949+0000","last_fullsized":"2023-06-10T21:43:39.562949+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":398,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.558034+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:43:39.558034+0000","last_peered":"2023-06-10T21:43:39.558034+0000","last_clean":"2023-06-10T21:43:39.558034+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:43:39.558034+0000","last_undegraded":"2023-06-10T21:43:39.558034+0000","last_fullsized":"2023-06-10T21:43:39.558034+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":322,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561134+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:43:39.561134+0000","last_peered":"2023-06-10T21:43:39.561134+0000","last_clean":"2023-06-10T21:43:39.561134+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:43:39.561134+0000","last_undegraded":"2023-06-10T21:43:39.561134+0000","last_fullsized":"2023-06-10T21:43:39.561134+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":113,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548827+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:43:39.548827+0000","last_peered":"2023-06-10T21:43:39.548827+0000","last_clean":"2023-06-10T21:43:39.548827+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:43:39.548827+0000","last_undegraded":"2023-06-10T21:43:39.548827+0000","last_fullsized":"2023-06-10T21:43:39.548827+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":458,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562522+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:43:39.562522+0000","last_peered":"2023-06-10T21:43:39.562522+0000","last_clean":"2023-06-10T21:43:39.562522+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:43:39.562522+0000","last_undegraded":"2023-06-10T21:43:39.562522+0000","last_fullsized":"2023-06-10T21:43:39.562522+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":345,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332489+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:43:35.332489+0000","last_peered":"2023-06-10T21:43:35.332489+0000","last_clean":"2023-06-10T21:43:35.332489+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:43:35.332489+0000","last_undegraded":"2023-06-10T21:43:35.332489+0000","last_fullsized":"2023-06-10T21:43:35.332489+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":269,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562066+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:43:39.562066+0000","last_peered":"2023-06-10T21:43:39.562066+0000","last_clean":"2023-06-10T21:43:39.562066+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:43:39.562066+0000","last_undegraded":"2023-06-10T21:43:39.562066+0000","last_fullsized":"2023-06-10T21:43:39.562066+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549250+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:43:39.549250+0000","last_peered":"2023-06-10T21:43:39.549250+0000","last_clean":"2023-06-10T21:43:39.549250+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:43:39.549250+0000","last_undegraded":"2023-06-10T21:43:39.549250+0000","last_fullsized":"2023-06-10T21:43:39.549250+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":411,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.559444+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:43:39.559444+0000","last_peered":"2023-06-10T21:43:39.559444+0000","last_clean":"2023-06-10T21:43:39.559444+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:43:39.559444+0000","last_undegraded":"2023-06-10T21:43:39.559444+0000","last_fullsized":"2023-06-10T21:43:39.559444+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.333082+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:43:35.333082+0000","last_peered":"2023-06-10T21:43:35.333082+0000","last_clean":"2023-06-10T21:43:35.333082+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:43:35.333082+0000","last_undegraded":"2023-06-10T21:43:35.333082+0000","last_fullsized":"2023-06-10T21:43:35.333082+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":423,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.545400+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:43:39.545400+0000","last_peered":"2023-06-10T21:43:39.545400+0000","last_clean":"2023-06-10T21:43:39.545400+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:43:39.545400+0000","last_undegraded":"2023-06-10T21:43:39.545400+0000","last_fullsized":"2023-06-10T21:43:39.545400+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332837+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:43:35.332837+0000","last_peered":"2023-06-10T21:43:35.332837+0000","last_clean":"2023-06-10T21:43:35.332837+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:43:35.332837+0000","last_undegraded":"2023-06-10T21:43:35.332837+0000","last_fullsized":"2023-06-10T21:43:35.332837+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"399'8741","reported_seq":8777,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:46.049855+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:43:46.049855+0000","last_peered":"2023-06-10T21:43:46.049855+0000","last_clean":"2023-06-10T21:43:46.049855+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:43:46.049855+0000","last_undegraded":"2023-06-10T21:43:46.049855+0000","last_fullsized":"2023-06-10T21:43:46.049855+0000","mapping_epoch":373,"log_start":"381'3700","ondisk_log_start":"381'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325779456,"num_objects":4971,"num_object_clones":0,"num_object_copies":14913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8741,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":191,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563557+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:43:39.563557+0000","last_peered":"2023-06-10T21:43:39.563557+0000","last_clean":"2023-06-10T21:43:39.563557+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:43:39.563557+0000","last_undegraded":"2023-06-10T21:43:39.563557+0000","last_fullsized":"2023-06-10T21:43:39.563557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.545199+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:43:39.545199+0000","last_peered":"2023-06-10T21:43:39.545199+0000","last_clean":"2023-06-10T21:43:39.545199+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:43:39.545199+0000","last_undegraded":"2023-06-10T21:43:39.545199+0000","last_fullsized":"2023-06-10T21:43:39.545199+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"399'8724","reported_seq":8776,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:44.181786+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:43:44.181786+0000","last_peered":"2023-06-10T21:43:44.181786+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:43:44.181786+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"382'4400","ondisk_log_start":"382'4400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4324,"log_dups_size":0,"ondisk_log_size":4324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335282176,"num_objects":5116,"num_object_clones":0,"num_object_copies":15348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8724,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":3226}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":191,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561904+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:43:39.561904+0000","last_peered":"2023-06-10T21:43:39.561904+0000","last_clean":"2023-06-10T21:43:39.561904+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:43:39.561904+0000","last_undegraded":"2023-06-10T21:43:39.561904+0000","last_fullsized":"2023-06-10T21:43:39.561904+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330599+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:43:35.330599+0000","last_peered":"2023-06-10T21:43:35.330599+0000","last_clean":"2023-06-10T21:43:35.330599+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:43:35.330599+0000","last_undegraded":"2023-06-10T21:43:35.330599+0000","last_fullsized":"2023-06-10T21:43:35.330599+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"398'8651","reported_seq":8716,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:42.493383+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:43:42.493383+0000","last_peered":"2023-06-10T21:43:42.493383+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:43:42.493383+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"382'4400","ondisk_log_start":"382'4400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4251,"log_dups_size":0,"ondisk_log_size":4251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8651,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3259}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":478,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562922+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:43:39.562922+0000","last_peered":"2023-06-10T21:43:39.562922+0000","last_clean":"2023-06-10T21:43:39.562922+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:43:39.562922+0000","last_undegraded":"2023-06-10T21:43:39.562922+0000","last_fullsized":"2023-06-10T21:43:39.562922+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":162,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560557+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:43:39.560557+0000","last_peered":"2023-06-10T21:43:39.560557+0000","last_clean":"2023-06-10T21:43:39.560557+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:43:39.560557+0000","last_undegraded":"2023-06-10T21:43:39.560557+0000","last_fullsized":"2023-06-10T21:43:39.560557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"398'8642","reported_seq":8684,"reported_epoch":398,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:42.523996+0000","last_change":"2023-06-10T21:43:02.519446+0000","last_active":"2023-06-10T21:43:42.523996+0000","last_peered":"2023-06-10T21:43:42.523996+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:43:02.514777+0000","last_became_peered":"2023-06-10T21:43:02.514777+0000","last_unstale":"2023-06-10T21:43:42.523996+0000","last_undegraded":"2023-06-10T21:43:02.215416+0000","last_fullsized":"2023-06-10T21:43:02.198507+0000","mapping_epoch":391,"log_start":"382'4384","ondisk_log_start":"382'4384","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4258,"log_dups_size":0,"ondisk_log_size":4258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:27:12.740821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338296832,"num_objects":5162,"num_object_clones":0,"num_object_copies":15486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8642,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3258}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549127+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:43:39.549127+0000","last_peered":"2023-06-10T21:43:39.549127+0000","last_clean":"2023-06-10T21:43:39.549127+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:43:39.549127+0000","last_undegraded":"2023-06-10T21:43:39.549127+0000","last_fullsized":"2023-06-10T21:43:39.549127+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332243+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:43:35.332243+0000","last_peered":"2023-06-10T21:43:35.332243+0000","last_clean":"2023-06-10T21:43:35.332243+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:43:35.332243+0000","last_undegraded":"2023-06-10T21:43:35.332243+0000","last_fullsized":"2023-06-10T21:43:35.332243+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"399'8707","reported_seq":8752,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.006013+0000","last_change":"2023-06-10T21:43:02.585044+0000","last_active":"2023-06-10T21:43:46.006013+0000","last_peered":"2023-06-10T21:43:46.006013+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:43:02.579281+0000","last_became_peered":"2023-06-10T21:43:02.579281+0000","last_unstale":"2023-06-10T21:43:46.006013+0000","last_undegraded":"2023-06-10T21:43:02.241138+0000","last_fullsized":"2023-06-10T21:43:02.196427+0000","mapping_epoch":391,"log_start":"381'3637","ondisk_log_start":"381'3637","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5070,"log_dups_size":0,"ondisk_log_size":5070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:58.809359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329056256,"num_objects":5021,"num_object_clones":0,"num_object_copies":15063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1894,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8707,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3127}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":55,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332152+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:43:35.332152+0000","last_peered":"2023-06-10T21:43:35.332152+0000","last_clean":"2023-06-10T21:43:35.332152+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:43:35.332152+0000","last_undegraded":"2023-06-10T21:43:35.332152+0000","last_fullsized":"2023-06-10T21:43:35.332152+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332419+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:43:35.332419+0000","last_peered":"2023-06-10T21:43:35.332419+0000","last_clean":"2023-06-10T21:43:35.332419+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:43:35.332419+0000","last_undegraded":"2023-06-10T21:43:35.332419+0000","last_fullsized":"2023-06-10T21:43:35.332419+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"399'8612","reported_seq":12504,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:46.041155+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:43:46.041155+0000","last_peered":"2023-06-10T21:43:46.041155+0000","last_clean":"2023-06-10T21:43:46.041155+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:43:46.041155+0000","last_undegraded":"2023-06-10T21:43:46.041155+0000","last_fullsized":"2023-06-10T21:43:46.041155+0000","mapping_epoch":395,"log_start":"381'3600","ondisk_log_start":"381'3600","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5012,"log_dups_size":0,"ondisk_log_size":5012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334233600,"num_objects":5100,"num_object_clones":0,"num_object_copies":15300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8612,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":218,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.331030+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:43:35.331030+0000","last_peered":"2023-06-10T21:43:35.331030+0000","last_clean":"2023-06-10T21:43:35.331030+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:43:35.331030+0000","last_undegraded":"2023-06-10T21:43:35.331030+0000","last_fullsized":"2023-06-10T21:43:35.331030+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":436,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.558472+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:43:39.558472+0000","last_peered":"2023-06-10T21:43:39.558472+0000","last_clean":"2023-06-10T21:43:39.558472+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:43:39.558472+0000","last_undegraded":"2023-06-10T21:43:39.558472+0000","last_fullsized":"2023-06-10T21:43:39.558472+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"399'8453","reported_seq":8502,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:44.101083+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:43:44.101083+0000","last_peered":"2023-06-10T21:43:44.101083+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:43:44.101083+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"382'4100","ondisk_log_start":"382'4100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8453,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":3266}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":414,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557982+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:43:39.557982+0000","last_peered":"2023-06-10T21:43:39.557982+0000","last_clean":"2023-06-10T21:43:39.557982+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:43:39.557982+0000","last_undegraded":"2023-06-10T21:43:39.557982+0000","last_fullsized":"2023-06-10T21:43:39.557982+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.333027+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:43:35.333027+0000","last_peered":"2023-06-10T21:43:35.333027+0000","last_clean":"2023-06-10T21:43:35.333027+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:43:35.333027+0000","last_undegraded":"2023-06-10T21:43:35.333027+0000","last_fullsized":"2023-06-10T21:43:35.333027+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"399'8638","reported_seq":8682,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.007752+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:43:46.007752+0000","last_peered":"2023-06-10T21:43:46.007752+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:43:46.007752+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"381'3574","ondisk_log_start":"381'3574","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330039296,"num_objects":5036,"num_object_clones":0,"num_object_copies":15108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8638,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3101}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330271+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:43:35.330271+0000","last_peered":"2023-06-10T21:43:35.330271+0000","last_clean":"2023-06-10T21:43:35.330271+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:43:35.330271+0000","last_undegraded":"2023-06-10T21:43:35.330271+0000","last_fullsized":"2023-06-10T21:43:35.330271+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":345,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332991+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:43:35.332991+0000","last_peered":"2023-06-10T21:43:35.332991+0000","last_clean":"2023-06-10T21:43:35.332991+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:43:35.332991+0000","last_undegraded":"2023-06-10T21:43:35.332991+0000","last_fullsized":"2023-06-10T21:43:35.332991+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"399'8648","reported_seq":8690,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.021920+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:43:46.021920+0000","last_peered":"2023-06-10T21:43:46.021920+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:43:46.021920+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"381'3619","ondisk_log_start":"381'3619","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5029,"log_dups_size":0,"ondisk_log_size":5029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324272128,"num_objects":4948,"num_object_clones":0,"num_object_copies":14844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8648,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3019}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":182,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561212+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:43:39.561212+0000","last_peered":"2023-06-10T21:43:39.561212+0000","last_clean":"2023-06-10T21:43:39.561212+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:43:39.561212+0000","last_undegraded":"2023-06-10T21:43:39.561212+0000","last_fullsized":"2023-06-10T21:43:39.561212+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":63,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549098+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:43:39.549098+0000","last_peered":"2023-06-10T21:43:39.549098+0000","last_clean":"2023-06-10T21:43:39.549098+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:43:39.549098+0000","last_undegraded":"2023-06-10T21:43:39.549098+0000","last_fullsized":"2023-06-10T21:43:39.549098+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548363+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:43:39.548363+0000","last_peered":"2023-06-10T21:43:39.548363+0000","last_clean":"2023-06-10T21:43:39.548363+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:43:39.548363+0000","last_undegraded":"2023-06-10T21:43:39.548363+0000","last_fullsized":"2023-06-10T21:43:39.548363+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"399'8599","reported_seq":8653,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:44.107535+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:43:44.107535+0000","last_peered":"2023-06-10T21:43:44.107535+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:43:44.107535+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"382'4300","ondisk_log_start":"382'4300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4299,"log_dups_size":0,"ondisk_log_size":4299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334495768,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8599,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":3188}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":82,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548262+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:43:39.548262+0000","last_peered":"2023-06-10T21:43:39.548262+0000","last_clean":"2023-06-10T21:43:39.548262+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:43:39.548262+0000","last_undegraded":"2023-06-10T21:43:39.548262+0000","last_fullsized":"2023-06-10T21:43:39.548262+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548493+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:43:39.548493+0000","last_peered":"2023-06-10T21:43:39.548493+0000","last_clean":"2023-06-10T21:43:39.548493+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:43:39.548493+0000","last_undegraded":"2023-06-10T21:43:39.548493+0000","last_fullsized":"2023-06-10T21:43:39.548493+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":263,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560288+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:43:39.560288+0000","last_peered":"2023-06-10T21:43:39.560288+0000","last_clean":"2023-06-10T21:43:39.560288+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:43:39.560288+0000","last_undegraded":"2023-06-10T21:43:39.560288+0000","last_fullsized":"2023-06-10T21:43:39.560288+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"399'8557","reported_seq":8611,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:44.164353+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:43:44.164353+0000","last_peered":"2023-06-10T21:43:44.164353+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:43:44.164353+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"382'4200","ondisk_log_start":"382'4200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4357,"log_dups_size":0,"ondisk_log_size":4357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335609856,"num_objects":5121,"num_object_clones":0,"num_object_copies":15363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8557,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":3248}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557905+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:43:39.557905+0000","last_peered":"2023-06-10T21:43:39.557905+0000","last_clean":"2023-06-10T21:43:39.557905+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:43:39.557905+0000","last_undegraded":"2023-06-10T21:43:39.557905+0000","last_fullsized":"2023-06-10T21:43:39.557905+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":288,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:41.051903+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:43:41.051903+0000","last_peered":"2023-06-10T21:43:41.051903+0000","last_clean":"2023-06-10T21:43:41.051903+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:43:41.051903+0000","last_undegraded":"2023-06-10T21:43:41.051903+0000","last_fullsized":"2023-06-10T21:43:41.051903+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549633+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:43:39.549633+0000","last_peered":"2023-06-10T21:43:39.549633+0000","last_clean":"2023-06-10T21:43:39.549633+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:43:39.549633+0000","last_undegraded":"2023-06-10T21:43:39.549633+0000","last_fullsized":"2023-06-10T21:43:39.549633+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"399'8741","reported_seq":8784,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.005956+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:43:46.005956+0000","last_peered":"2023-06-10T21:43:46.005956+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:43:46.005956+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"381'3700","ondisk_log_start":"381'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8741,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3177}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":73,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548750+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:43:39.548750+0000","last_peered":"2023-06-10T21:43:39.548750+0000","last_clean":"2023-06-10T21:43:39.548750+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:43:39.548750+0000","last_undegraded":"2023-06-10T21:43:39.548750+0000","last_fullsized":"2023-06-10T21:43:39.548750+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":599,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560534+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:43:39.560534+0000","last_peered":"2023-06-10T21:43:39.560534+0000","last_clean":"2023-06-10T21:43:39.560534+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:43:39.560534+0000","last_undegraded":"2023-06-10T21:43:39.560534+0000","last_fullsized":"2023-06-10T21:43:39.560534+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330839+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:43:35.330839+0000","last_peered":"2023-06-10T21:43:35.330839+0000","last_clean":"2023-06-10T21:43:35.330839+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:43:35.330839+0000","last_undegraded":"2023-06-10T21:43:35.330839+0000","last_fullsized":"2023-06-10T21:43:35.330839+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":282,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563065+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:43:39.563065+0000","last_peered":"2023-06-10T21:43:39.563065+0000","last_clean":"2023-06-10T21:43:39.563065+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:43:39.563065+0000","last_undegraded":"2023-06-10T21:43:39.563065+0000","last_fullsized":"2023-06-10T21:43:39.563065+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"399'8551","reported_seq":10656,"reported_epoch":399,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:44.196443+0000","last_change":"2023-06-10T21:43:32.135184+0000","last_active":"2023-06-10T21:43:44.196443+0000","last_peered":"2023-06-10T21:43:44.196443+0000","last_clean":"2023-06-10T21:42:30.256970+0000","last_became_active":"2023-06-10T21:43:02.545711+0000","last_became_peered":"2023-06-10T21:43:02.545711+0000","last_unstale":"2023-06-10T21:43:44.196443+0000","last_undegraded":"2023-06-10T21:43:02.227101+0000","last_fullsized":"2023-06-10T21:43:02.203404+0000","mapping_epoch":391,"log_start":"382'4200","ondisk_log_start":"382'4200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4351,"log_dups_size":0,"ondisk_log_size":4351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331284480,"num_objects":5055,"num_object_clones":0,"num_object_copies":15165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":891,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8551,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1028,"num_bytes_recovered":67371008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":4164}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":451,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560741+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:43:39.560741+0000","last_peered":"2023-06-10T21:43:39.560741+0000","last_clean":"2023-06-10T21:43:39.560741+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:43:39.560741+0000","last_undegraded":"2023-06-10T21:43:39.560741+0000","last_fullsized":"2023-06-10T21:43:39.560741+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330205+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:43:35.330205+0000","last_peered":"2023-06-10T21:43:35.330205+0000","last_clean":"2023-06-10T21:43:35.330205+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:43:35.330205+0000","last_undegraded":"2023-06-10T21:43:35.330205+0000","last_fullsized":"2023-06-10T21:43:35.330205+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":149,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562238+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:43:39.562238+0000","last_peered":"2023-06-10T21:43:39.562238+0000","last_clean":"2023-06-10T21:43:39.562238+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:43:39.562238+0000","last_undegraded":"2023-06-10T21:43:39.562238+0000","last_fullsized":"2023-06-10T21:43:39.562238+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"399'8703","reported_seq":8739,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:46.035542+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:43:46.035542+0000","last_peered":"2023-06-10T21:43:46.035542+0000","last_clean":"2023-06-10T21:43:46.035542+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:43:46.035542+0000","last_undegraded":"2023-06-10T21:43:46.035542+0000","last_fullsized":"2023-06-10T21:43:46.035542+0000","mapping_epoch":373,"log_start":"381'3700","ondisk_log_start":"381'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329711616,"num_objects":5031,"num_object_clones":0,"num_object_copies":15093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8703,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":430,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562982+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:43:39.562982+0000","last_peered":"2023-06-10T21:43:39.562982+0000","last_clean":"2023-06-10T21:43:39.562982+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:43:39.562982+0000","last_undegraded":"2023-06-10T21:43:39.562982+0000","last_fullsized":"2023-06-10T21:43:39.562982+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":278,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560389+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:43:39.560389+0000","last_peered":"2023-06-10T21:43:39.560389+0000","last_clean":"2023-06-10T21:43:39.560389+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:43:39.560389+0000","last_undegraded":"2023-06-10T21:43:39.560389+0000","last_fullsized":"2023-06-10T21:43:39.560389+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548567+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:43:39.548567+0000","last_peered":"2023-06-10T21:43:39.548567+0000","last_clean":"2023-06-10T21:43:39.548567+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:43:39.548567+0000","last_undegraded":"2023-06-10T21:43:39.548567+0000","last_fullsized":"2023-06-10T21:43:39.548567+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"399'8910","reported_seq":8954,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.054793+0000","last_change":"2023-06-10T21:43:02.588633+0000","last_active":"2023-06-10T21:43:46.054793+0000","last_peered":"2023-06-10T21:43:46.054793+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:43:46.054793+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"381'3900","ondisk_log_start":"381'3900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339738624,"num_objects":5184,"num_object_clones":0,"num_object_copies":15552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1976,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8910,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3208}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":407,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332901+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:43:35.332901+0000","last_peered":"2023-06-10T21:43:35.332901+0000","last_clean":"2023-06-10T21:43:35.332901+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:43:35.332901+0000","last_undegraded":"2023-06-10T21:43:35.332901+0000","last_fullsized":"2023-06-10T21:43:35.332901+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":218,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.333240+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:43:35.333240+0000","last_peered":"2023-06-10T21:43:35.333240+0000","last_clean":"2023-06-10T21:43:35.333240+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:43:35.333240+0000","last_undegraded":"2023-06-10T21:43:35.333240+0000","last_fullsized":"2023-06-10T21:43:35.333240+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330237+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:43:35.330237+0000","last_peered":"2023-06-10T21:43:35.330237+0000","last_clean":"2023-06-10T21:43:35.330237+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:43:35.330237+0000","last_undegraded":"2023-06-10T21:43:35.330237+0000","last_fullsized":"2023-06-10T21:43:35.330237+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"399'8764","reported_seq":8814,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:46.055326+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:43:46.055326+0000","last_peered":"2023-06-10T21:43:46.055326+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:43:46.055326+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"381'3700","ondisk_log_start":"381'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331743232,"num_objects":5062,"num_object_clones":0,"num_object_copies":15186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8764,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3141}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":147,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563636+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:43:39.563636+0000","last_peered":"2023-06-10T21:43:39.563636+0000","last_clean":"2023-06-10T21:43:39.563636+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:43:39.563636+0000","last_undegraded":"2023-06-10T21:43:39.563636+0000","last_fullsized":"2023-06-10T21:43:39.563636+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":261,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561482+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:43:39.561482+0000","last_peered":"2023-06-10T21:43:39.561482+0000","last_clean":"2023-06-10T21:43:39.561482+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:43:39.561482+0000","last_undegraded":"2023-06-10T21:43:39.561482+0000","last_fullsized":"2023-06-10T21:43:39.561482+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.547986+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:43:39.547986+0000","last_peered":"2023-06-10T21:43:39.547986+0000","last_clean":"2023-06-10T21:43:39.547986+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:43:39.547986+0000","last_undegraded":"2023-06-10T21:43:39.547986+0000","last_fullsized":"2023-06-10T21:43:39.547986+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":98,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562652+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:43:39.562652+0000","last_peered":"2023-06-10T21:43:39.562652+0000","last_clean":"2023-06-10T21:43:39.562652+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:43:39.562652+0000","last_undegraded":"2023-06-10T21:43:39.562652+0000","last_fullsized":"2023-06-10T21:43:39.562652+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":295,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560781+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:43:39.560781+0000","last_peered":"2023-06-10T21:43:39.560781+0000","last_clean":"2023-06-10T21:43:39.560781+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:43:39.560781+0000","last_undegraded":"2023-06-10T21:43:39.560781+0000","last_fullsized":"2023-06-10T21:43:39.560781+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548949+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:43:39.548949+0000","last_peered":"2023-06-10T21:43:39.548949+0000","last_clean":"2023-06-10T21:43:39.548949+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:43:39.548949+0000","last_undegraded":"2023-06-10T21:43:39.548949+0000","last_fullsized":"2023-06-10T21:43:39.548949+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":301,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.332329+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:43:35.332329+0000","last_peered":"2023-06-10T21:43:35.332329+0000","last_clean":"2023-06-10T21:43:35.332329+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:43:35.332329+0000","last_undegraded":"2023-06-10T21:43:35.332329+0000","last_fullsized":"2023-06-10T21:43:35.332329+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548777+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:43:39.548777+0000","last_peered":"2023-06-10T21:43:39.548777+0000","last_clean":"2023-06-10T21:43:39.548777+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:43:39.548777+0000","last_undegraded":"2023-06-10T21:43:39.548777+0000","last_fullsized":"2023-06-10T21:43:39.548777+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":372,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.331107+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:43:35.331107+0000","last_peered":"2023-06-10T21:43:35.331107+0000","last_clean":"2023-06-10T21:43:35.331107+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:43:35.331107+0000","last_undegraded":"2023-06-10T21:43:35.331107+0000","last_fullsized":"2023-06-10T21:43:35.331107+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":51,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.333155+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:43:35.333155+0000","last_peered":"2023-06-10T21:43:35.333155+0000","last_clean":"2023-06-10T21:43:35.333155+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:43:35.333155+0000","last_undegraded":"2023-06-10T21:43:35.333155+0000","last_fullsized":"2023-06-10T21:43:35.333155+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330336+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:43:35.330336+0000","last_peered":"2023-06-10T21:43:35.330336+0000","last_clean":"2023-06-10T21:43:35.330336+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:43:35.330336+0000","last_undegraded":"2023-06-10T21:43:35.330336+0000","last_fullsized":"2023-06-10T21:43:35.330336+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":320,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550820+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:43:39.550820+0000","last_peered":"2023-06-10T21:43:39.550820+0000","last_clean":"2023-06-10T21:43:39.550820+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:43:39.550820+0000","last_undegraded":"2023-06-10T21:43:39.550820+0000","last_fullsized":"2023-06-10T21:43:39.550820+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-06-10T21:43:35.330318+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:43:35.330318+0000","last_peered":"2023-06-10T21:43:35.330318+0000","last_clean":"2023-06-10T21:43:35.330318+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:43:35.330318+0000","last_undegraded":"2023-06-10T21:43:35.330318+0000","last_fullsized":"2023-06-10T21:43:35.330318+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5331877912,"num_objects":81359,"num_object_clones":0,"num_object_copies":244077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23875,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81359,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138641,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2954,"num_bytes_recovered":193593344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14526984192,"data_stored":14526971976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75527,"ondisk_log_size":75527,"up":48,"acting":35,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430760,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4549800,"kb_used_data":3629216,"kb_used_omap":0,"kb_used_meta":920512,"kb_avail":89195352,"statfs":{"total":95995035648,"available":91336040448,"internally_reserved":0,"allocated":3716317184,"data_stored":3715610400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":942604288},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:42:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.0049999999999999,"15min":3.673},"min":{"1min":0.29299999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":1.4690000000000001,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":2.411,"15min":3.9580000000000002},"min":{"1min":0.307,"5min":0.25,"15min":0.223},"max":{"1min":5.0149999999999997,"5min":56.628,"15min":185.58000000000001},"last":3.0099999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.1309999999999998,"15min":2.6059999999999999},"min":{"1min":0.33400000000000002,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":4.3540000000000001,"5min":38.078000000000003,"15min":87.417000000000002},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6469999999999998,"15min":2.851},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":7.3710000000000004,"5min":38.341000000000001,"15min":87.438000000000002},"last":2.903}]},{"osd":2,"last update":"Sat Jun 10 21:43:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.9649999999999999,"15min":4.8200000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7959999999999998,"5min":75.219999999999999,"15min":172.33600000000001},"last":3.04},{"interface":"front","average":{"1min":2,"5min":2.6970000000000001,"15min":4.7949999999999999},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":4.8799999999999999,"5min":75.108999999999995,"15min":172.185},"last":2.9729999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":3.1800000000000002,"15min":4.4429999999999996},"min":{"1min":0.443,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":7.5019999999999998,"5min":89.504999999999995,"15min":147.00700000000001},"last":4.7140000000000004},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.786,"15min":4.1669999999999998},"min":{"1min":0.47399999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":8.2889999999999997,"5min":90.215999999999994,"15min":146.93899999999999},"last":2.9510000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59199999999999997,"15min":0.49099999999999999},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":1.7,"5min":5.859,"15min":5.859},"last":13.429},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.998,"15min":0.69299999999999995},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":5.1219999999999999,"5min":22.77,"15min":22.77},"last":3.718}]},{"osd":5,"last update":"Sat Jun 10 21:42:08 2023","interfaces":[{"interface":"back","average":{"1min":11.141999999999999,"5min":2.6240000000000001,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.252,"15min":0.252},"max":{"1min":174.13200000000001,"5min":174.13200000000001,"15min":174.13200000000001},"last":13.472},{"interface":"front","average":{"1min":11.669,"5min":2.806,"15min":1.302},"min":{"1min":0.38700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":177.81200000000001,"5min":177.81200000000001,"15min":177.81200000000001},"last":4.843}]},{"osd":6,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.77300000000000002,"15min":0.63300000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7709999999999999,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.70499999999999996,"15min":0.57699999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":2.29,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":4.3739999999999997}]}]},{"osd":6,"up_from":0,"seq":180388626988,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020407,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075819,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4857556,"kb_used_data":3904588,"kb_used_omap":7,"kb_used_meta":952952,"kb_avail":88887596,"statfs":{"total":95995035648,"available":91020898304,"internally_reserved":0,"allocated":3998298112,"data_stored":3996973997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":975822929},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,4,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:42:55 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":5.609,"15min":4.6210000000000004},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":7.069,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.4119999999999999,"5min":4.9340000000000002,"15min":4.5919999999999996},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.7590000000000003,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.309}]},{"osd":1,"last update":"Sat Jun 10 21:42:47 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":8.2249999999999996,"15min":5.3739999999999997},"min":{"1min":0.311,"5min":0.217,"15min":0.19},"max":{"1min":6.399,"5min":171.286,"15min":171.286},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.798,"5min":4.4930000000000003,"15min":3.8690000000000002},"min":{"1min":0.29499999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":7.6260000000000003,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:43:24 2023","interfaces":[{"interface":"back","average":{"1min":11.114000000000001,"5min":9.2509999999999994,"15min":6.2389999999999999},"min":{"1min":0.41599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":145.07599999999999,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":11.140000000000001,"5min":8.5540000000000003,"15min":6.2969999999999997},"min":{"1min":0.39300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":144.99000000000001,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:42:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":3.3050000000000002,"15min":2.7109999999999999},"min":{"1min":0.33400000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.4749999999999996,"5min":171.48699999999999,"15min":171.48699999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.661,"5min":4.9199999999999999,"15min":3.6139999999999999},"min":{"1min":0.39300000000000002,"5min":0.312,"15min":0.27000000000000002},"max":{"1min":4.1120000000000001,"5min":171.18100000000001,"15min":171.18100000000001},"last":3.222}]},{"osd":5,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.613,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.7759999999999998,"5min":171.303,"15min":171.303},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.5430000000000001,"15min":2.3279999999999998},"min":{"1min":0.38,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1280000000000001,"5min":171.203,"15min":171.203},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:42:47 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":3.4580000000000002,"15min":2.427},"min":{"1min":0.45200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.5460000000000003,"5min":171.441,"15min":171.441},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.1040000000000001,"5min":5.056,"15min":3.323},"min":{"1min":0.48699999999999999,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":4.1669999999999998,"5min":171.24600000000001,"15min":171.24600000000001},"last":3.262}]},{"osd":7,"last update":"Sat Jun 10 21:42:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6639999999999999,"5min":3.8300000000000001,"15min":4.3890000000000002},"min":{"1min":0.5,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":7.282,"5min":171.26499999999999,"15min":219.131},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.369,"5min":3.4329999999999998,"15min":4.3840000000000003},"min":{"1min":0.375,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5129999999999999,"5min":171.52000000000001,"15min":219.19},"last":0.77100000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824178,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4382920,"kb_used_data":3703296,"kb_used_omap":7,"kb_used_meta":679608,"kb_avail":89362232,"statfs":{"total":95995035648,"available":91506925568,"internally_reserved":0,"allocated":3792175104,"data_stored":3790734203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":695918611},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":2.1589999999999998,"15min":3.5760000000000001},"min":{"1min":0.23599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.1949999999999998,"5min":30.744,"15min":105.09},"last":0.317},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.4580000000000002,"15min":3.6379999999999999},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.6970000000000001,"5min":30.805,"15min":99.852999999999994},"last":0.63900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.5310000000000001,"15min":3.5099999999999998},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.6459999999999999,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.484,"15min":3.4199999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23000000000000001},"max":{"1min":3.9399999999999999,"5min":28.695,"15min":94.691000000000003},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":4.6109999999999998,"15min":5.0359999999999996},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6609999999999996,"5min":231.07900000000001,"15min":231.07900000000001},"last":4.5},{"interface":"front","average":{"1min":2.7050000000000001,"5min":5.3769999999999998,"15min":6.3760000000000003},"min":{"1min":0.35899999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":8.6489999999999991,"5min":231.58600000000001,"15min":231.58600000000001},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.5249999999999999,"15min":1.4410000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.7300000000000004,"5min":10.368,"15min":13.417},"last":4.4660000000000002},{"interface":"front","average":{"1min":1.514,"5min":2.085,"15min":2.3199999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":6.5359999999999996,"5min":19.244,"15min":19.728000000000002},"last":3.4820000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.847,"15min":1.5489999999999999},"min":{"1min":0.42899999999999999,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":3.8559999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.409,"5min":1.4830000000000001,"15min":1.5049999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":9.1210000000000004,"5min":10.414999999999999,"15min":13.470000000000001},"last":4.3940000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.1240000000000001,"15min":2.3679999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":6.6150000000000002,"5min":19.164999999999999,"15min":19.805},"last":3.4420000000000002},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.508,"15min":1.514},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7610000000000001,"5min":9.9719999999999995,"15min":17.327000000000002},"last":0.97099999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.2839999999999998,"15min":2.6339999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":6.5830000000000002,"5min":19.193000000000001,"15min":79.984999999999999},"last":3.3929999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.603,"15min":1.802},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.1749999999999998,"5min":9.9380000000000006,"15min":80.272999999999996},"last":0.94899999999999995}]}]},{"osd":4,"up_from":0,"seq":128849019444,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245450,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3212448,"kb_used_data":3064984,"kb_used_omap":11,"kb_used_meta":147380,"kb_avail":90532704,"statfs":{"total":95995035648,"available":92705488896,"internally_reserved":0,"allocated":3138543616,"data_stored":3137844629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":150917622},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34300000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.6200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0469999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.173}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":8.4160000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.1559999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3965976576,"data_stored":3965976576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3741323264,"data_stored":3741319192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3129806848,"data_stored":3129802776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3689877504,"data_stored":3689873432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:43:48.984 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:43:49.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:49.113 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:49.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:49.217 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:49.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:49 smithi152 ceph-mon[156819]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 692 op/s; 23875/242730 objects degraded (9.836%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:49.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:49.241 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:49.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:49.433 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:49.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[186706]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 692 op/s; 23875/242730 objects degraded (9.836%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:49.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:49.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:49.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:49.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[180297]: pgmap v1472: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 692 op/s; 23875/242730 objects degraded (9.836%); 3.0 MiB/s, 47 objects/s recovering 2023-06-10T21:43:49.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:43:49.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:43:49.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:49.606 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:49.662 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:49.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:49.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:49.786 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:49.792 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:49.834 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:49.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:49.941 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:50.015 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:50.108 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:50.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:50.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:50.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:50.262 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:50.306 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:50.437 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:50.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:50.545 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:50.619 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:50.843 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:50.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:50.934 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:43:50.935 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:43:50.952 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:51.022 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:51.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:51.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:51.259 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:43:51.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[180297]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 23441/239142 objects degraded (9.802%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:51.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/384570772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[186706]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 23441/239142 objects degraded (9.802%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:51.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/384570772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.298 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:51.360 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:51.405 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:51.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:51.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:51.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:51.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:51 smithi152 ceph-mon[156819]: pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 23441/239142 objects degraded (9.802%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:51.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/384570772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.548 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:43:51.550 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:51.610 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:51.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:51.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:51.750 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:51.943 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:51.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:51.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:51.987 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:43:52.067 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:52.120 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:52.349 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:52.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:52.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:52.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 23441/239142 objects degraded (9.802%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[180297]: failed to encode map e400 with expected crc 2023-06-10T21:43:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:52.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[180297]: osdmap e400: 8 total, 8 up, 4 in 2023-06-10T21:43:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 23441/239142 objects degraded (9.802%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[186706]: failed to encode map e400 with expected crc 2023-06-10T21:43:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:52.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:52 smithi064 ceph-mon[186706]: osdmap e400: 8 total, 8 up, 4 in 2023-06-10T21:43:52.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:52.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:52 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:43:52.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:52 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 23441/239142 objects degraded (9.802%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:52.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:52 smithi152 ceph-mon[156819]: failed to encode map e400 with expected crc 2023-06-10T21:43:52.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:43:52.483 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:52 smithi152 ceph-mon[156819]: osdmap e400: 8 total, 8 up, 4 in 2023-06-10T21:43:52.485 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:52.546 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:52.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:52.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:52.580 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:52.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:43:52.680 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:52.834 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:43:52.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:52.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:43:52.945 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:53.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:53.127 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:43:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:43:52] "GET /metrics HTTP/1.1" 200 224002 "" "Prometheus/2.33.4" 2023-06-10T21:43:53.314 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:53.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:53.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:53 smithi064 ceph-mon[180297]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 23441/232839 objects degraded (10.067%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:43:53.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:53 smithi064 ceph-mon[186706]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 23441/232839 objects degraded (10.067%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:43:53.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:53.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:53 smithi152 ceph-mon[156819]: pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 753 op/s; 23441/232839 objects degraded (10.067%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:43:53.451 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:43:53.491 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:43:53.493 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:53.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:43:53.547 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:43:53.553 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:53.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:53.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:43:53.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:43:53.705 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:53.714 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:43:53.767 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:43:53.809 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:43:53.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:53.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:43:53.820 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:43:53.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:43:53.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:43:54.027 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:54.126 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:43:54.127 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:43:54.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:43:54.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:54.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:43:54.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:54.380 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:54.440 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:54 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4097460009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:54.441 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:54 smithi152 ceph-mon[156819]: from='client.56800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:54.441 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:54 smithi152 ceph-mon[156819]: pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 23441/231609 objects degraded (10.121%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4097460009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[180297]: from='client.56800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[180297]: pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 23441/231609 objects degraded (10.121%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4097460009' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:43:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[186706]: from='client.56800 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:43:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:54 smithi064 ceph-mon[186706]: pgmap v1476: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.7 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 23441/231609 objects degraded (10.121%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:43:54.661 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:43:54.662 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:43:54.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:43:54.662 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [] 2023-06-10T21:43:54.662 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:43:54.777 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:43:55.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[180297]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[180297]: osdmap e401: 8 total, 8 up, 4 in 2023-06-10T21:43:55.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[180297]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[186706]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[186706]: osdmap e401: 8 total, 8 up, 4 in 2023-06-10T21:43:55.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:55 smithi064 ceph-mon[186706]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:43:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:55 smithi152 ceph-mon[156819]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:55 smithi152 ceph-mon[156819]: osdmap e401: 8 total, 8 up, 4 in 2023-06-10T21:43:55.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:55 smithi152 ceph-mon[156819]: failed to encode map e401 with expected crc 2023-06-10T21:43:55.764 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1475,"stamp":"2023-06-10T21:43:52.174561+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5085403046,"num_objects":77613,"num_object_clones":0,"num_object_copies":232839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77613,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":142672,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3463,"num_bytes_recovered":224413710,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75892,"ondisk_log_size":75892,"up":267,"acting":254,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":16342572,"kb_used_data":13618444,"kb_used_omap":27,"kb_used_meta":2723684,"kb_avail":358638036,"statfs":{"total":383980142592,"available":367245348864,"internally_reserved":0,"allocated":13945286656,"data_stored":13941115778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2789053018},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-474939392,"num_objects":-7247,"num_object_clones":0,"num_object_copies":-21741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-843,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7247,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":843,"num_bytes_recovered":55246848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.615636"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":438,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557952+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:43:39.557952+0000","last_peered":"2023-06-10T21:43:39.557952+0000","last_clean":"2023-06-10T21:43:39.557952+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:43:39.557952+0000","last_undegraded":"2023-06-10T21:43:39.557952+0000","last_fullsized":"2023-06-10T21:43:39.557952+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.553788+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:43:39.553788+0000","last_peered":"2023-06-10T21:43:39.553788+0000","last_clean":"2023-06-10T21:43:39.553788+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:43:39.553788+0000","last_undegraded":"2023-06-10T21:43:39.553788+0000","last_fullsized":"2023-06-10T21:43:39.553788+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":330,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551326+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:43:39.551326+0000","last_peered":"2023-06-10T21:43:39.551326+0000","last_clean":"2023-06-10T21:43:39.551326+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:43:39.551326+0000","last_undegraded":"2023-06-10T21:43:39.551326+0000","last_fullsized":"2023-06-10T21:43:39.551326+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":284,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561683+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:43:39.561683+0000","last_peered":"2023-06-10T21:43:39.561683+0000","last_clean":"2023-06-10T21:43:39.561683+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:43:39.561683+0000","last_undegraded":"2023-06-10T21:43:39.561683+0000","last_fullsized":"2023-06-10T21:43:39.561683+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550172+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:43:39.550172+0000","last_peered":"2023-06-10T21:43:39.550172+0000","last_clean":"2023-06-10T21:43:39.550172+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:43:39.550172+0000","last_undegraded":"2023-06-10T21:43:39.550172+0000","last_fullsized":"2023-06-10T21:43:39.550172+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549687+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:43:39.549687+0000","last_peered":"2023-06-10T21:43:39.549687+0000","last_clean":"2023-06-10T21:43:39.549687+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:43:39.549687+0000","last_undegraded":"2023-06-10T21:43:39.549687+0000","last_fullsized":"2023-06-10T21:43:39.549687+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":264,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549590+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:43:39.549590+0000","last_peered":"2023-06-10T21:43:39.549590+0000","last_clean":"2023-06-10T21:43:39.549590+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:43:39.549590+0000","last_undegraded":"2023-06-10T21:43:39.549590+0000","last_fullsized":"2023-06-10T21:43:39.549590+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550386+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:43:39.550386+0000","last_peered":"2023-06-10T21:43:39.550386+0000","last_clean":"2023-06-10T21:43:39.550386+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:43:39.550386+0000","last_undegraded":"2023-06-10T21:43:39.550386+0000","last_fullsized":"2023-06-10T21:43:39.550386+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":72,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548813+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:43:39.548813+0000","last_peered":"2023-06-10T21:43:39.548813+0000","last_clean":"2023-06-10T21:43:39.548813+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:43:39.548813+0000","last_undegraded":"2023-06-10T21:43:39.548813+0000","last_fullsized":"2023-06-10T21:43:39.548813+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":149,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551267+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:43:39.551267+0000","last_peered":"2023-06-10T21:43:39.551267+0000","last_clean":"2023-06-10T21:43:39.551267+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:43:39.551267+0000","last_undegraded":"2023-06-10T21:43:39.551267+0000","last_fullsized":"2023-06-10T21:43:39.551267+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":156,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562949+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:43:39.562949+0000","last_peered":"2023-06-10T21:43:39.562949+0000","last_clean":"2023-06-10T21:43:39.562949+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:43:39.562949+0000","last_undegraded":"2023-06-10T21:43:39.562949+0000","last_fullsized":"2023-06-10T21:43:39.562949+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":398,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.558034+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:43:39.558034+0000","last_peered":"2023-06-10T21:43:39.558034+0000","last_clean":"2023-06-10T21:43:39.558034+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:43:39.558034+0000","last_undegraded":"2023-06-10T21:43:39.558034+0000","last_fullsized":"2023-06-10T21:43:39.558034+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":322,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561134+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:43:39.561134+0000","last_peered":"2023-06-10T21:43:39.561134+0000","last_clean":"2023-06-10T21:43:39.561134+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:43:39.561134+0000","last_undegraded":"2023-06-10T21:43:39.561134+0000","last_fullsized":"2023-06-10T21:43:39.561134+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":113,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548827+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:43:39.548827+0000","last_peered":"2023-06-10T21:43:39.548827+0000","last_clean":"2023-06-10T21:43:39.548827+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:43:39.548827+0000","last_undegraded":"2023-06-10T21:43:39.548827+0000","last_fullsized":"2023-06-10T21:43:39.548827+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":458,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562522+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:43:39.562522+0000","last_peered":"2023-06-10T21:43:39.562522+0000","last_clean":"2023-06-10T21:43:39.562522+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:43:39.562522+0000","last_undegraded":"2023-06-10T21:43:39.562522+0000","last_fullsized":"2023-06-10T21:43:39.562522+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":346,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552898+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:43:39.552898+0000","last_peered":"2023-06-10T21:43:39.552898+0000","last_clean":"2023-06-10T21:43:39.552898+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:43:39.552898+0000","last_undegraded":"2023-06-10T21:43:39.552898+0000","last_fullsized":"2023-06-10T21:43:39.552898+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":269,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562066+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:43:39.562066+0000","last_peered":"2023-06-10T21:43:39.562066+0000","last_clean":"2023-06-10T21:43:39.562066+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:43:39.562066+0000","last_undegraded":"2023-06-10T21:43:39.562066+0000","last_fullsized":"2023-06-10T21:43:39.562066+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549250+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:43:39.549250+0000","last_peered":"2023-06-10T21:43:39.549250+0000","last_clean":"2023-06-10T21:43:39.549250+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:43:39.549250+0000","last_undegraded":"2023-06-10T21:43:39.549250+0000","last_fullsized":"2023-06-10T21:43:39.549250+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":411,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.559444+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:43:39.559444+0000","last_peered":"2023-06-10T21:43:39.559444+0000","last_clean":"2023-06-10T21:43:39.559444+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:43:39.559444+0000","last_undegraded":"2023-06-10T21:43:39.559444+0000","last_fullsized":"2023-06-10T21:43:39.559444+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551383+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:43:39.551383+0000","last_peered":"2023-06-10T21:43:39.551383+0000","last_clean":"2023-06-10T21:43:39.551383+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:43:39.551383+0000","last_undegraded":"2023-06-10T21:43:39.551383+0000","last_fullsized":"2023-06-10T21:43:39.551383+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":423,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.545400+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:43:39.545400+0000","last_peered":"2023-06-10T21:43:39.545400+0000","last_clean":"2023-06-10T21:43:39.545400+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:43:39.545400+0000","last_undegraded":"2023-06-10T21:43:39.545400+0000","last_fullsized":"2023-06-10T21:43:39.545400+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550867+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:43:39.550867+0000","last_peered":"2023-06-10T21:43:39.550867+0000","last_clean":"2023-06-10T21:43:39.550867+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:43:39.550867+0000","last_undegraded":"2023-06-10T21:43:39.550867+0000","last_fullsized":"2023-06-10T21:43:39.550867+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"399'8966","reported_seq":9001,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:51.011950+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:43:51.011950+0000","last_peered":"2023-06-10T21:43:51.011950+0000","last_clean":"2023-06-10T21:43:51.011950+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:43:51.011950+0000","last_undegraded":"2023-06-10T21:43:51.011950+0000","last_fullsized":"2023-06-10T21:43:51.011950+0000","mapping_epoch":373,"log_start":"381'3900","ondisk_log_start":"381'3900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311033856,"num_objects":4746,"num_object_clones":0,"num_object_copies":14238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8966,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":191,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563557+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:43:39.563557+0000","last_peered":"2023-06-10T21:43:39.563557+0000","last_clean":"2023-06-10T21:43:39.563557+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:43:39.563557+0000","last_undegraded":"2023-06-10T21:43:39.563557+0000","last_fullsized":"2023-06-10T21:43:39.563557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.545199+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:43:39.545199+0000","last_peered":"2023-06-10T21:43:39.545199+0000","last_clean":"2023-06-10T21:43:39.545199+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:43:39.545199+0000","last_undegraded":"2023-06-10T21:43:39.545199+0000","last_fullsized":"2023-06-10T21:43:39.545199+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"399'8950","reported_seq":9002,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:49.183326+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:43:49.183326+0000","last_peered":"2023-06-10T21:43:49.183326+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:43:49.183326+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"382'4600","ondisk_log_start":"382'4600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4350,"log_dups_size":0,"ondisk_log_size":4350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320471040,"num_objects":4890,"num_object_clones":0,"num_object_copies":14670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8950,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":3000}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":191,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561904+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:43:39.561904+0000","last_peered":"2023-06-10T21:43:39.561904+0000","last_clean":"2023-06-10T21:43:39.561904+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:43:39.561904+0000","last_undegraded":"2023-06-10T21:43:39.561904+0000","last_fullsized":"2023-06-10T21:43:39.561904+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551411+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:43:39.551411+0000","last_peered":"2023-06-10T21:43:39.551411+0000","last_clean":"2023-06-10T21:43:39.551411+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:43:39.551411+0000","last_undegraded":"2023-06-10T21:43:39.551411+0000","last_fullsized":"2023-06-10T21:43:39.551411+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"399'8898","reported_seq":8964,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:47.476531+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:43:47.476531+0000","last_peered":"2023-06-10T21:43:47.476531+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:43:47.476531+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"383'4600","ondisk_log_start":"383'4600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4298,"log_dups_size":0,"ondisk_log_size":4298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747840,"num_objects":4940,"num_object_clones":0,"num_object_copies":14820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8898,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3012}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":478,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562922+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:43:39.562922+0000","last_peered":"2023-06-10T21:43:39.562922+0000","last_clean":"2023-06-10T21:43:39.562922+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:43:39.562922+0000","last_undegraded":"2023-06-10T21:43:39.562922+0000","last_fullsized":"2023-06-10T21:43:39.562922+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":162,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560557+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:43:39.560557+0000","last_peered":"2023-06-10T21:43:39.560557+0000","last_clean":"2023-06-10T21:43:39.560557+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:43:39.560557+0000","last_undegraded":"2023-06-10T21:43:39.560557+0000","last_fullsized":"2023-06-10T21:43:39.560557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"399'8868","reported_seq":8911,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:47.464279+0000","last_change":"2023-06-10T21:43:02.519446+0000","last_active":"2023-06-10T21:43:47.464279+0000","last_peered":"2023-06-10T21:43:47.464279+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:43:02.514777+0000","last_became_peered":"2023-06-10T21:43:02.514777+0000","last_unstale":"2023-06-10T21:43:47.464279+0000","last_undegraded":"2023-06-10T21:43:02.215416+0000","last_fullsized":"2023-06-10T21:43:02.198507+0000","mapping_epoch":391,"log_start":"382'4584","ondisk_log_start":"382'4584","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4284,"log_dups_size":0,"ondisk_log_size":4284,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:27:12.740821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323485696,"num_objects":4936,"num_object_clones":0,"num_object_copies":14808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8868,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3032}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549127+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:43:39.549127+0000","last_peered":"2023-06-10T21:43:39.549127+0000","last_clean":"2023-06-10T21:43:39.549127+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:43:39.549127+0000","last_undegraded":"2023-06-10T21:43:39.549127+0000","last_fullsized":"2023-06-10T21:43:39.549127+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552767+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:43:39.552767+0000","last_peered":"2023-06-10T21:43:39.552767+0000","last_clean":"2023-06-10T21:43:39.552767+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:43:39.552767+0000","last_undegraded":"2023-06-10T21:43:39.552767+0000","last_fullsized":"2023-06-10T21:43:39.552767+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"399'8967","reported_seq":9011,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.044553+0000","last_change":"2023-06-10T21:43:02.585044+0000","last_active":"2023-06-10T21:43:51.044553+0000","last_peered":"2023-06-10T21:43:51.044553+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:43:02.579281+0000","last_became_peered":"2023-06-10T21:43:02.579281+0000","last_unstale":"2023-06-10T21:43:51.044553+0000","last_undegraded":"2023-06-10T21:43:02.241138+0000","last_fullsized":"2023-06-10T21:43:02.196427+0000","mapping_epoch":391,"log_start":"382'3937","ondisk_log_start":"382'3937","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5030,"log_dups_size":0,"ondisk_log_size":5030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:58.809359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312016896,"num_objects":4761,"num_object_clones":0,"num_object_copies":14283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1894,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8967,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2867}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":56,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552709+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:43:39.552709+0000","last_peered":"2023-06-10T21:43:39.552709+0000","last_clean":"2023-06-10T21:43:39.552709+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:43:39.552709+0000","last_undegraded":"2023-06-10T21:43:39.552709+0000","last_fullsized":"2023-06-10T21:43:39.552709+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552809+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:43:39.552809+0000","last_peered":"2023-06-10T21:43:39.552809+0000","last_clean":"2023-06-10T21:43:39.552809+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:43:39.552809+0000","last_undegraded":"2023-06-10T21:43:39.552809+0000","last_fullsized":"2023-06-10T21:43:39.552809+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"399'8864","reported_seq":12756,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:51.016124+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:43:51.016124+0000","last_peered":"2023-06-10T21:43:51.016124+0000","last_clean":"2023-06-10T21:43:51.016124+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:43:51.016124+0000","last_undegraded":"2023-06-10T21:43:51.016124+0000","last_fullsized":"2023-06-10T21:43:51.016124+0000","mapping_epoch":395,"log_start":"381'3800","ondisk_log_start":"381'3800","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317718528,"num_objects":4848,"num_object_clones":0,"num_object_copies":14544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8864,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552898+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:43:39.552898+0000","last_peered":"2023-06-10T21:43:39.552898+0000","last_clean":"2023-06-10T21:43:39.552898+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:43:39.552898+0000","last_undegraded":"2023-06-10T21:43:39.552898+0000","last_fullsized":"2023-06-10T21:43:39.552898+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":436,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.558472+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:43:39.558472+0000","last_peered":"2023-06-10T21:43:39.558472+0000","last_clean":"2023-06-10T21:43:39.558472+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:43:39.558472+0000","last_undegraded":"2023-06-10T21:43:39.558472+0000","last_fullsized":"2023-06-10T21:43:39.558472+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"399'8695","reported_seq":8744,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:49.181663+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:43:49.181663+0000","last_peered":"2023-06-10T21:43:49.181663+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:43:49.181663+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"382'4400","ondisk_log_start":"382'4400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4295,"log_dups_size":0,"ondisk_log_size":4295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319619072,"num_objects":4877,"num_object_clones":0,"num_object_copies":14631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8695,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":3024}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":414,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557982+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:43:39.557982+0000","last_peered":"2023-06-10T21:43:39.557982+0000","last_clean":"2023-06-10T21:43:39.557982+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:43:39.557982+0000","last_undegraded":"2023-06-10T21:43:39.557982+0000","last_fullsized":"2023-06-10T21:43:39.557982+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551304+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:43:39.551304+0000","last_peered":"2023-06-10T21:43:39.551304+0000","last_clean":"2023-06-10T21:43:39.551304+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:43:39.551304+0000","last_undegraded":"2023-06-10T21:43:39.551304+0000","last_fullsized":"2023-06-10T21:43:39.551304+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"399'8890","reported_seq":8934,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.041630+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:43:51.041630+0000","last_peered":"2023-06-10T21:43:51.041630+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:43:51.041630+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"381'3874","ondisk_log_start":"381'3874","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5016,"log_dups_size":0,"ondisk_log_size":5016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313524224,"num_objects":4784,"num_object_clones":0,"num_object_copies":14352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8890,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2849}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550559+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:43:39.550559+0000","last_peered":"2023-06-10T21:43:39.550559+0000","last_clean":"2023-06-10T21:43:39.550559+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:43:39.550559+0000","last_undegraded":"2023-06-10T21:43:39.550559+0000","last_fullsized":"2023-06-10T21:43:39.550559+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":346,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550972+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:43:39.550972+0000","last_peered":"2023-06-10T21:43:39.550972+0000","last_clean":"2023-06-10T21:43:39.550972+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:43:39.550972+0000","last_undegraded":"2023-06-10T21:43:39.550972+0000","last_fullsized":"2023-06-10T21:43:39.550972+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"399'8863","reported_seq":8905,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.052458+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:43:51.052458+0000","last_peered":"2023-06-10T21:43:51.052458+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:43:51.052458+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"382'3819","ondisk_log_start":"382'3819","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181888,"num_objects":4733,"num_object_clones":0,"num_object_copies":14199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8863,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2804}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":182,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561212+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:43:39.561212+0000","last_peered":"2023-06-10T21:43:39.561212+0000","last_clean":"2023-06-10T21:43:39.561212+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:43:39.561212+0000","last_undegraded":"2023-06-10T21:43:39.561212+0000","last_fullsized":"2023-06-10T21:43:39.561212+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":63,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549098+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:43:39.549098+0000","last_peered":"2023-06-10T21:43:39.549098+0000","last_clean":"2023-06-10T21:43:39.549098+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:43:39.549098+0000","last_undegraded":"2023-06-10T21:43:39.549098+0000","last_fullsized":"2023-06-10T21:43:39.549098+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":66,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548363+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:43:39.548363+0000","last_peered":"2023-06-10T21:43:39.548363+0000","last_clean":"2023-06-10T21:43:39.548363+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:43:39.548363+0000","last_undegraded":"2023-06-10T21:43:39.548363+0000","last_fullsized":"2023-06-10T21:43:39.548363+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"399'8842","reported_seq":8896,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:49.162796+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:43:49.162796+0000","last_peered":"2023-06-10T21:43:49.162796+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:43:49.162796+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"382'4500","ondisk_log_start":"382'4500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4342,"log_dups_size":0,"ondisk_log_size":4342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318570520,"num_objects":4862,"num_object_clones":0,"num_object_copies":14586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4862,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8842,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2945}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":82,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548262+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:43:39.548262+0000","last_peered":"2023-06-10T21:43:39.548262+0000","last_clean":"2023-06-10T21:43:39.548262+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:43:39.548262+0000","last_undegraded":"2023-06-10T21:43:39.548262+0000","last_fullsized":"2023-06-10T21:43:39.548262+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548493+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:43:39.548493+0000","last_peered":"2023-06-10T21:43:39.548493+0000","last_clean":"2023-06-10T21:43:39.548493+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:43:39.548493+0000","last_undegraded":"2023-06-10T21:43:39.548493+0000","last_fullsized":"2023-06-10T21:43:39.548493+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":263,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560288+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:43:39.560288+0000","last_peered":"2023-06-10T21:43:39.560288+0000","last_clean":"2023-06-10T21:43:39.560288+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:43:39.560288+0000","last_undegraded":"2023-06-10T21:43:39.560288+0000","last_fullsized":"2023-06-10T21:43:39.560288+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"399'8812","reported_seq":8864,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:49.204771+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:43:49.204771+0000","last_peered":"2023-06-10T21:43:49.204771+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:43:49.204771+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"382'4500","ondisk_log_start":"382'4500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4312,"log_dups_size":0,"ondisk_log_size":4312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318898176,"num_objects":4866,"num_object_clones":0,"num_object_copies":14598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8812,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2993}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.557905+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:43:39.557905+0000","last_peered":"2023-06-10T21:43:39.557905+0000","last_clean":"2023-06-10T21:43:39.557905+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:43:39.557905+0000","last_undegraded":"2023-06-10T21:43:39.557905+0000","last_fullsized":"2023-06-10T21:43:39.557905+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":288,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:41.051903+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:43:41.051903+0000","last_peered":"2023-06-10T21:43:41.051903+0000","last_clean":"2023-06-10T21:43:41.051903+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:43:41.051903+0000","last_undegraded":"2023-06-10T21:43:41.051903+0000","last_fullsized":"2023-06-10T21:43:41.051903+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.549633+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:43:39.549633+0000","last_peered":"2023-06-10T21:43:39.549633+0000","last_clean":"2023-06-10T21:43:39.549633+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:43:39.549633+0000","last_undegraded":"2023-06-10T21:43:39.549633+0000","last_fullsized":"2023-06-10T21:43:39.549633+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"399'8949","reported_seq":8992,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.035470+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:43:51.035470+0000","last_peered":"2023-06-10T21:43:51.035470+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:43:51.035470+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"381'3900","ondisk_log_start":"381'3900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323289088,"num_objects":4933,"num_object_clones":0,"num_object_copies":14799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8949,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2969}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":73,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548750+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:43:39.548750+0000","last_peered":"2023-06-10T21:43:39.548750+0000","last_clean":"2023-06-10T21:43:39.548750+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:43:39.548750+0000","last_undegraded":"2023-06-10T21:43:39.548750+0000","last_fullsized":"2023-06-10T21:43:39.548750+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":599,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560534+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:43:39.560534+0000","last_peered":"2023-06-10T21:43:39.560534+0000","last_clean":"2023-06-10T21:43:39.560534+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:43:39.560534+0000","last_undegraded":"2023-06-10T21:43:39.560534+0000","last_fullsized":"2023-06-10T21:43:39.560534+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552799+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:43:39.552799+0000","last_peered":"2023-06-10T21:43:39.552799+0000","last_clean":"2023-06-10T21:43:39.552799+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:43:39.552799+0000","last_undegraded":"2023-06-10T21:43:39.552799+0000","last_fullsized":"2023-06-10T21:43:39.552799+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":282,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563065+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:43:39.563065+0000","last_peered":"2023-06-10T21:43:39.563065+0000","last_clean":"2023-06-10T21:43:39.563065+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:43:39.563065+0000","last_undegraded":"2023-06-10T21:43:39.563065+0000","last_fullsized":"2023-06-10T21:43:39.563065+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"399'8781","reported_seq":11755,"reported_epoch":399,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:49.201480+0000","last_change":"2023-06-10T21:43:32.135184+0000","last_active":"2023-06-10T21:43:49.201480+0000","last_peered":"2023-06-10T21:43:49.201480+0000","last_clean":"2023-06-10T21:42:30.256970+0000","last_became_active":"2023-06-10T21:43:02.545711+0000","last_became_peered":"2023-06-10T21:43:02.545711+0000","last_unstale":"2023-06-10T21:43:49.201480+0000","last_undegraded":"2023-06-10T21:43:02.227101+0000","last_fullsized":"2023-06-10T21:43:02.203404+0000","mapping_epoch":391,"log_start":"382'4400","ondisk_log_start":"382'4400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4381,"log_dups_size":0,"ondisk_log_size":4381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316211200,"num_objects":4825,"num_object_clones":0,"num_object_copies":14475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8781,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1462,"num_bytes_recovered":95813632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":4368}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":451,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560741+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:43:39.560741+0000","last_peered":"2023-06-10T21:43:39.560741+0000","last_clean":"2023-06-10T21:43:39.560741+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:43:39.560741+0000","last_undegraded":"2023-06-10T21:43:39.560741+0000","last_fullsized":"2023-06-10T21:43:39.560741+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550985+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:43:39.550985+0000","last_peered":"2023-06-10T21:43:39.550985+0000","last_clean":"2023-06-10T21:43:39.550985+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:43:39.550985+0000","last_undegraded":"2023-06-10T21:43:39.550985+0000","last_fullsized":"2023-06-10T21:43:39.550985+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":150,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:42.584806+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:43:42.584806+0000","last_peered":"2023-06-10T21:43:42.584806+0000","last_clean":"2023-06-10T21:43:42.584806+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:43:42.584806+0000","last_undegraded":"2023-06-10T21:43:42.584806+0000","last_fullsized":"2023-06-10T21:43:42.584806+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"399'8926","reported_seq":8962,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-06-10T21:43:51.038206+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:43:51.038206+0000","last_peered":"2023-06-10T21:43:51.038206+0000","last_clean":"2023-06-10T21:43:51.038206+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:43:51.038206+0000","last_undegraded":"2023-06-10T21:43:51.038206+0000","last_fullsized":"2023-06-10T21:43:51.038206+0000","mapping_epoch":373,"log_start":"381'3900","ondisk_log_start":"381'3900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315097088,"num_objects":4808,"num_object_clones":0,"num_object_copies":14424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8926,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":430,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562982+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:43:39.562982+0000","last_peered":"2023-06-10T21:43:39.562982+0000","last_clean":"2023-06-10T21:43:39.562982+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:43:39.562982+0000","last_undegraded":"2023-06-10T21:43:39.562982+0000","last_fullsized":"2023-06-10T21:43:39.562982+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":278,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560389+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:43:39.560389+0000","last_peered":"2023-06-10T21:43:39.560389+0000","last_clean":"2023-06-10T21:43:39.560389+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:43:39.560389+0000","last_undegraded":"2023-06-10T21:43:39.560389+0000","last_fullsized":"2023-06-10T21:43:39.560389+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548567+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:43:39.548567+0000","last_peered":"2023-06-10T21:43:39.548567+0000","last_clean":"2023-06-10T21:43:39.548567+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:43:39.548567+0000","last_undegraded":"2023-06-10T21:43:39.548567+0000","last_fullsized":"2023-06-10T21:43:39.548567+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"399'9140","reported_seq":9183,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.023918+0000","last_change":"2023-06-10T21:43:02.588633+0000","last_active":"2023-06-10T21:43:51.023918+0000","last_peered":"2023-06-10T21:43:51.023918+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:43:51.023918+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"382'4100","ondisk_log_start":"382'4100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5040,"log_dups_size":0,"ondisk_log_size":5040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324665344,"num_objects":4954,"num_object_clones":0,"num_object_copies":14862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1976,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9140,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2978}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":408,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550891+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:43:39.550891+0000","last_peered":"2023-06-10T21:43:39.550891+0000","last_clean":"2023-06-10T21:43:39.550891+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:43:39.550891+0000","last_undegraded":"2023-06-10T21:43:39.550891+0000","last_fullsized":"2023-06-10T21:43:39.550891+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":219,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.553201+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:43:39.553201+0000","last_peered":"2023-06-10T21:43:39.553201+0000","last_clean":"2023-06-10T21:43:39.553201+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:43:39.553201+0000","last_undegraded":"2023-06-10T21:43:39.553201+0000","last_fullsized":"2023-06-10T21:43:39.553201+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550739+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:43:39.550739+0000","last_peered":"2023-06-10T21:43:39.550739+0000","last_clean":"2023-06-10T21:43:39.550739+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:43:39.550739+0000","last_undegraded":"2023-06-10T21:43:39.550739+0000","last_fullsized":"2023-06-10T21:43:39.550739+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"399'9000","reported_seq":9051,"reported_epoch":399,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:43:51.035438+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:43:51.035438+0000","last_peered":"2023-06-10T21:43:51.035438+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:43:51.035438+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"381'3900","ondisk_log_start":"381'3900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316276736,"num_objects":4826,"num_object_clones":0,"num_object_copies":14478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9000,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2905}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":147,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.563636+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:43:39.563636+0000","last_peered":"2023-06-10T21:43:39.563636+0000","last_clean":"2023-06-10T21:43:39.563636+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:43:39.563636+0000","last_undegraded":"2023-06-10T21:43:39.563636+0000","last_fullsized":"2023-06-10T21:43:39.563636+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":261,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.561482+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:43:39.561482+0000","last_peered":"2023-06-10T21:43:39.561482+0000","last_clean":"2023-06-10T21:43:39.561482+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:43:39.561482+0000","last_undegraded":"2023-06-10T21:43:39.561482+0000","last_fullsized":"2023-06-10T21:43:39.561482+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.547986+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:43:39.547986+0000","last_peered":"2023-06-10T21:43:39.547986+0000","last_clean":"2023-06-10T21:43:39.547986+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:43:39.547986+0000","last_undegraded":"2023-06-10T21:43:39.547986+0000","last_fullsized":"2023-06-10T21:43:39.547986+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":98,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.562652+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:43:39.562652+0000","last_peered":"2023-06-10T21:43:39.562652+0000","last_clean":"2023-06-10T21:43:39.562652+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:43:39.562652+0000","last_undegraded":"2023-06-10T21:43:39.562652+0000","last_fullsized":"2023-06-10T21:43:39.562652+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":295,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.560781+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:43:39.560781+0000","last_peered":"2023-06-10T21:43:39.560781+0000","last_clean":"2023-06-10T21:43:39.560781+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:43:39.560781+0000","last_undegraded":"2023-06-10T21:43:39.560781+0000","last_fullsized":"2023-06-10T21:43:39.560781+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":61,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548949+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:43:39.548949+0000","last_peered":"2023-06-10T21:43:39.548949+0000","last_clean":"2023-06-10T21:43:39.548949+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:43:39.548949+0000","last_undegraded":"2023-06-10T21:43:39.548949+0000","last_fullsized":"2023-06-10T21:43:39.548949+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":302,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552865+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:43:39.552865+0000","last_peered":"2023-06-10T21:43:39.552865+0000","last_clean":"2023-06-10T21:43:39.552865+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:43:39.552865+0000","last_undegraded":"2023-06-10T21:43:39.552865+0000","last_fullsized":"2023-06-10T21:43:39.552865+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.548777+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:43:39.548777+0000","last_peered":"2023-06-10T21:43:39.548777+0000","last_clean":"2023-06-10T21:43:39.548777+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:43:39.548777+0000","last_undegraded":"2023-06-10T21:43:39.548777+0000","last_fullsized":"2023-06-10T21:43:39.548777+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":373,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.552928+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:43:39.552928+0000","last_peered":"2023-06-10T21:43:39.552928+0000","last_clean":"2023-06-10T21:43:39.552928+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:43:39.552928+0000","last_undegraded":"2023-06-10T21:43:39.552928+0000","last_fullsized":"2023-06-10T21:43:39.552928+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":52,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551465+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:43:39.551465+0000","last_peered":"2023-06-10T21:43:39.551465+0000","last_clean":"2023-06-10T21:43:39.551465+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:43:39.551465+0000","last_undegraded":"2023-06-10T21:43:39.551465+0000","last_fullsized":"2023-06-10T21:43:39.551465+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.551129+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:43:39.551129+0000","last_peered":"2023-06-10T21:43:39.551129+0000","last_clean":"2023-06-10T21:43:39.551129+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:43:39.551129+0000","last_undegraded":"2023-06-10T21:43:39.551129+0000","last_fullsized":"2023-06-10T21:43:39.551129+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":320,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550820+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:43:39.550820+0000","last_peered":"2023-06-10T21:43:39.550820+0000","last_clean":"2023-06-10T21:43:39.550820+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:43:39.550820+0000","last_undegraded":"2023-06-10T21:43:39.550820+0000","last_fullsized":"2023-06-10T21:43:39.550820+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":398,"state":"active+clean","last_fresh":"2023-06-10T21:43:39.550638+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:43:39.550638+0000","last_peered":"2023-06-10T21:43:39.550638+0000","last_clean":"2023-06-10T21:43:39.550638+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:43:39.550638+0000","last_undegraded":"2023-06-10T21:43:39.550638+0000","last_fullsized":"2023-06-10T21:43:39.550638+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5084807192,"num_objects":77589,"num_object_clones":0,"num_object_copies":232767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23441,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77589,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142411,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3388,"num_bytes_recovered":222035968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13807136768,"data_stored":13807124552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75697,"ondisk_log_size":75697,"up":48,"acting":35,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430761,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4379816,"kb_used_data":3464608,"kb_used_omap":0,"kb_used_meta":915200,"kb_avail":89365336,"statfs":{"total":95995035648,"available":91510104064,"internally_reserved":0,"allocated":3547758592,"data_stored":3547051808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":937164800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:42:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":2.0049999999999999,"15min":3.673},"min":{"1min":0.29299999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":1.4690000000000001,"5min":56.639000000000003,"15min":184.83699999999999},"last":0.621},{"interface":"front","average":{"1min":0.89000000000000001,"5min":2.411,"15min":3.9580000000000002},"min":{"1min":0.307,"5min":0.25,"15min":0.223},"max":{"1min":5.0149999999999997,"5min":56.628,"15min":185.58000000000001},"last":2.681}]},{"osd":1,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.1309999999999998,"15min":2.6059999999999999},"min":{"1min":0.33400000000000002,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":4.3540000000000001,"5min":38.078000000000003,"15min":87.417000000000002},"last":1.278},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6469999999999998,"15min":2.851},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":7.3710000000000004,"5min":38.341000000000001,"15min":87.438000000000002},"last":7.0709999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:43:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.9649999999999999,"15min":4.8200000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7959999999999998,"5min":75.219999999999999,"15min":172.33600000000001},"last":5.0199999999999996},{"interface":"front","average":{"1min":2,"5min":2.6970000000000001,"15min":4.7949999999999999},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":4.8799999999999999,"5min":75.108999999999995,"15min":172.185},"last":1.6899999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":3.1800000000000002,"15min":4.4429999999999996},"min":{"1min":0.443,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":7.5019999999999998,"5min":89.504999999999995,"15min":147.00700000000001},"last":3.2040000000000002},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.786,"15min":4.1669999999999998},"min":{"1min":0.47399999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":8.2889999999999997,"5min":90.215999999999994,"15min":146.93899999999999},"last":3}]},{"osd":4,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59199999999999997,"15min":0.49099999999999999},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":1.7,"5min":5.859,"15min":5.859},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.998,"15min":0.69299999999999995},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":5.1219999999999999,"5min":22.77,"15min":22.77},"last":2.6440000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:42:08 2023","interfaces":[{"interface":"back","average":{"1min":11.141999999999999,"5min":2.6240000000000001,"15min":1.3560000000000001},"min":{"1min":0.32000000000000001,"5min":0.252,"15min":0.252},"max":{"1min":174.13200000000001,"5min":174.13200000000001,"15min":174.13200000000001},"last":1.726},{"interface":"front","average":{"1min":11.669,"5min":2.806,"15min":1.302},"min":{"1min":0.38700000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":177.81200000000001,"5min":177.81200000000001,"15min":177.81200000000001},"last":2.706}]},{"osd":6,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.77300000000000002,"15min":0.63300000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7709999999999999,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.70499999999999996,"15min":0.57699999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":2.29,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":1.7569999999999999}]}]},{"osd":6,"up_from":0,"seq":180388626989,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566432,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020408,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075820,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4687060,"kb_used_data":3725580,"kb_used_omap":7,"kb_used_meta":961144,"kb_avail":89058092,"statfs":{"total":95995035648,"available":91195486208,"internally_reserved":0,"allocated":3814993920,"data_stored":3813669805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":984211537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:42:55 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":5.609,"15min":4.6210000000000004},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":7.069,"5min":171.33699999999999,"15min":171.33699999999999},"last":0.376},{"interface":"front","average":{"1min":1.4119999999999999,"5min":4.9340000000000002,"15min":4.5919999999999996},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.7590000000000003,"5min":171.56200000000001,"15min":171.56200000000001},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":4.4500000000000002,"15min":5.3940000000000001},"min":{"1min":0.23499999999999999,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":4.1449999999999996,"5min":85.709999999999994,"15min":171.286},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.4020000000000001,"5min":4.109,"15min":3.9550000000000001},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.0500000000000007,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:43:24 2023","interfaces":[{"interface":"back","average":{"1min":11.114000000000001,"5min":9.2509999999999994,"15min":6.2389999999999999},"min":{"1min":0.41599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":145.07599999999999,"5min":171.78299999999999,"15min":171.78299999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":11.140000000000001,"5min":8.5540000000000003,"15min":6.2969999999999997},"min":{"1min":0.39300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":144.99000000000001,"5min":171.65100000000001,"15min":171.65100000000001},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.343,"15min":2.7839999999999998},"min":{"1min":0.46700000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.681,"5min":9.9390000000000001,"15min":171.48699999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.1859999999999999,"15min":3.6680000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":4.673,"5min":15.544,"15min":171.18100000000001},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:41:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":3.613,"15min":2.282},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29699999999999999},"max":{"1min":5.7759999999999998,"5min":171.303,"15min":171.303},"last":75.896000000000001},{"interface":"front","average":{"1min":0.76000000000000001,"5min":3.5430000000000001,"15min":2.3279999999999998},"min":{"1min":0.38,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.1280000000000001,"5min":171.203,"15min":171.203},"last":76.337000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.5449999999999999,"15min":2.472},"min":{"1min":0.496,"5min":0.307,"15min":0.307},"max":{"1min":5.7220000000000004,"5min":9.9049999999999994,"15min":171.441},"last":0.67600000000000005},{"interface":"front","average":{"1min":1.8200000000000001,"5min":2.343,"15min":3.3370000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.1600000000000001,"5min":15.590999999999999,"15min":171.24600000000001},"last":0.626}]},{"osd":7,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.9370000000000001,"15min":4.4699999999999998},"min":{"1min":0.46500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.6560000000000006,"5min":18.16,"15min":219.131},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.835,"5min":1.4419999999999999,"15min":4.4370000000000003},"min":{"1min":0.503,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5789999999999997,"5min":9.8719999999999999,"15min":219.19},"last":0.57399999999999995}]}]},{"osd":3,"up_from":250,"seq":1073741824179,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4214736,"kb_used_data":3526920,"kb_used_omap":7,"kb_used_meta":687800,"kb_avail":89530416,"statfs":{"total":95995035648,"available":91679145984,"internally_reserved":0,"allocated":3611566080,"data_stored":3610125088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":704307219},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":2.1589999999999998,"15min":3.5760000000000001},"min":{"1min":0.23599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.1949999999999998,"5min":30.744,"15min":105.09},"last":2.77},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.4580000000000002,"15min":3.6379999999999999},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.6970000000000001,"5min":30.805,"15min":99.852999999999994},"last":2.081}]},{"osd":1,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.5310000000000001,"15min":3.5099999999999998},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.6459999999999999,"5min":21.704999999999998,"15min":94.766000000000005},"last":2.6920000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.484,"15min":3.4199999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23000000000000001},"max":{"1min":3.9399999999999999,"5min":28.695,"15min":94.691000000000003},"last":5.4109999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":4.6109999999999998,"15min":5.0359999999999996},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6609999999999996,"5min":231.07900000000001,"15min":231.07900000000001},"last":7.9649999999999999},{"interface":"front","average":{"1min":2.7050000000000001,"5min":5.3769999999999998,"15min":6.3760000000000003},"min":{"1min":0.35899999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":8.6489999999999991,"5min":231.58600000000001,"15min":231.58600000000001},"last":9.3930000000000007}]},{"osd":4,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.5249999999999999,"15min":1.4410000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.7300000000000004,"5min":10.368,"15min":13.417},"last":2.1469999999999998},{"interface":"front","average":{"1min":1.514,"5min":2.085,"15min":2.3199999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":6.5359999999999996,"5min":19.244,"15min":19.728000000000002},"last":2.7269999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:41:30 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.847,"15min":1.5489999999999999},"min":{"1min":0.42899999999999999,"5min":0.33100000000000002,"15min":0.32400000000000001},"max":{"1min":3.8559999999999999,"5min":17.298999999999999,"15min":17.298999999999999},"last":2.8420000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.4830000000000001,"15min":1.5049999999999999},"min":{"1min":0.36099999999999999,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":9.1210000000000004,"5min":10.414999999999999,"15min":13.470000000000001},"last":2.1139999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.1240000000000001,"15min":2.3679999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":6.6150000000000002,"5min":19.164999999999999,"15min":19.805},"last":2.8290000000000002},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.508,"15min":1.514},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7610000000000001,"5min":9.9719999999999995,"15min":17.327000000000002},"last":2.9129999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.2839999999999998,"15min":2.6339999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":6.5830000000000002,"5min":19.193000000000001,"15min":79.984999999999999},"last":2.7930000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.603,"15min":1.802},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.1749999999999998,"5min":9.9380000000000006,"15min":80.272999999999996},"last":2.879}]}]},{"osd":4,"up_from":0,"seq":128849019445,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245451,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3060960,"kb_used_data":2901336,"kb_used_omap":11,"kb_used_meta":159540,"kb_avail":90684192,"statfs":{"total":95995035648,"available":92860612608,"internally_reserved":0,"allocated":2970968064,"data_stored":2970269077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":163369462},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.1550000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.2569999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.129}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.4050000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3782344704,"data_stored":3782344704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3570995200,"data_stored":3570991128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2934640640,"data_stored":2934636568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3519156224,"data_stored":3519152152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:43:55.771 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:43:55.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:55.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:55.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:55.901 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:55.911 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:56.033 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:56.249 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:56.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:56.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:56.422 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:56.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: osdmap e402: 8 total, 8 up, 4 in 2023-06-10T21:43:56.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.545 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 921 op/s; 22869/222540 objects degraded (10.276%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 23441/231609 objects degraded (10.121%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.546 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: osdmap e402: 8 total, 8 up, 4 in 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 921 op/s; 22869/222540 objects degraded (10.276%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:43:56.547 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 23441/231609 objects degraded (10.121%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: osdmap e402: 8 total, 8 up, 4 in 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.557 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: failed to encode map e402 with expected crc 2023-06-10T21:43:56.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: pgmap v1479: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 921 op/s; 22869/222540 objects degraded (10.276%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:43:56.558 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 23441/231609 objects degraded (10.121%), 13 pgs degraded (PG_DEGRADED) 2023-06-10T21:43:56.749 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:56.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:56.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:56.943 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:57.048 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:57.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:57.165 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:57.231 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:57.450 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:57.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:57.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:57.638 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:57.742 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:43:57.796 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:57.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:57.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:57.805 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:43:57.904 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:43:57.948 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:58.046 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:43:58.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:58.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:43:58.154 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:43:58.208 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:43:58.208 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:43:58.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:148061 13.1 e402/373 13:8983a31a:::benchmark_data_smithi064_501823_object38059:head v 402'9269, mlcod=402'9267)", 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:58.203510+0000", 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0022991050000000001, 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023271849999999998, 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:58.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203510+0000", 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:58.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203510+0000", 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203508+0000", 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1831000000000001e-05 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203520+0000", 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7125e-05 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:58.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203537+0000", 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6210000000000004e-06 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203545+0000", 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8551999999999999e-05 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203593+0000", 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4006e-05 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.203617+0000", 2023-06-10T21:43:58.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107598 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:148064 13.c e402/391 13:363dde6b:::benchmark_data_smithi064_501823_object38062:head v 402'9267, mlcod=383'4742)", 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:58.204888+0000", 2023-06-10T21:43:58.215 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00092051999999999998, 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001016519, 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:58.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204888+0000", 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204888+0000", 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204887+0000", 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8639999999999996e-06 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204895+0000", 2023-06-10T21:43:58.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3145e-05 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204908+0000", 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9689999999999997e-06 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204915+0000", 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.359e-05 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204969+0000", 2023-06-10T21:43:58.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4393000000000001e-05 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.205003+0000", 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011508199999999999 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.221 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.46829.0:148058 13.e e402/391 ondisk, result = 0)", 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:58.204929+0000", 2023-06-10T21:43:58.222 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00087980100000000004, 2023-06-10T21:43:58.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00103837, 2023-06-10T21:43:58.223 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:58.223 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:43:58.223 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:58.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204929+0000", 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:58.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204929+0000", 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204928+0000", 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.207e-06 2023-06-10T21:43:58.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204934+0000", 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0532e-05 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204945+0000", 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4999999999999999e-06 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204950+0000", 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9918000000000003e-05 2023-06-10T21:43:58.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.205030+0000", 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.575e-06 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.205036+0000", 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010690400000000001 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.229 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:148058 13.e 13:701bc9d7:::benchmark_data_smithi064_501823_object38056:head [delete] snapc 0=[] ondisk+write+known_if_redirected e402)", 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:43:58.191364+0000", 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.014444859000000001, 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.014669632, 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:43:58.230 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 148058 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191364+0000", 2023-06-10T21:43:58.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191364+0000", 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191362+0000", 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4100000000000002e-06 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:43:58.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191370+0000", 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4625e-05 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191384+0000", 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7129999999999999e-06 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191391+0000", 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7199e-05 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191438+0000", 2023-06-10T21:43:58.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012991700000000001 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191568+0000", 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015834099999999999 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.191726+0000", 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023749999999999999 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:43:58.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.194101+0000", 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010244827 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.204346+0000", 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000694292 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.205040+0000", 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3053000000000002e-05 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:43:58.205093+0000", 2023-06-10T21:43:58.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.01372953 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 4 2023-06-10T21:43:58.240 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:43:58.241 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:58.377 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:58 smithi064 ceph-mon[186706]: pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 22869/222540 objects degraded (10.276%); 5.1 MiB/s, 82 objects/s recovering 2023-06-10T21:43:58.378 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:58 smithi064 ceph-mon[180297]: pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 22869/222540 objects degraded (10.276%); 5.1 MiB/s, 82 objects/s recovering 2023-06-10T21:43:58.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:43:58.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:58.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:43:58.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:43:58.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:58 smithi152 ceph-mon[156819]: pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 76 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 22869/222540 objects degraded (10.276%); 5.1 MiB/s, 82 objects/s recovering 2023-06-10T21:43:58.583 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:58.694 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:43:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:58.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:43:58.801 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:58.860 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:59.140 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:59.147 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:43:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:59.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:43:59.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:43:59.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3278522923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.295 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:43:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3278522923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:43:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:43:59.473 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3278522923' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:43:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:43:59.676 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:43:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:59.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:43:59.783 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:43:59.785 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:43:59.815 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:43:59.979 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:43:59.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:43:59.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:00.088 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:00.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:00.371 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:00.371 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:00.371 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: osdmap e403: 8 total, 8 up, 4 in 2023-06-10T21:44:00.372 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.372 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.372 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.372 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:00 smithi152 ceph-mon[156819]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 22869/219096 objects degraded (10.438%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:44:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: osdmap e403: 8 total, 8 up, 4 in 2023-06-10T21:44:00.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[186706]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 22869/219096 objects degraded (10.438%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:00.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: osdmap e403: 8 total, 8 up, 4 in 2023-06-10T21:44:00.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: failed to encode map e403 with expected crc 2023-06-10T21:44:00.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:00 smithi064 ceph-mon[180297]: pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.5 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 22869/219096 objects degraded (10.438%); 6.0 MiB/s, 95 objects/s recovering 2023-06-10T21:44:00.806 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:00.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:00.913 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:00.928 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:00.951 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:01.119 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:01.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:01.233 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:01.297 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:01.503 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:01.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:01.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:01 smithi152 ceph-mon[156819]: failed to encode map e403 with expected crc 2023-06-10T21:44:01.556 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22869/219096 objects degraded (10.438%), 12 pgs degraded (PG_DEGRADED) 2023-06-10T21:44:01.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:01 smithi064 ceph-mon[180297]: failed to encode map e403 with expected crc 2023-06-10T21:44:01.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22869/219096 objects degraded (10.438%), 12 pgs degraded (PG_DEGRADED) 2023-06-10T21:44:01.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:01 smithi064 ceph-mon[186706]: failed to encode map e403 with expected crc 2023-06-10T21:44:01.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22869/219096 objects degraded (10.438%), 12 pgs degraded (PG_DEGRADED) 2023-06-10T21:44:01.613 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:01.718 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:01.835 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:44:01.835 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:44:01.835 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:44:01.836 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:44:01.836 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:44:01.919 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:01.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:01.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:02.028 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:02.088 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:02.088 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22869/219096 objects degraded (10.438%), 12 pgs degraded","count":12},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2115,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":403,"num_osds":8,"num_up_osds":8,"osd_up_since":1686433380,"num_in_osds":4,"osd_in_since":1686433189,"num_remapped_pgs":12},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":77},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":11},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":73032,"data_bytes":4785182630,"bytes_used":15628410880,"bytes_avail":368351731712,"bytes_total":383980142592,"degraded_objects":22869,"degraded_total":219096,"degraded_ratio":0.10437890239894841,"recovering_objects_per_sec":95,"recovering_bytes_per_sec":6243435,"recovering_keys_per_sec":0,"num_objects_recovered":572,"num_bytes_recovered":37486592,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":694},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":29,"modified":"2023-06-10T21:44:00.179441+0000","services":{"osd":{"daemons":{"summary":"","3":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true},"9ab7ee0e-4ecd-48a4-8ef6-12d592a043b5":{"message":"Global Recovery Event (85s)\n [=======================.....] (remaining: 14s)","progress":0.85393255949020386,"add_to_ceph_s":true}}} 2023-06-10T21:44:02.102 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:02.145 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:02.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:02.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:02.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:02.275 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:02.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:02.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:02.421 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:02.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2392597764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:02.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:02 smithi152 ceph-mon[156819]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 748 op/s; 22597/210678 objects degraded (10.726%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:44:02.480 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2392597764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:02 smithi064 ceph-mon[186706]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 748 op/s; 22597/210678 objects degraded (10.726%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:44:02.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2392597764' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:02.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:02 smithi064 ceph-mon[180297]: pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 748 op/s; 22597/210678 objects degraded (10.726%); 3.1 MiB/s, 48 objects/s recovering 2023-06-10T21:44:02.838 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:02.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:44:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:02.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:02.948 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:02.951 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:44:02.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:03.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:02] "GET /metrics HTTP/1.1" 200 224002 "" "Prometheus/2.33.4" 2023-06-10T21:44:03.418 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:44:03.564 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:03.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:03.671 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:44:03.671 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:44:03.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:03.715 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:03.833 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:03.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:03.944 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:04.016 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:04.069 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:04.255 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:44:04.257 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:04.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:04.260 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:04.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:04.290 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:44:04.327 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:44:04.362 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:04.379 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:44:04.408 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:04.412 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:44:04.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:04.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:04.553 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:04.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:04.581 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:04.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:04.582 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:04.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:44:04.624 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:04.647 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:44:04.687 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.698 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.699 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:44:04.699 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:44:05.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:05 smithi064 ceph-mon[186706]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 22597/210678 objects degraded (10.726%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:44:05.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1481391287' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:05.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:05 smithi064 ceph-mon[180297]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 22597/210678 objects degraded (10.726%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:44:05.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1481391287' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:05.637 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:44:05.638 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:44:05.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:05 smithi152 ceph-mon[156819]: pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.3 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 22597/210678 objects degraded (10.726%); 2.7 MiB/s, 43 objects/s recovering 2023-06-10T21:44:05.660 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1481391287' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:06.489 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:06.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:06.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:44:06.597 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:06.624 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:06.690 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:06.964 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:06.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:07.072 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:07.119 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:07.203 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:07.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:07.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:07.357 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:07.430 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:07.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:07 smithi152 ceph-mon[156819]: pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:07.439 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:07 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22335/200154 objects degraded (11.159%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:07.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:07.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:07.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:07 smithi064 ceph-mon[180297]: pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:07.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:07 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22335/200154 objects degraded (11.159%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:07.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:07 smithi064 ceph-mon[186706]: pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:07.444 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:07 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22335/200154 objects degraded (11.159%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:07.541 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:07.609 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:07.692 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:07.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:07.803 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:07.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:07.973 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:07.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:08.085 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:08.113 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:08.157 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:08.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:08.217 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:44:08.243 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:08.243 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:08.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:08.331 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:08.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:08.444 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:08.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:08.557 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:08.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:08.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:44:09.184 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:09.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:09.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:09.294 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:09.376 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:09.457 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.465 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:09.466 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:09 smithi152 ceph-mon[156819]: osdmap e404: 8 total, 8 up, 4 in 2023-06-10T21:44:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:09.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:09.567 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:09.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:09.604 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[180297]: osdmap e404: 8 total, 8 up, 4 in 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: pgmap v1486: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.1 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 22335/200154 objects degraded (11.159%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: failed to encode map e404 with expected crc 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1977480670' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:09.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:09 smithi064 ceph-mon[186706]: osdmap e404: 8 total, 8 up, 4 in 2023-06-10T21:44:09.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:09.758 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:09.792 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:09.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:09.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:09.905 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:09.992 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:10.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:10.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:10.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:10.608 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:10.702 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:10.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:10.804 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:10.814 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:44:10.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:10.886 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:10.981 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:10.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:11.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:11.166 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:11.262 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:11.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:11.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:11.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:11 smithi064 ceph-mon[180297]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 741 op/s; 22335/196830 objects degraded (11.347%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:11.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22335/196830 objects degraded (11.347%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:11.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:11 smithi064 ceph-mon[186706]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 741 op/s; 22335/196830 objects degraded (11.347%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:11.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22335/196830 objects degraded (11.347%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:11.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:11 smithi152 ceph-mon[156819]: pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 4.0 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 741 op/s; 22335/196830 objects degraded (11.347%); 3.3 MiB/s, 53 objects/s recovering 2023-06-10T21:44:11.435 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22335/196830 objects degraded (11.347%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:11.440 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:11.543 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:11.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:11.653 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:11.689 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:11.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:44:12.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:12 smithi064 ceph-mon[186706]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:12.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:12 smithi064 ceph-mon[180297]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:12 smithi152 ceph-mon[156819]: from='client.36969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:13.152 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1488,"stamp":"2023-06-10T21:44:10.185435+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4298774438,"num_objects":65610,"num_object_clones":0,"num_object_copies":196830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22335,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65610,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":154675,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4569,"num_bytes_recovered":296896526,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75995,"ondisk_log_size":75995,"up":267,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":13908944,"kb_used_data":11139568,"kb_used_omap":27,"kb_used_meta":2769252,"kb_avail":361071664,"statfs":{"total":383980142592,"available":369737383936,"internally_reserved":0,"allocated":11406917632,"data_stored":11402725640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2835714650},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,8,7,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-486408192,"num_objects":-7422,"num_object_clones":0,"num_object_copies":-22266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-534,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7422,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":534,"num_bytes_recovered":34996224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006152"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":442,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.259458+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:43:55.259458+0000","last_peered":"2023-06-10T21:43:55.259458+0000","last_clean":"2023-06-10T21:43:55.259458+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:43:55.259458+0000","last_undegraded":"2023-06-10T21:43:55.259458+0000","last_fullsized":"2023-06-10T21:43:55.259458+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275261+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:43:55.275261+0000","last_peered":"2023-06-10T21:43:55.275261+0000","last_clean":"2023-06-10T21:43:55.275261+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:43:55.275261+0000","last_undegraded":"2023-06-10T21:43:55.275261+0000","last_fullsized":"2023-06-10T21:43:55.275261+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273187+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:43:55.273187+0000","last_peered":"2023-06-10T21:43:55.273187+0000","last_clean":"2023-06-10T21:43:55.273187+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:43:55.273187+0000","last_undegraded":"2023-06-10T21:43:55.273187+0000","last_fullsized":"2023-06-10T21:43:55.273187+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":289,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326338+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:43:59.326338+0000","last_peered":"2023-06-10T21:43:59.326338+0000","last_clean":"2023-06-10T21:43:59.326338+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:43:59.326338+0000","last_undegraded":"2023-06-10T21:43:59.326338+0000","last_fullsized":"2023-06-10T21:43:59.326338+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272673+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:43:55.272673+0000","last_peered":"2023-06-10T21:43:55.272673+0000","last_clean":"2023-06-10T21:43:55.272673+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:43:55.272673+0000","last_undegraded":"2023-06-10T21:43:55.272673+0000","last_fullsized":"2023-06-10T21:43:55.272673+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321328+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:43:59.321328+0000","last_peered":"2023-06-10T21:43:59.321328+0000","last_clean":"2023-06-10T21:43:59.321328+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:43:59.321328+0000","last_undegraded":"2023-06-10T21:43:59.321328+0000","last_fullsized":"2023-06-10T21:43:59.321328+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321169+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:43:59.321169+0000","last_peered":"2023-06-10T21:43:59.321169+0000","last_clean":"2023-06-10T21:43:59.321169+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:43:59.321169+0000","last_undegraded":"2023-06-10T21:43:59.321169+0000","last_fullsized":"2023-06-10T21:43:59.321169+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272828+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:43:55.272828+0000","last_peered":"2023-06-10T21:43:55.272828+0000","last_clean":"2023-06-10T21:43:55.272828+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:43:55.272828+0000","last_undegraded":"2023-06-10T21:43:55.272828+0000","last_fullsized":"2023-06-10T21:43:55.272828+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":77,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321718+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:43:59.321718+0000","last_peered":"2023-06-10T21:43:59.321718+0000","last_clean":"2023-06-10T21:43:59.321718+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:43:59.321718+0000","last_undegraded":"2023-06-10T21:43:59.321718+0000","last_fullsized":"2023-06-10T21:43:59.321718+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273138+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:43:55.273138+0000","last_peered":"2023-06-10T21:43:55.273138+0000","last_clean":"2023-06-10T21:43:55.273138+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:43:55.273138+0000","last_undegraded":"2023-06-10T21:43:55.273138+0000","last_fullsized":"2023-06-10T21:43:55.273138+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":161,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323522+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:43:59.323522+0000","last_peered":"2023-06-10T21:43:59.323522+0000","last_clean":"2023-06-10T21:43:59.323522+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:43:59.323522+0000","last_undegraded":"2023-06-10T21:43:59.323522+0000","last_fullsized":"2023-06-10T21:43:59.323522+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":402,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.259507+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:43:55.259507+0000","last_peered":"2023-06-10T21:43:55.259507+0000","last_clean":"2023-06-10T21:43:55.259507+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:43:55.259507+0000","last_undegraded":"2023-06-10T21:43:55.259507+0000","last_fullsized":"2023-06-10T21:43:55.259507+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":327,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323104+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:43:59.323104+0000","last_peered":"2023-06-10T21:43:59.323104+0000","last_clean":"2023-06-10T21:43:59.323104+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:43:59.323104+0000","last_undegraded":"2023-06-10T21:43:59.323104+0000","last_fullsized":"2023-06-10T21:43:59.323104+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":118,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320716+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:43:59.320716+0000","last_peered":"2023-06-10T21:43:59.320716+0000","last_clean":"2023-06-10T21:43:59.320716+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:43:59.320716+0000","last_undegraded":"2023-06-10T21:43:59.320716+0000","last_fullsized":"2023-06-10T21:43:59.320716+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":463,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321517+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:43:59.321517+0000","last_peered":"2023-06-10T21:43:59.321517+0000","last_clean":"2023-06-10T21:43:59.321517+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:43:59.321517+0000","last_undegraded":"2023-06-10T21:43:59.321517+0000","last_fullsized":"2023-06-10T21:43:59.321517+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":350,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273403+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:43:55.273403+0000","last_peered":"2023-06-10T21:43:55.273403+0000","last_clean":"2023-06-10T21:43:55.273403+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:43:55.273403+0000","last_undegraded":"2023-06-10T21:43:55.273403+0000","last_fullsized":"2023-06-10T21:43:55.273403+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":274,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326188+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:43:59.326188+0000","last_peered":"2023-06-10T21:43:59.326188+0000","last_clean":"2023-06-10T21:43:59.326188+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:43:59.326188+0000","last_undegraded":"2023-06-10T21:43:59.326188+0000","last_fullsized":"2023-06-10T21:43:59.326188+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":86,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322801+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:43:59.322801+0000","last_peered":"2023-06-10T21:43:59.322801+0000","last_clean":"2023-06-10T21:43:59.322801+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:43:59.322801+0000","last_undegraded":"2023-06-10T21:43:59.322801+0000","last_fullsized":"2023-06-10T21:43:59.322801+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":416,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321987+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:43:59.321987+0000","last_peered":"2023-06-10T21:43:59.321987+0000","last_clean":"2023-06-10T21:43:59.321987+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:43:59.321987+0000","last_undegraded":"2023-06-10T21:43:59.321987+0000","last_fullsized":"2023-06-10T21:43:59.321987+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275721+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:43:55.275721+0000","last_peered":"2023-06-10T21:43:55.275721+0000","last_clean":"2023-06-10T21:43:55.275721+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:43:55.275721+0000","last_undegraded":"2023-06-10T21:43:55.275721+0000","last_fullsized":"2023-06-10T21:43:55.275721+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":428,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.315119+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:43:59.315119+0000","last_peered":"2023-06-10T21:43:59.315119+0000","last_clean":"2023-06-10T21:43:59.315119+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:43:59.315119+0000","last_undegraded":"2023-06-10T21:43:59.315119+0000","last_fullsized":"2023-06-10T21:43:59.315119+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275396+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:43:55.275396+0000","last_peered":"2023-06-10T21:43:55.275396+0000","last_clean":"2023-06-10T21:43:55.275396+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:43:55.275396+0000","last_undegraded":"2023-06-10T21:43:55.275396+0000","last_fullsized":"2023-06-10T21:43:55.275396+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"403'9623","reported_seq":9663,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:05.997247+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:44:05.997247+0000","last_peered":"2023-06-10T21:44:05.997247+0000","last_clean":"2023-06-10T21:44:05.997247+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:44:05.997247+0000","last_undegraded":"2023-06-10T21:44:05.997247+0000","last_fullsized":"2023-06-10T21:44:05.997247+0000","mapping_epoch":373,"log_start":"383'4600","ondisk_log_start":"383'4600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267976704,"num_objects":4089,"num_object_clones":0,"num_object_copies":12267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9623,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":196,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325171+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:43:59.325171+0000","last_peered":"2023-06-10T21:43:59.325171+0000","last_clean":"2023-06-10T21:43:59.325171+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:43:59.325171+0000","last_undegraded":"2023-06-10T21:43:59.325171+0000","last_fullsized":"2023-06-10T21:43:59.325171+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.314983+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:43:59.314983+0000","last_peered":"2023-06-10T21:43:59.314983+0000","last_clean":"2023-06-10T21:43:59.314983+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:43:59.314983+0000","last_undegraded":"2023-06-10T21:43:59.314983+0000","last_fullsized":"2023-06-10T21:43:59.314983+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"404'9859","reported_seq":9916,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:09.102179+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:44:09.102179+0000","last_peered":"2023-06-10T21:44:09.102179+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:44:09.102179+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"386'5500","ondisk_log_start":"386'5500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4359,"log_dups_size":0,"ondisk_log_size":4359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260898816,"num_objects":3981,"num_object_clones":0,"num_object_copies":11943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9859,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2091}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":196,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326496+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:43:59.326496+0000","last_peered":"2023-06-10T21:43:59.326496+0000","last_clean":"2023-06-10T21:43:59.326496+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:43:59.326496+0000","last_undegraded":"2023-06-10T21:43:59.326496+0000","last_fullsized":"2023-06-10T21:43:59.326496+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273235+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:43:55.273235+0000","last_peered":"2023-06-10T21:43:55.273235+0000","last_clean":"2023-06-10T21:43:55.273235+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:43:55.273235+0000","last_undegraded":"2023-06-10T21:43:55.273235+0000","last_fullsized":"2023-06-10T21:43:55.273235+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"403'9676","reported_seq":9744,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.854319+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:44:05.854319+0000","last_peered":"2023-06-10T21:44:05.854319+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:44:05.854319+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"386'5300","ondisk_log_start":"386'5300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4376,"log_dups_size":0,"ondisk_log_size":4376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272760832,"num_objects":4162,"num_object_clones":0,"num_object_copies":12486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9676,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2234}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":483,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323602+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:43:59.323602+0000","last_peered":"2023-06-10T21:43:59.323602+0000","last_clean":"2023-06-10T21:43:59.323602+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:43:59.323602+0000","last_undegraded":"2023-06-10T21:43:59.323602+0000","last_fullsized":"2023-06-10T21:43:59.323602+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":167,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322869+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:43:59.322869+0000","last_peered":"2023-06-10T21:43:59.322869+0000","last_clean":"2023-06-10T21:43:59.322869+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:43:59.322869+0000","last_undegraded":"2023-06-10T21:43:59.322869+0000","last_fullsized":"2023-06-10T21:43:59.322869+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"403'9678","reported_seq":9723,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.850558+0000","last_change":"2023-06-10T21:43:02.519446+0000","last_active":"2023-06-10T21:44:05.850558+0000","last_peered":"2023-06-10T21:44:05.850558+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:43:02.514777+0000","last_became_peered":"2023-06-10T21:43:02.514777+0000","last_unstale":"2023-06-10T21:44:05.850558+0000","last_undegraded":"2023-06-10T21:43:02.215416+0000","last_fullsized":"2023-06-10T21:43:02.198507+0000","mapping_epoch":391,"log_start":"386'5384","ondisk_log_start":"386'5384","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4294,"log_dups_size":0,"ondisk_log_size":4294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:27:12.740821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270401536,"num_objects":4126,"num_object_clones":0,"num_object_copies":12378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9678,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2222}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":87,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321949+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:43:59.321949+0000","last_peered":"2023-06-10T21:43:59.321949+0000","last_clean":"2023-06-10T21:43:59.321949+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:43:59.321949+0000","last_undegraded":"2023-06-10T21:43:59.321949+0000","last_fullsized":"2023-06-10T21:43:59.321949+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.274207+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:43:55.274207+0000","last_peered":"2023-06-10T21:43:55.274207+0000","last_clean":"2023-06-10T21:43:55.274207+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:43:55.274207+0000","last_undegraded":"2023-06-10T21:43:55.274207+0000","last_fullsized":"2023-06-10T21:43:55.274207+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"403'9602","reported_seq":10949,"reported_epoch":403,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:06.002568+0000","last_change":"2023-06-10T21:43:53.697228+0000","last_active":"2023-06-10T21:44:06.002568+0000","last_peered":"2023-06-10T21:44:06.002568+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:43:02.579281+0000","last_became_peered":"2023-06-10T21:43:02.579281+0000","last_unstale":"2023-06-10T21:44:06.002568+0000","last_undegraded":"2023-06-10T21:43:02.241138+0000","last_fullsized":"2023-06-10T21:43:02.196427+0000","mapping_epoch":391,"log_start":"383'4537","ondisk_log_start":"383'4537","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5065,"log_dups_size":0,"ondisk_log_size":5065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:58.809359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270401536,"num_objects":4126,"num_object_clones":0,"num_object_copies":12378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":649,"num_bytes_recovered":42532864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2881}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":60,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.274157+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:43:55.274157+0000","last_peered":"2023-06-10T21:43:55.274157+0000","last_clean":"2023-06-10T21:43:55.274157+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:43:55.274157+0000","last_undegraded":"2023-06-10T21:43:55.274157+0000","last_fullsized":"2023-06-10T21:43:55.274157+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273298+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:43:55.273298+0000","last_peered":"2023-06-10T21:43:55.273298+0000","last_clean":"2023-06-10T21:43:55.273298+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:43:55.273298+0000","last_undegraded":"2023-06-10T21:43:55.273298+0000","last_fullsized":"2023-06-10T21:43:55.273298+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"403'9532","reported_seq":13428,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:05.988847+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:44:05.988847+0000","last_peered":"2023-06-10T21:44:05.988847+0000","last_clean":"2023-06-10T21:44:05.988847+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:44:05.988847+0000","last_undegraded":"2023-06-10T21:44:05.988847+0000","last_fullsized":"2023-06-10T21:44:05.988847+0000","mapping_epoch":395,"log_start":"382'4500","ondisk_log_start":"382'4500","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273940480,"num_objects":4180,"num_object_clones":0,"num_object_copies":12540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9532,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":223,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.274700+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:43:55.274700+0000","last_peered":"2023-06-10T21:43:55.274700+0000","last_clean":"2023-06-10T21:43:55.274700+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:43:55.274700+0000","last_undegraded":"2023-06-10T21:43:55.274700+0000","last_fullsized":"2023-06-10T21:43:55.274700+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":441,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320353+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:43:59.320353+0000","last_peered":"2023-06-10T21:43:59.320353+0000","last_clean":"2023-06-10T21:43:59.320353+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:43:59.320353+0000","last_undegraded":"2023-06-10T21:43:59.320353+0000","last_fullsized":"2023-06-10T21:43:59.320353+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"404'9645","reported_seq":9699,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:09.185263+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:44:09.185263+0000","last_peered":"2023-06-10T21:44:09.185263+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:44:09.185263+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"386'5300","ondisk_log_start":"386'5300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4345,"log_dups_size":0,"ondisk_log_size":4345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257359872,"num_objects":3927,"num_object_clones":0,"num_object_copies":11781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9645,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2074}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":418,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.260724+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:43:55.260724+0000","last_peered":"2023-06-10T21:43:55.260724+0000","last_clean":"2023-06-10T21:43:55.260724+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:43:55.260724+0000","last_undegraded":"2023-06-10T21:43:55.260724+0000","last_fullsized":"2023-06-10T21:43:55.260724+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275649+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:43:55.275649+0000","last_peered":"2023-06-10T21:43:55.275649+0000","last_clean":"2023-06-10T21:43:55.275649+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:43:55.275649+0000","last_undegraded":"2023-06-10T21:43:55.275649+0000","last_fullsized":"2023-06-10T21:43:55.275649+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"403'9503","reported_seq":9551,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.861425+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:44:05.861425+0000","last_peered":"2023-06-10T21:44:05.861425+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:44:05.861425+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"382'4474","ondisk_log_start":"382'4474","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5029,"log_dups_size":0,"ondisk_log_size":5029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273350656,"num_objects":4171,"num_object_clones":0,"num_object_copies":12513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9503,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2236}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272998+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:43:55.272998+0000","last_peered":"2023-06-10T21:43:55.272998+0000","last_clean":"2023-06-10T21:43:55.272998+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:43:55.272998+0000","last_undegraded":"2023-06-10T21:43:55.272998+0000","last_fullsized":"2023-06-10T21:43:55.272998+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":350,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275630+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:43:55.275630+0000","last_peered":"2023-06-10T21:43:55.275630+0000","last_clean":"2023-06-10T21:43:55.275630+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:43:55.275630+0000","last_undegraded":"2023-06-10T21:43:55.275630+0000","last_fullsized":"2023-06-10T21:43:55.275630+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"403'9502","reported_seq":9548,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.842262+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:44:05.842262+0000","last_peered":"2023-06-10T21:44:05.842262+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:44:05.842262+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"382'4419","ondisk_log_start":"382'4419","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268304384,"num_objects":4094,"num_object_clones":0,"num_object_copies":12282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9502,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2165}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":187,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323183+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:43:59.323183+0000","last_peered":"2023-06-10T21:43:59.323183+0000","last_clean":"2023-06-10T21:43:59.323183+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:43:59.323183+0000","last_undegraded":"2023-06-10T21:43:59.323183+0000","last_fullsized":"2023-06-10T21:43:59.323183+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":68,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322569+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:43:59.322569+0000","last_peered":"2023-06-10T21:43:59.322569+0000","last_clean":"2023-06-10T21:43:59.322569+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:43:59.322569+0000","last_undegraded":"2023-06-10T21:43:59.322569+0000","last_fullsized":"2023-06-10T21:43:59.322569+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.315267+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:43:59.315267+0000","last_peered":"2023-06-10T21:43:59.315267+0000","last_clean":"2023-06-10T21:43:59.315267+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:43:59.315267+0000","last_undegraded":"2023-06-10T21:43:59.315267+0000","last_fullsized":"2023-06-10T21:43:59.315267+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"404'9705","reported_seq":9764,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:09.196269+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:44:09.196269+0000","last_peered":"2023-06-10T21:44:09.196269+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:44:09.196269+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"386'5400","ondisk_log_start":"386'5400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4305,"log_dups_size":0,"ondisk_log_size":4305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262012952,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9705,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2082}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":87,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.315226+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:43:59.315226+0000","last_peered":"2023-06-10T21:43:59.315226+0000","last_clean":"2023-06-10T21:43:59.315226+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:43:59.315226+0000","last_undegraded":"2023-06-10T21:43:59.315226+0000","last_fullsized":"2023-06-10T21:43:59.315226+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":365,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320338+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:43:59.320338+0000","last_peered":"2023-06-10T21:43:59.320338+0000","last_clean":"2023-06-10T21:43:59.320338+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:43:59.320338+0000","last_undegraded":"2023-06-10T21:43:59.320338+0000","last_fullsized":"2023-06-10T21:43:59.320338+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":267,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.263903+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:43:55.263903+0000","last_peered":"2023-06-10T21:43:55.263903+0000","last_clean":"2023-06-10T21:43:55.263903+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:43:55.263903+0000","last_undegraded":"2023-06-10T21:43:55.263903+0000","last_fullsized":"2023-06-10T21:43:55.263903+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"404'9743","reported_seq":9802,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:09.208004+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:44:09.208004+0000","last_peered":"2023-06-10T21:44:09.208004+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:44:09.208004+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"386'5400","ondisk_log_start":"386'5400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4343,"log_dups_size":0,"ondisk_log_size":4343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9743,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2062}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":287,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.260641+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:43:55.260641+0000","last_peered":"2023-06-10T21:43:55.260641+0000","last_clean":"2023-06-10T21:43:55.260641+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:43:55.260641+0000","last_undegraded":"2023-06-10T21:43:55.260641+0000","last_fullsized":"2023-06-10T21:43:55.260641+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":292,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.259103+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:43:55.259103+0000","last_peered":"2023-06-10T21:43:55.259103+0000","last_clean":"2023-06-10T21:43:55.259103+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:43:55.259103+0000","last_undegraded":"2023-06-10T21:43:55.259103+0000","last_fullsized":"2023-06-10T21:43:55.259103+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321250+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:43:59.321250+0000","last_peered":"2023-06-10T21:43:59.321250+0000","last_clean":"2023-06-10T21:43:59.321250+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:43:59.321250+0000","last_undegraded":"2023-06-10T21:43:59.321250+0000","last_fullsized":"2023-06-10T21:43:59.321250+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"403'9600","reported_seq":9647,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.959463+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:44:05.959463+0000","last_peered":"2023-06-10T21:44:05.959463+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:44:05.959463+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"382'4500","ondisk_log_start":"382'4500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280625152,"num_objects":4282,"num_object_clones":0,"num_object_copies":12846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2318}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":78,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320668+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:43:59.320668+0000","last_peered":"2023-06-10T21:43:59.320668+0000","last_clean":"2023-06-10T21:43:59.320668+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:43:59.320668+0000","last_undegraded":"2023-06-10T21:43:59.320668+0000","last_fullsized":"2023-06-10T21:43:59.320668+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":604,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324070+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:43:59.324070+0000","last_peered":"2023-06-10T21:43:59.324070+0000","last_clean":"2023-06-10T21:43:59.324070+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:43:59.324070+0000","last_undegraded":"2023-06-10T21:43:59.324070+0000","last_fullsized":"2023-06-10T21:43:59.324070+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.274568+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:43:55.274568+0000","last_peered":"2023-06-10T21:43:55.274568+0000","last_clean":"2023-06-10T21:43:55.274568+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:43:55.274568+0000","last_undegraded":"2023-06-10T21:43:55.274568+0000","last_fullsized":"2023-06-10T21:43:55.274568+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323749+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:43:59.323749+0000","last_peered":"2023-06-10T21:43:59.323749+0000","last_clean":"2023-06-10T21:43:59.323749+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:43:59.323749+0000","last_undegraded":"2023-06-10T21:43:59.323749+0000","last_fullsized":"2023-06-10T21:43:59.323749+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"404'9629","reported_seq":13531,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:09.209581+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:44:09.209581+0000","last_peered":"2023-06-10T21:44:09.209581+0000","last_clean":"2023-06-10T21:44:09.209581+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:44:09.209581+0000","last_undegraded":"2023-06-10T21:44:09.209581+0000","last_fullsized":"2023-06-10T21:44:09.209581+0000","mapping_epoch":401,"log_start":"386'5300","ondisk_log_start":"386'5300","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4329,"log_dups_size":0,"ondisk_log_size":4329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260636672,"num_objects":3977,"num_object_clones":0,"num_object_copies":11931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9629,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":456,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324143+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:43:59.324143+0000","last_peered":"2023-06-10T21:43:59.324143+0000","last_clean":"2023-06-10T21:43:59.324143+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:43:59.324143+0000","last_undegraded":"2023-06-10T21:43:59.324143+0000","last_fullsized":"2023-06-10T21:43:59.324143+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272909+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:43:55.272909+0000","last_peered":"2023-06-10T21:43:55.272909+0000","last_clean":"2023-06-10T21:43:55.272909+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:43:55.272909+0000","last_undegraded":"2023-06-10T21:43:55.272909+0000","last_fullsized":"2023-06-10T21:43:55.272909+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":154,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.262557+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:43:55.262557+0000","last_peered":"2023-06-10T21:43:55.262557+0000","last_clean":"2023-06-10T21:43:55.262557+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:43:55.262557+0000","last_undegraded":"2023-06-10T21:43:55.262557+0000","last_fullsized":"2023-06-10T21:43:55.262557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"403'9622","reported_seq":9662,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:05.842156+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:44:05.842156+0000","last_peered":"2023-06-10T21:44:05.842156+0000","last_clean":"2023-06-10T21:44:05.842156+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:44:05.842156+0000","last_undegraded":"2023-06-10T21:44:05.842156+0000","last_fullsized":"2023-06-10T21:44:05.842156+0000","mapping_epoch":373,"log_start":"383'4600","ondisk_log_start":"383'4600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5022,"log_dups_size":0,"ondisk_log_size":5022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269484032,"num_objects":4112,"num_object_clones":0,"num_object_copies":12336,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4112,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9622,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":435,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323680+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:43:59.323680+0000","last_peered":"2023-06-10T21:43:59.323680+0000","last_clean":"2023-06-10T21:43:59.323680+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:43:59.323680+0000","last_undegraded":"2023-06-10T21:43:59.323680+0000","last_fullsized":"2023-06-10T21:43:59.323680+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":282,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.264000+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:43:55.264000+0000","last_peered":"2023-06-10T21:43:55.264000+0000","last_clean":"2023-06-10T21:43:55.264000+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:43:55.264000+0000","last_undegraded":"2023-06-10T21:43:55.264000+0000","last_fullsized":"2023-06-10T21:43:55.264000+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":208,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320421+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:43:59.320421+0000","last_peered":"2023-06-10T21:43:59.320421+0000","last_clean":"2023-06-10T21:43:59.320421+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:43:59.320421+0000","last_undegraded":"2023-06-10T21:43:59.320421+0000","last_fullsized":"2023-06-10T21:43:59.320421+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"403'9812","reported_seq":9860,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:06.000684+0000","last_change":"2023-06-10T21:43:02.588633+0000","last_active":"2023-06-10T21:44:06.000684+0000","last_peered":"2023-06-10T21:44:06.000684+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:44:06.000684+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"383'4800","ondisk_log_start":"383'4800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5012,"log_dups_size":0,"ondisk_log_size":5012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280625152,"num_objects":4282,"num_object_clones":0,"num_object_copies":12846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1976,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9812,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":2306}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":413,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324878+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:43:59.324878+0000","last_peered":"2023-06-10T21:43:59.324878+0000","last_clean":"2023-06-10T21:43:59.324878+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:43:59.324878+0000","last_undegraded":"2023-06-10T21:43:59.324878+0000","last_fullsized":"2023-06-10T21:43:59.324878+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":224,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324796+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:43:59.324796+0000","last_peered":"2023-06-10T21:43:59.324796+0000","last_clean":"2023-06-10T21:43:59.324796+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:43:59.324796+0000","last_undegraded":"2023-06-10T21:43:59.324796+0000","last_fullsized":"2023-06-10T21:43:59.324796+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272980+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:43:55.272980+0000","last_peered":"2023-06-10T21:43:55.272980+0000","last_clean":"2023-06-10T21:43:55.272980+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:43:55.272980+0000","last_undegraded":"2023-06-10T21:43:55.272980+0000","last_fullsized":"2023-06-10T21:43:55.272980+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"403'9683","reported_seq":9738,"reported_epoch":403,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:05.953340+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:44:05.953340+0000","last_peered":"2023-06-10T21:44:05.953340+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:44:05.953340+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"383'4600","ondisk_log_start":"383'4600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5083,"log_dups_size":0,"ondisk_log_size":5083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9683,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2222}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":152,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325182+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:43:59.325182+0000","last_peered":"2023-06-10T21:43:59.325182+0000","last_clean":"2023-06-10T21:43:59.325182+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:43:59.325182+0000","last_undegraded":"2023-06-10T21:43:59.325182+0000","last_fullsized":"2023-06-10T21:43:59.325182+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":266,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326165+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:43:59.326165+0000","last_peered":"2023-06-10T21:43:59.326165+0000","last_clean":"2023-06-10T21:43:59.326165+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:43:59.326165+0000","last_undegraded":"2023-06-10T21:43:59.326165+0000","last_fullsized":"2023-06-10T21:43:59.326165+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.318160+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:43:59.318160+0000","last_peered":"2023-06-10T21:43:59.318160+0000","last_clean":"2023-06-10T21:43:59.318160+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:43:59.318160+0000","last_undegraded":"2023-06-10T21:43:59.318160+0000","last_fullsized":"2023-06-10T21:43:59.318160+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321672+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:43:59.321672+0000","last_peered":"2023-06-10T21:43:59.321672+0000","last_clean":"2023-06-10T21:43:59.321672+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:43:59.321672+0000","last_undegraded":"2023-06-10T21:43:59.321672+0000","last_fullsized":"2023-06-10T21:43:59.321672+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":299,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.264092+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:43:55.264092+0000","last_peered":"2023-06-10T21:43:55.264092+0000","last_clean":"2023-06-10T21:43:55.264092+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:43:55.264092+0000","last_undegraded":"2023-06-10T21:43:55.264092+0000","last_fullsized":"2023-06-10T21:43:55.264092+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":66,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321795+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:43:59.321795+0000","last_peered":"2023-06-10T21:43:59.321795+0000","last_clean":"2023-06-10T21:43:59.321795+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:43:59.321795+0000","last_undegraded":"2023-06-10T21:43:59.321795+0000","last_fullsized":"2023-06-10T21:43:59.321795+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":307,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.893880+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:43:55.893880+0000","last_peered":"2023-06-10T21:43:55.893880+0000","last_clean":"2023-06-10T21:43:55.893880+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:43:55.893880+0000","last_undegraded":"2023-06-10T21:43:55.893880+0000","last_fullsized":"2023-06-10T21:43:55.893880+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":89,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322498+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:43:59.322498+0000","last_peered":"2023-06-10T21:43:59.322498+0000","last_clean":"2023-06-10T21:43:59.322498+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:43:59.322498+0000","last_undegraded":"2023-06-10T21:43:59.322498+0000","last_fullsized":"2023-06-10T21:43:59.322498+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":378,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.893804+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:43:55.893804+0000","last_peered":"2023-06-10T21:43:55.893804+0000","last_clean":"2023-06-10T21:43:55.893804+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:43:55.893804+0000","last_undegraded":"2023-06-10T21:43:55.893804+0000","last_fullsized":"2023-06-10T21:43:55.893804+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":56,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.275802+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:43:55.275802+0000","last_peered":"2023-06-10T21:43:55.275802+0000","last_clean":"2023-06-10T21:43:55.275802+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:43:55.275802+0000","last_undegraded":"2023-06-10T21:43:55.275802+0000","last_fullsized":"2023-06-10T21:43:55.275802+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.272995+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:43:55.272995+0000","last_peered":"2023-06-10T21:43:55.272995+0000","last_clean":"2023-06-10T21:43:55.272995+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:43:55.272995+0000","last_undegraded":"2023-06-10T21:43:55.272995+0000","last_fullsized":"2023-06-10T21:43:55.272995+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":325,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321561+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:43:59.321561+0000","last_peered":"2023-06-10T21:43:59.321561+0000","last_clean":"2023-06-10T21:43:59.321561+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:43:59.321561+0000","last_undegraded":"2023-06-10T21:43:59.321561+0000","last_fullsized":"2023-06-10T21:43:59.321561+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-06-10T21:43:55.273068+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:43:55.273068+0000","last_peered":"2023-06-10T21:43:55.273068+0000","last_clean":"2023-06-10T21:43:55.273068+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:43:55.273068+0000","last_undegraded":"2023-06-10T21:43:55.273068+0000","last_fullsized":"2023-06-10T21:43:55.273068+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4298178584,"num_objects":65586,"num_object_clones":0,"num_object_copies":196758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22335,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":65586,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":154414,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4494,"num_bytes_recovered":294518784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11320504320,"data_stored":11320492104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75800,"ondisk_log_size":75800,"up":48,"acting":36,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430765,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3772816,"kb_used_data":2830344,"kb_used_omap":0,"kb_used_meta":942400,"kb_avail":89972336,"statfs":{"total":95995035648,"available":92131672064,"internally_reserved":0,"allocated":2898272256,"data_stored":2897564901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":965017600},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.3089999999999999,"15min":3.1179999999999999},"min":{"1min":0.28599999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.2699999999999996,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.372},{"interface":"front","average":{"1min":1.244,"5min":1.552,"15min":3.371},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":3.0960000000000001,"5min":22.462,"15min":185.58000000000001},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.1309999999999998,"15min":2.6059999999999999},"min":{"1min":0.33400000000000002,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":4.3540000000000001,"5min":38.078000000000003,"15min":87.417000000000002},"last":8.1500000000000004},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6469999999999998,"15min":2.851},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":7.3710000000000004,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:43:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.9649999999999999,"15min":4.8200000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7959999999999998,"5min":75.219999999999999,"15min":172.33600000000001},"last":10.666},{"interface":"front","average":{"1min":2,"5min":2.6970000000000001,"15min":4.7949999999999999},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":4.8799999999999999,"5min":75.108999999999995,"15min":172.185},"last":10.869999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":3.1800000000000002,"15min":4.4429999999999996},"min":{"1min":0.443,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":7.5019999999999998,"5min":89.504999999999995,"15min":147.00700000000001},"last":8.7420000000000009},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.786,"15min":4.1669999999999998},"min":{"1min":0.47399999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":8.2889999999999997,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59199999999999997,"15min":0.49099999999999999},"min":{"1min":0.27500000000000002,"5min":0.27400000000000002,"15min":0.249},"max":{"1min":1.7,"5min":5.859,"15min":5.859},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.0489999999999999,"5min":0.998,"15min":0.69299999999999995},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":5.1219999999999999,"5min":22.77,"15min":22.77},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:44:04 2023","interfaces":[{"interface":"back","average":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":131.619,"5min":131.619,"15min":131.619},"last":0.40100000000000002},{"interface":"front","average":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":134.928,"5min":134.928,"15min":134.928},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:43:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.77300000000000002,"15min":0.63300000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.7709999999999999,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.70499999999999996,"15min":0.57699999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":2.29,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.439}]}]},{"osd":6,"up_from":0,"seq":180388626993,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566435,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020412,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075823,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4171068,"kb_used_data":3195956,"kb_used_omap":7,"kb_used_meta":975096,"kb_avail":89574084,"statfs":{"total":95995035648,"available":91723862016,"internally_reserved":0,"allocated":3272658944,"data_stored":3271326740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":998498385},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.903,"15min":4.4560000000000004},"min":{"1min":0.27500000000000002,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":8.5899999999999999,"5min":83.733000000000004,"15min":171.33699999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.673,"5min":2.4100000000000001,"15min":4.3440000000000003},"min":{"1min":0.255,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.9400000000000004,"5min":82.322999999999993,"15min":171.56200000000001},"last":1.3839999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":4.4500000000000002,"15min":5.3940000000000001},"min":{"1min":0.23499999999999999,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":4.1449999999999996,"5min":85.709999999999994,"15min":171.286},"last":5.2370000000000001},{"interface":"front","average":{"1min":2.4020000000000001,"5min":4.109,"15min":3.9550000000000001},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.0500000000000007,"5min":85.501999999999995,"15min":147.81299999999999},"last":1.1879999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:43:24 2023","interfaces":[{"interface":"back","average":{"1min":11.114000000000001,"5min":9.2509999999999994,"15min":6.2389999999999999},"min":{"1min":0.41599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":145.07599999999999,"5min":171.78299999999999,"15min":171.78299999999999},"last":1.337},{"interface":"front","average":{"1min":11.140000000000001,"5min":8.5540000000000003,"15min":6.2969999999999997},"min":{"1min":0.39300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":144.99000000000001,"5min":171.65100000000001,"15min":171.65100000000001},"last":1.431}]},{"osd":4,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.343,"15min":2.7839999999999998},"min":{"1min":0.46700000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.681,"5min":9.9390000000000001,"15min":171.48699999999999},"last":1.4139999999999999},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.1859999999999999,"15min":3.6680000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":4.673,"5min":15.544,"15min":171.18100000000001},"last":4.8890000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:44:02 2023","interfaces":[{"interface":"back","average":{"1min":6.9820000000000002,"5min":6.9820000000000002,"15min":6.9820000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":75.896000000000001,"5min":75.896000000000001,"15min":75.896000000000001},"last":1.6639999999999999},{"interface":"front","average":{"1min":6.3419999999999996,"5min":6.3419999999999996,"15min":6.3419999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":76.337000000000003,"5min":76.337000000000003,"15min":76.337000000000003},"last":1.452}]},{"osd":6,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.5449999999999999,"15min":2.472},"min":{"1min":0.496,"5min":0.307,"15min":0.307},"max":{"1min":5.7220000000000004,"5min":9.9049999999999994,"15min":171.441},"last":4.6509999999999998},{"interface":"front","average":{"1min":1.8200000000000001,"5min":2.343,"15min":3.3370000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.1600000000000001,"5min":15.590999999999999,"15min":171.24600000000001},"last":4.9199999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.9370000000000001,"15min":4.4699999999999998},"min":{"1min":0.46500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.6560000000000006,"5min":18.16,"15min":219.131},"last":6.6749999999999998},{"interface":"front","average":{"1min":1.835,"5min":1.4419999999999999,"15min":4.4370000000000003},"min":{"1min":0.503,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5789999999999997,"5min":9.8719999999999999,"15min":219.19},"last":4.3620000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824183,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3609148,"kb_used_data":2925428,"kb_used_omap":7,"kb_used_meta":683704,"kb_avail":90136004,"statfs":{"total":95995035648,"available":92299268096,"internally_reserved":0,"allocated":2995638272,"data_stored":2994185397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":700112915},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,7,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":2.1589999999999998,"15min":3.5760000000000001},"min":{"1min":0.23599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.1949999999999998,"5min":30.744,"15min":105.09},"last":3.5880000000000001},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.4580000000000002,"15min":3.6379999999999999},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.6970000000000001,"5min":30.805,"15min":99.852999999999994},"last":0.63900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.5310000000000001,"15min":3.5099999999999998},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.6459999999999999,"5min":21.704999999999998,"15min":94.766000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.484,"15min":3.4199999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23000000000000001},"max":{"1min":3.9399999999999999,"5min":28.695,"15min":94.691000000000003},"last":3.7200000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":4.6109999999999998,"15min":5.0359999999999996},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6609999999999996,"5min":231.07900000000001,"15min":231.07900000000001},"last":2.2210000000000001},{"interface":"front","average":{"1min":2.7050000000000001,"5min":5.3769999999999998,"15min":6.3760000000000003},"min":{"1min":0.35899999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":8.6489999999999991,"5min":231.58600000000001,"15min":231.58600000000001},"last":1.125}]},{"osd":4,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.5249999999999999,"15min":1.4410000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.7300000000000004,"5min":10.368,"15min":13.417},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.514,"5min":2.085,"15min":2.3199999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":6.5359999999999996,"5min":19.244,"15min":19.728000000000002},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:44:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.532,"5min":6.532,"15min":6.532},"last":3.6269999999999998},{"interface":"front","average":{"1min":2.7040000000000002,"5min":2.7040000000000002,"15min":2.7040000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.1240000000000001,"15min":2.3679999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":6.6150000000000002,"5min":19.164999999999999,"15min":19.805},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.508,"15min":1.514},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7610000000000001,"5min":9.9719999999999995,"15min":17.327000000000002},"last":3.694}]},{"osd":7,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.2839999999999998,"15min":2.6339999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":6.5830000000000002,"5min":19.193000000000001,"15min":79.984999999999999},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.603,"15min":1.802},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.1749999999999998,"5min":9.9380000000000006,"15min":80.272999999999996},"last":3.6629999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019449,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245455,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2355912,"kb_used_data":2187840,"kb_used_omap":11,"kb_used_meta":168052,"kb_avail":91389240,"statfs":{"total":95995035648,"available":93582581760,"internally_reserved":0,"allocated":2240348160,"data_stored":2239648602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":172085750},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":2.0800000000000001,"15min":2.0800000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":2.5019999999999998},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.266,"5min":4.266,"15min":4.266},"last":0.72799999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.54,"5min":7.54,"15min":7.54},"last":1.7789999999999999},{"interface":"front","average":{"1min":2.282,"5min":2.282,"15min":2.282},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.7989999999999995,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.81399999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":1.8620000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.306,"5min":2.306,"15min":2.306},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.641,"5min":12.641,"15min":12.641},"last":0.59999999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":8.2330000000000005,"5min":8.2330000000000005,"15min":8.2330000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":133.07499999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":2.9359999999999999},{"interface":"front","average":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":7.8949999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":133.03800000000001,"5min":133.03800000000001,"15min":133.03800000000001},"last":3.7080000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.173,"5min":5.173,"15min":5.173},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.3079999999999998,"15min":2.3079999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.476,"5min":7.476,"15min":7.476},"last":0.629}]},{"osd":6,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.8879999999999999,"15min":1.8879999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.345,"5min":1.345,"15min":1.345},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":1.3440000000000001},{"interface":"front","average":{"1min":2.891,"5min":2.891,"15min":2.891},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.4160000000000004,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":2.903}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3263692800,"data_stored":3263692800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2988183552,"data_stored":2988179480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2196836352,"data_stored":2196832280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2871791616,"data_stored":2871787544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:44:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:12] "GET /metrics HTTP/1.1" 200 224020 "" "Prometheus/2.33.4" 2023-06-10T21:44:13.192 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:13.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:13.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:13.367 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:13.368 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:13.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:13 smithi064 ceph-mon[186706]: pgmap v1489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:13.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:13 smithi064 ceph-mon[180297]: pgmap v1489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:13.460 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:13.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:13.571 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:13.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:13.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:13 smithi152 ceph-mon[156819]: pgmap v1489: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:13.988 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:44:14.030 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:44:14.030 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:44:14.030 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:44:14.031 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:44:14.036 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:44:14.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:14.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:14.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:14.078 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:44:14.119 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:44:14.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:14.153 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:44:14.186 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:44:14.221 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:14.227 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:44:14.267 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:14 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/532081187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:14.268 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:14 smithi064 ceph-mon[180297]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:14.268 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:44:14.286 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.286 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.286 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.287 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:44:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:14 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/532081187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:14.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:14 smithi064 ceph-mon[186706]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:14 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/532081187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:14.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:14 smithi152 ceph-mon[156819]: pgmap v1490: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.8 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 22033/188472 objects degraded (11.690%); 3.5 MiB/s, 56 objects/s recovering 2023-06-10T21:44:16.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:44:16.391 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:44:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:16 smithi064 ceph-mon[180297]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 22033/188472 objects degraded (11.690%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:16 smithi064 ceph-mon[186706]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 22033/188472 objects degraded (11.690%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:16.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:16 smithi152 ceph-mon[156819]: pgmap v1491: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:16.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 22033/188472 objects degraded (11.690%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:17.225 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:17.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:17.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:17.398 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:17.427 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:17.475 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:18.008 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:18.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:18.119 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:18.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:18.321 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:18.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:18.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:18.497 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:18.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:18.504 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:18 smithi064 ceph-mon[180297]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:18.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:18.504 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:18 smithi064 ceph-mon[186706]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:18.601 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:18.616 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:18.616 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:18 smithi152 ceph-mon[156819]: pgmap v1492: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.6 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 793 op/s; 21675/176355 objects degraded (12.291%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:18.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:18.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:18.718 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:18.795 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:18.922 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:18.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:18.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:19.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:19.073 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:19.117 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:19.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:19.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:44:19.225 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:19.282 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:19.315 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:44:19.371 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:19.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:19.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:19.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:19.507 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:19 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:19.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:19 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:19.543 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:19 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:19.546 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:19.587 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:19.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:19.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:19.758 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:19.861 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:19.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:19.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:20.034 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:20.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:44:20.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: osdmap e405: 8 total, 8 up, 4 in 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[180297]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 21675/172470 objects degraded (12.567%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: osdmap e405: 8 total, 8 up, 4 in 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:20 smithi064 ceph-mon[186706]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 21675/172470 objects degraded (12.567%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:44:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/876505090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: osdmap e405: 8 total, 8 up, 4 in 2023-06-10T21:44:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: failed to encode map e405 with expected crc 2023-06-10T21:44:20.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:20 smithi152 ceph-mon[156819]: pgmap v1494: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.5 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 21675/172470 objects degraded (12.567%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:44:20.914 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:20.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:21.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:21.061 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:21.178 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:21.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:21.286 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:21.299 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21675/172470 objects degraded (12.567%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:21.300 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21675/172470 objects degraded (12.567%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:21.310 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:21.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21675/172470 objects degraded (12.567%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:21.345 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:21.423 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:21.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:21.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:21.532 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:21.542 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:44:21.542 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:44:21.542 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:44:21.542 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:44:21.542 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:44:21.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:21.681 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:21.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:21.794 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:21.847 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:21.936 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:21.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:21.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:22.045 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:22.097 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:22.130 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:22.140 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:44:22.171 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:22.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:22.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:22.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:22.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:22.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:22 smithi064 ceph-mon[180297]: from='client.47015 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:22 smithi064 ceph-mon[180297]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:44:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:22 smithi064 ceph-mon[186706]: from='client.47015 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:22 smithi064 ceph-mon[186706]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:44:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:22 smithi152 ceph-mon[156819]: from='client.47015 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:22 smithi152 ceph-mon[156819]: pgmap v1495: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:44:23.046 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:44:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:22] "GET /metrics HTTP/1.1" 200 224011 "" "Prometheus/2.33.4" 2023-06-10T21:44:23.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:23 smithi064 ceph-mon[186706]: failed to encode map e406 with expected crc 2023-06-10T21:44:23.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:23 smithi064 ceph-mon[186706]: osdmap e406: 8 total, 8 up, 4 in 2023-06-10T21:44:23.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:23 smithi064 ceph-mon[180297]: failed to encode map e406 with expected crc 2023-06-10T21:44:23.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:23 smithi064 ceph-mon[180297]: osdmap e406: 8 total, 8 up, 4 in 2023-06-10T21:44:23.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:23.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:23 smithi152 ceph-mon[156819]: failed to encode map e406 with expected crc 2023-06-10T21:44:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:23 smithi152 ceph-mon[156819]: osdmap e406: 8 total, 8 up, 4 in 2023-06-10T21:44:23.768 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:23.819 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:24.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:44:24.350 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:24.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:24.463 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:24.514 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:24.594 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:24.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[180297]: failed to encode map e406 with expected crc 2023-06-10T21:44:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[180297]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[180297]: osdmap e407: 8 total, 8 up, 4 in 2023-06-10T21:44:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[180297]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[180297]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 672 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:44:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[186706]: failed to encode map e406 with expected crc 2023-06-10T21:44:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[186706]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[186706]: osdmap e407: 8 total, 8 up, 4 in 2023-06-10T21:44:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[186706]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:24 smithi064 ceph-mon[186706]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 672 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:44:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:24 smithi152 ceph-mon[156819]: failed to encode map e406 with expected crc 2023-06-10T21:44:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:24 smithi152 ceph-mon[156819]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:24 smithi152 ceph-mon[156819]: osdmap e407: 8 total, 8 up, 4 in 2023-06-10T21:44:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:24 smithi152 ceph-mon[156819]: failed to encode map e407 with expected crc 2023-06-10T21:44:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:24 smithi152 ceph-mon[156819]: pgmap v1498: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.3 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 672 op/s; 21134/164253 objects degraded (12.867%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:44:24.703 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:24.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:25.102 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1494,"stamp":"2023-06-10T21:44:20.188910+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3766622118,"num_objects":57490,"num_object_clones":0,"num_object_copies":172470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57490,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":162795,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5229,"num_bytes_recovered":340150286,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75915,"ondisk_log_size":75915,"up":267,"acting":255,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":12436512,"kb_used_data":9637888,"kb_used_omap":27,"kb_used_meta":2798564,"kb_avail":362544096,"statfs":{"total":383980142592,"available":371245154304,"internally_reserved":0,"allocated":9869197312,"data_stored":9865005080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2865730138},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-532152320,"num_objects":-8120,"num_object_clones":0,"num_object_copies":-24360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8120,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":660,"num_bytes_recovered":43253760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003474"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":443,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.318929+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:43:59.318929+0000","last_peered":"2023-06-10T21:43:59.318929+0000","last_clean":"2023-06-10T21:43:59.318929+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:43:59.318929+0000","last_undegraded":"2023-06-10T21:43:59.318929+0000","last_fullsized":"2023-06-10T21:43:59.318929+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324784+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:43:59.324784+0000","last_peered":"2023-06-10T21:43:59.324784+0000","last_clean":"2023-06-10T21:43:59.324784+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:43:59.324784+0000","last_undegraded":"2023-06-10T21:43:59.324784+0000","last_fullsized":"2023-06-10T21:43:59.324784+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":335,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324041+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:43:59.324041+0000","last_peered":"2023-06-10T21:43:59.324041+0000","last_clean":"2023-06-10T21:43:59.324041+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:43:59.324041+0000","last_undegraded":"2023-06-10T21:43:59.324041+0000","last_fullsized":"2023-06-10T21:43:59.324041+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":289,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326338+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:43:59.326338+0000","last_peered":"2023-06-10T21:43:59.326338+0000","last_clean":"2023-06-10T21:43:59.326338+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:43:59.326338+0000","last_undegraded":"2023-06-10T21:43:59.326338+0000","last_fullsized":"2023-06-10T21:43:59.326338+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323466+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:43:59.323466+0000","last_peered":"2023-06-10T21:43:59.323466+0000","last_clean":"2023-06-10T21:43:59.323466+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:43:59.323466+0000","last_undegraded":"2023-06-10T21:43:59.323466+0000","last_fullsized":"2023-06-10T21:43:59.323466+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.215063+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:44:08.215063+0000","last_peered":"2023-06-10T21:44:08.215063+0000","last_clean":"2023-06-10T21:44:08.215063+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:44:08.215063+0000","last_undegraded":"2023-06-10T21:44:08.215063+0000","last_fullsized":"2023-06-10T21:44:08.215063+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":270,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214915+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:44:08.214915+0000","last_peered":"2023-06-10T21:44:08.214915+0000","last_clean":"2023-06-10T21:44:08.214915+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:44:08.214915+0000","last_undegraded":"2023-06-10T21:44:08.214915+0000","last_fullsized":"2023-06-10T21:44:08.214915+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323703+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:43:59.323703+0000","last_peered":"2023-06-10T21:43:59.323703+0000","last_clean":"2023-06-10T21:43:59.323703+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:43:59.323703+0000","last_undegraded":"2023-06-10T21:43:59.323703+0000","last_fullsized":"2023-06-10T21:43:59.323703+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.215009+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:44:08.215009+0000","last_peered":"2023-06-10T21:44:08.215009+0000","last_clean":"2023-06-10T21:44:08.215009+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:44:08.215009+0000","last_undegraded":"2023-06-10T21:44:08.215009+0000","last_fullsized":"2023-06-10T21:44:08.215009+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":154,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323952+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:43:59.323952+0000","last_peered":"2023-06-10T21:43:59.323952+0000","last_clean":"2023-06-10T21:43:59.323952+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:43:59.323952+0000","last_undegraded":"2023-06-10T21:43:59.323952+0000","last_fullsized":"2023-06-10T21:43:59.323952+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":161,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323522+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:43:59.323522+0000","last_peered":"2023-06-10T21:43:59.323522+0000","last_clean":"2023-06-10T21:43:59.323522+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:43:59.323522+0000","last_undegraded":"2023-06-10T21:43:59.323522+0000","last_fullsized":"2023-06-10T21:43:59.323522+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":403,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.318927+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:43:59.318927+0000","last_peered":"2023-06-10T21:43:59.318927+0000","last_clean":"2023-06-10T21:43:59.318927+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:43:59.318927+0000","last_undegraded":"2023-06-10T21:43:59.318927+0000","last_fullsized":"2023-06-10T21:43:59.318927+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":327,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323104+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:43:59.323104+0000","last_peered":"2023-06-10T21:43:59.323104+0000","last_clean":"2023-06-10T21:43:59.323104+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:43:59.323104+0000","last_undegraded":"2023-06-10T21:43:59.323104+0000","last_fullsized":"2023-06-10T21:43:59.323104+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":119,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.215028+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:44:08.215028+0000","last_peered":"2023-06-10T21:44:08.215028+0000","last_clean":"2023-06-10T21:44:08.215028+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:44:08.215028+0000","last_undegraded":"2023-06-10T21:44:08.215028+0000","last_fullsized":"2023-06-10T21:44:08.215028+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":463,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321517+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:43:59.321517+0000","last_peered":"2023-06-10T21:43:59.321517+0000","last_clean":"2023-06-10T21:43:59.321517+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:43:59.321517+0000","last_undegraded":"2023-06-10T21:43:59.321517+0000","last_fullsized":"2023-06-10T21:43:59.321517+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":351,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324526+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:43:59.324526+0000","last_peered":"2023-06-10T21:43:59.324526+0000","last_clean":"2023-06-10T21:43:59.324526+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:43:59.324526+0000","last_undegraded":"2023-06-10T21:43:59.324526+0000","last_fullsized":"2023-06-10T21:43:59.324526+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":274,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326188+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:43:59.326188+0000","last_peered":"2023-06-10T21:43:59.326188+0000","last_clean":"2023-06-10T21:43:59.326188+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:43:59.326188+0000","last_undegraded":"2023-06-10T21:43:59.326188+0000","last_fullsized":"2023-06-10T21:43:59.326188+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214841+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:44:08.214841+0000","last_peered":"2023-06-10T21:44:08.214841+0000","last_clean":"2023-06-10T21:44:08.214841+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:44:08.214841+0000","last_undegraded":"2023-06-10T21:44:08.214841+0000","last_fullsized":"2023-06-10T21:44:08.214841+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":416,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321987+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:43:59.321987+0000","last_peered":"2023-06-10T21:43:59.321987+0000","last_clean":"2023-06-10T21:43:59.321987+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:43:59.321987+0000","last_undegraded":"2023-06-10T21:43:59.321987+0000","last_fullsized":"2023-06-10T21:43:59.321987+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324566+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:43:59.324566+0000","last_peered":"2023-06-10T21:43:59.324566+0000","last_clean":"2023-06-10T21:43:59.324566+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:43:59.324566+0000","last_undegraded":"2023-06-10T21:43:59.324566+0000","last_fullsized":"2023-06-10T21:43:59.324566+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":429,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214632+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:44:08.214632+0000","last_peered":"2023-06-10T21:44:08.214632+0000","last_clean":"2023-06-10T21:44:08.214632+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:44:08.214632+0000","last_undegraded":"2023-06-10T21:44:08.214632+0000","last_fullsized":"2023-06-10T21:44:08.214632+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324783+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:43:59.324783+0000","last_peered":"2023-06-10T21:43:59.324783+0000","last_clean":"2023-06-10T21:43:59.324783+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:43:59.324783+0000","last_undegraded":"2023-06-10T21:43:59.324783+0000","last_fullsized":"2023-06-10T21:43:59.324783+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"404'10106","reported_seq":10147,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:16.003535+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:44:16.003535+0000","last_peered":"2023-06-10T21:44:16.003535+0000","last_clean":"2023-06-10T21:44:16.003535+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:44:16.003535+0000","last_undegraded":"2023-06-10T21:44:16.003535+0000","last_fullsized":"2023-06-10T21:44:16.003535+0000","mapping_epoch":373,"log_start":"386'5100","ondisk_log_start":"386'5100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236322816,"num_objects":3606,"num_object_clones":0,"num_object_copies":10818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10106,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":196,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325171+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:43:59.325171+0000","last_peered":"2023-06-10T21:43:59.325171+0000","last_clean":"2023-06-10T21:43:59.325171+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:43:59.325171+0000","last_undegraded":"2023-06-10T21:43:59.325171+0000","last_fullsized":"2023-06-10T21:43:59.325171+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214390+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:44:08.214390+0000","last_peered":"2023-06-10T21:44:08.214390+0000","last_clean":"2023-06-10T21:44:08.214390+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:44:08.214390+0000","last_undegraded":"2023-06-10T21:44:08.214390+0000","last_fullsized":"2023-06-10T21:44:08.214390+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"404'10368","reported_seq":10425,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:19.061751+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:44:19.061751+0000","last_peered":"2023-06-10T21:44:19.061751+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:44:19.061751+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"387'6000","ondisk_log_start":"387'6000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4368,"log_dups_size":0,"ondisk_log_size":4368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227540992,"num_objects":3472,"num_object_clones":0,"num_object_copies":10416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10368,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1582}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":196,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326496+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:43:59.326496+0000","last_peered":"2023-06-10T21:43:59.326496+0000","last_clean":"2023-06-10T21:43:59.326496+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:43:59.326496+0000","last_undegraded":"2023-06-10T21:43:59.326496+0000","last_fullsized":"2023-06-10T21:43:59.326496+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324118+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:43:59.324118+0000","last_peered":"2023-06-10T21:43:59.324118+0000","last_clean":"2023-06-10T21:43:59.324118+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:43:59.324118+0000","last_undegraded":"2023-06-10T21:43:59.324118+0000","last_fullsized":"2023-06-10T21:43:59.324118+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"404'10161","reported_seq":10232,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:15.912266+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:44:15.912266+0000","last_peered":"2023-06-10T21:44:15.912266+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:44:15.912266+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"387'5800","ondisk_log_start":"387'5800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4361,"log_dups_size":0,"ondisk_log_size":4361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240975872,"num_objects":3677,"num_object_clones":0,"num_object_copies":11031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10161,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1749}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":483,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323602+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:43:59.323602+0000","last_peered":"2023-06-10T21:43:59.323602+0000","last_clean":"2023-06-10T21:43:59.323602+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:43:59.323602+0000","last_undegraded":"2023-06-10T21:43:59.323602+0000","last_fullsized":"2023-06-10T21:43:59.323602+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":167,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322869+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:43:59.322869+0000","last_peered":"2023-06-10T21:43:59.322869+0000","last_clean":"2023-06-10T21:43:59.322869+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:43:59.322869+0000","last_undegraded":"2023-06-10T21:43:59.322869+0000","last_fullsized":"2023-06-10T21:43:59.322869+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"404'10198","reported_seq":10246,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:15.894009+0000","last_change":"2023-06-10T21:43:02.519446+0000","last_active":"2023-06-10T21:44:15.894009+0000","last_peered":"2023-06-10T21:44:15.894009+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:43:02.514777+0000","last_became_peered":"2023-06-10T21:43:02.514777+0000","last_unstale":"2023-06-10T21:44:15.894009+0000","last_undegraded":"2023-06-10T21:43:02.215416+0000","last_fullsized":"2023-06-10T21:43:02.198507+0000","mapping_epoch":391,"log_start":"387'5884","ondisk_log_start":"387'5884","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4314,"log_dups_size":0,"ondisk_log_size":4314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:27:12.740821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236322816,"num_objects":3606,"num_object_clones":0,"num_object_copies":10818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10198,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1702}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":88,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214667+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:44:08.214667+0000","last_peered":"2023-06-10T21:44:08.214667+0000","last_clean":"2023-06-10T21:44:08.214667+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:44:08.214667+0000","last_undegraded":"2023-06-10T21:44:08.214667+0000","last_fullsized":"2023-06-10T21:44:08.214667+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325136+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:43:59.325136+0000","last_peered":"2023-06-10T21:43:59.325136+0000","last_clean":"2023-06-10T21:43:59.325136+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:43:59.325136+0000","last_undegraded":"2023-06-10T21:43:59.325136+0000","last_fullsized":"2023-06-10T21:43:59.325136+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"404'10110","reported_seq":12779,"reported_epoch":404,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:16.099015+0000","last_change":"2023-06-10T21:43:53.697228+0000","last_active":"2023-06-10T21:44:16.099015+0000","last_peered":"2023-06-10T21:44:16.099015+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:43:02.579281+0000","last_became_peered":"2023-06-10T21:43:02.579281+0000","last_unstale":"2023-06-10T21:44:16.099015+0000","last_undegraded":"2023-06-10T21:43:02.241138+0000","last_fullsized":"2023-06-10T21:43:02.196427+0000","mapping_epoch":391,"log_start":"386'5037","ondisk_log_start":"386'5037","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5073,"log_dups_size":0,"ondisk_log_size":5073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:58.809359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":585,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10110,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1309,"num_bytes_recovered":85786624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3033}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":61,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325146+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:43:59.325146+0000","last_peered":"2023-06-10T21:43:59.325146+0000","last_clean":"2023-06-10T21:43:59.325146+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:43:59.325146+0000","last_undegraded":"2023-06-10T21:43:59.325146+0000","last_fullsized":"2023-06-10T21:43:59.325146+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324449+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:43:59.324449+0000","last_peered":"2023-06-10T21:43:59.324449+0000","last_clean":"2023-06-10T21:43:59.324449+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:43:59.324449+0000","last_undegraded":"2023-06-10T21:43:59.324449+0000","last_fullsized":"2023-06-10T21:43:59.324449+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"404'10051","reported_seq":13948,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:16.013066+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:44:16.013066+0000","last_peered":"2023-06-10T21:44:16.013066+0000","last_clean":"2023-06-10T21:44:16.013066+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:44:16.013066+0000","last_undegraded":"2023-06-10T21:44:16.013066+0000","last_fullsized":"2023-06-10T21:44:16.013066+0000","mapping_epoch":395,"log_start":"386'5000","ondisk_log_start":"386'5000","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5051,"log_dups_size":0,"ondisk_log_size":5051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239927296,"num_objects":3661,"num_object_clones":0,"num_object_copies":10983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10051,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":225,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.284528+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:44:08.284528+0000","last_peered":"2023-06-10T21:44:08.284528+0000","last_clean":"2023-06-10T21:44:08.284528+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:44:08.284528+0000","last_undegraded":"2023-06-10T21:44:08.284528+0000","last_fullsized":"2023-06-10T21:44:08.284528+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":442,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:06.082084+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:44:06.082084+0000","last_peered":"2023-06-10T21:44:06.082084+0000","last_clean":"2023-06-10T21:44:06.082084+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:44:06.082084+0000","last_undegraded":"2023-06-10T21:44:06.082084+0000","last_fullsized":"2023-06-10T21:44:06.082084+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"404'10144","reported_seq":10198,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:19.045650+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:44:19.045650+0000","last_peered":"2023-06-10T21:44:19.045650+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:44:19.045650+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"387'5800","ondisk_log_start":"387'5800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4344,"log_dups_size":0,"ondisk_log_size":4344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224657408,"num_objects":3428,"num_object_clones":0,"num_object_copies":10284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10144,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1575}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":419,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.322077+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:43:59.322077+0000","last_peered":"2023-06-10T21:43:59.322077+0000","last_clean":"2023-06-10T21:43:59.322077+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:43:59.322077+0000","last_undegraded":"2023-06-10T21:43:59.322077+0000","last_fullsized":"2023-06-10T21:43:59.322077+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324489+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:43:59.324489+0000","last_peered":"2023-06-10T21:43:59.324489+0000","last_clean":"2023-06-10T21:43:59.324489+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:43:59.324489+0000","last_undegraded":"2023-06-10T21:43:59.324489+0000","last_fullsized":"2023-06-10T21:43:59.324489+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"404'10023","reported_seq":10072,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:15.989289+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:44:15.989289+0000","last_peered":"2023-06-10T21:44:15.989289+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:44:15.989289+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"386'4974","ondisk_log_start":"386'4974","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239271936,"num_objects":3651,"num_object_clones":0,"num_object_copies":10953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1716}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324056+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:43:59.324056+0000","last_peered":"2023-06-10T21:43:59.324056+0000","last_clean":"2023-06-10T21:43:59.324056+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:43:59.324056+0000","last_undegraded":"2023-06-10T21:43:59.324056+0000","last_fullsized":"2023-06-10T21:43:59.324056+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":351,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325032+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:43:59.325032+0000","last_peered":"2023-06-10T21:43:59.325032+0000","last_clean":"2023-06-10T21:43:59.325032+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:43:59.325032+0000","last_undegraded":"2023-06-10T21:43:59.325032+0000","last_fullsized":"2023-06-10T21:43:59.325032+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"404'9964","reported_seq":10011,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:16.092230+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:44:16.092230+0000","last_peered":"2023-06-10T21:44:16.092230+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:44:16.092230+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"386'4919","ondisk_log_start":"386'4919","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5045,"log_dups_size":0,"ondisk_log_size":5045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238026752,"num_objects":3632,"num_object_clones":0,"num_object_copies":10896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9964,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1703}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":187,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323183+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:43:59.323183+0000","last_peered":"2023-06-10T21:43:59.323183+0000","last_clean":"2023-06-10T21:43:59.323183+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:43:59.323183+0000","last_undegraded":"2023-06-10T21:43:59.323183+0000","last_fullsized":"2023-06-10T21:43:59.323183+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":69,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214683+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:44:08.214683+0000","last_peered":"2023-06-10T21:44:08.214683+0000","last_clean":"2023-06-10T21:44:08.214683+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:44:08.214683+0000","last_undegraded":"2023-06-10T21:44:08.214683+0000","last_fullsized":"2023-06-10T21:44:08.214683+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":72,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214420+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:44:08.214420+0000","last_peered":"2023-06-10T21:44:08.214420+0000","last_clean":"2023-06-10T21:44:08.214420+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:44:08.214420+0000","last_undegraded":"2023-06-10T21:44:08.214420+0000","last_fullsized":"2023-06-10T21:44:08.214420+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"404'10208","reported_seq":10267,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:19.008655+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:44:19.008655+0000","last_peered":"2023-06-10T21:44:19.008655+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:44:19.008655+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"387'5900","ondisk_log_start":"387'5900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4308,"log_dups_size":0,"ondisk_log_size":4308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048344,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10208,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1579}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":88,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214396+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:44:08.214396+0000","last_peered":"2023-06-10T21:44:08.214396+0000","last_clean":"2023-06-10T21:44:08.214396+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:44:08.214396+0000","last_undegraded":"2023-06-10T21:44:08.214396+0000","last_fullsized":"2023-06-10T21:44:08.214396+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":366,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214408+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:44:08.214408+0000","last_peered":"2023-06-10T21:44:08.214408+0000","last_clean":"2023-06-10T21:44:08.214408+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:44:08.214408+0000","last_undegraded":"2023-06-10T21:44:08.214408+0000","last_fullsized":"2023-06-10T21:44:08.214408+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":268,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.319596+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:43:59.319596+0000","last_peered":"2023-06-10T21:43:59.319596+0000","last_clean":"2023-06-10T21:43:59.319596+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:43:59.319596+0000","last_undegraded":"2023-06-10T21:43:59.319596+0000","last_fullsized":"2023-06-10T21:43:59.319596+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"404'10218","reported_seq":10277,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:19.032084+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:44:19.032084+0000","last_peered":"2023-06-10T21:44:19.032084+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:44:19.032084+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"387'5900","ondisk_log_start":"387'5900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4318,"log_dups_size":0,"ondisk_log_size":4318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10218,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1587}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":288,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321996+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:43:59.321996+0000","last_peered":"2023-06-10T21:43:59.321996+0000","last_clean":"2023-06-10T21:43:59.321996+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:43:59.321996+0000","last_undegraded":"2023-06-10T21:43:59.321996+0000","last_fullsized":"2023-06-10T21:43:59.321996+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":293,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321553+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:43:59.321553+0000","last_peered":"2023-06-10T21:43:59.321553+0000","last_clean":"2023-06-10T21:43:59.321553+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:43:59.321553+0000","last_undegraded":"2023-06-10T21:43:59.321553+0000","last_fullsized":"2023-06-10T21:43:59.321553+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":87,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.215015+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:44:08.215015+0000","last_peered":"2023-06-10T21:44:08.215015+0000","last_clean":"2023-06-10T21:44:08.215015+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:44:08.215015+0000","last_undegraded":"2023-06-10T21:44:08.215015+0000","last_fullsized":"2023-06-10T21:44:08.215015+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"404'10146","reported_seq":10194,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:16.069089+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:44:16.069089+0000","last_peered":"2023-06-10T21:44:16.069089+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:44:16.069089+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"386'5100","ondisk_log_start":"386'5100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5046,"log_dups_size":0,"ondisk_log_size":5046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244842496,"num_objects":3736,"num_object_clones":0,"num_object_copies":11208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10146,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1772}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":79,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214960+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:44:08.214960+0000","last_peered":"2023-06-10T21:44:08.214960+0000","last_clean":"2023-06-10T21:44:08.214960+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:44:08.214960+0000","last_undegraded":"2023-06-10T21:44:08.214960+0000","last_fullsized":"2023-06-10T21:44:08.214960+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":604,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324070+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:43:59.324070+0000","last_peered":"2023-06-10T21:43:59.324070+0000","last_clean":"2023-06-10T21:43:59.324070+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:43:59.324070+0000","last_undegraded":"2023-06-10T21:43:59.324070+0000","last_fullsized":"2023-06-10T21:43:59.324070+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325101+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:43:59.325101+0000","last_peered":"2023-06-10T21:43:59.325101+0000","last_clean":"2023-06-10T21:43:59.325101+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:43:59.325101+0000","last_undegraded":"2023-06-10T21:43:59.325101+0000","last_fullsized":"2023-06-10T21:43:59.325101+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":287,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323749+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:43:59.323749+0000","last_peered":"2023-06-10T21:43:59.323749+0000","last_clean":"2023-06-10T21:43:59.323749+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:43:59.323749+0000","last_undegraded":"2023-06-10T21:43:59.323749+0000","last_fullsized":"2023-06-10T21:43:59.323749+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"404'10153","reported_seq":14055,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:19.058922+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:44:19.058922+0000","last_peered":"2023-06-10T21:44:19.058922+0000","last_clean":"2023-06-10T21:44:19.058922+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:44:19.058922+0000","last_undegraded":"2023-06-10T21:44:19.058922+0000","last_fullsized":"2023-06-10T21:44:19.058922+0000","mapping_epoch":401,"log_start":"387'5800","ondisk_log_start":"387'5800","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226295808,"num_objects":3453,"num_object_clones":0,"num_object_copies":10359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10153,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":456,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324143+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:43:59.324143+0000","last_peered":"2023-06-10T21:43:59.324143+0000","last_clean":"2023-06-10T21:43:59.324143+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:43:59.324143+0000","last_undegraded":"2023-06-10T21:43:59.324143+0000","last_fullsized":"2023-06-10T21:43:59.324143+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323664+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:43:59.323664+0000","last_peered":"2023-06-10T21:43:59.323664+0000","last_clean":"2023-06-10T21:43:59.323664+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:43:59.323664+0000","last_undegraded":"2023-06-10T21:43:59.323664+0000","last_fullsized":"2023-06-10T21:43:59.323664+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":155,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.320785+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:43:59.320785+0000","last_peered":"2023-06-10T21:43:59.320785+0000","last_clean":"2023-06-10T21:43:59.320785+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:43:59.320785+0000","last_undegraded":"2023-06-10T21:43:59.320785+0000","last_fullsized":"2023-06-10T21:43:59.320785+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"404'10113","reported_seq":10154,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:16.014728+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:44:16.014728+0000","last_peered":"2023-06-10T21:44:16.014728+0000","last_clean":"2023-06-10T21:44:16.014728+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:44:16.014728+0000","last_undegraded":"2023-06-10T21:44:16.014728+0000","last_fullsized":"2023-06-10T21:44:16.014728+0000","mapping_epoch":373,"log_start":"386'5100","ondisk_log_start":"386'5100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5013,"log_dups_size":0,"ondisk_log_size":5013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10113,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":435,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323680+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:43:59.323680+0000","last_peered":"2023-06-10T21:43:59.323680+0000","last_clean":"2023-06-10T21:43:59.323680+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:43:59.323680+0000","last_undegraded":"2023-06-10T21:43:59.323680+0000","last_fullsized":"2023-06-10T21:43:59.323680+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":283,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.319669+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:43:59.319669+0000","last_peered":"2023-06-10T21:43:59.319669+0000","last_clean":"2023-06-10T21:43:59.319669+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:43:59.319669+0000","last_undegraded":"2023-06-10T21:43:59.319669+0000","last_fullsized":"2023-06-10T21:43:59.319669+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":209,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214412+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:44:08.214412+0000","last_peered":"2023-06-10T21:44:08.214412+0000","last_clean":"2023-06-10T21:44:08.214412+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:44:08.214412+0000","last_undegraded":"2023-06-10T21:44:08.214412+0000","last_fullsized":"2023-06-10T21:44:08.214412+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"404'10363","reported_seq":10412,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:16.070231+0000","last_change":"2023-06-10T21:43:02.588633+0000","last_active":"2023-06-10T21:44:16.070231+0000","last_peered":"2023-06-10T21:44:16.070231+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:44:16.070231+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"386'5300","ondisk_log_start":"386'5300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5063,"log_dups_size":0,"ondisk_log_size":5063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244514816,"num_objects":3731,"num_object_clones":0,"num_object_copies":11193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1976,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10363,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1755}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":414,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:05.906823+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:44:05.906823+0000","last_peered":"2023-06-10T21:44:05.906823+0000","last_clean":"2023-06-10T21:44:05.906823+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:44:05.906823+0000","last_undegraded":"2023-06-10T21:44:05.906823+0000","last_fullsized":"2023-06-10T21:44:05.906823+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":225,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:44:05.906805+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:44:05.906805+0000","last_peered":"2023-06-10T21:44:05.906805+0000","last_clean":"2023-06-10T21:44:05.906805+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:44:05.906805+0000","last_undegraded":"2023-06-10T21:44:05.906805+0000","last_fullsized":"2023-06-10T21:44:05.906805+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323971+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:43:59.323971+0000","last_peered":"2023-06-10T21:43:59.323971+0000","last_clean":"2023-06-10T21:43:59.323971+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:43:59.323971+0000","last_undegraded":"2023-06-10T21:43:59.323971+0000","last_fullsized":"2023-06-10T21:43:59.323971+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"404'10208","reported_seq":10263,"reported_epoch":404,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:16.090944+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:44:16.090944+0000","last_peered":"2023-06-10T21:44:16.090944+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:44:16.090944+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"386'5200","ondisk_log_start":"386'5200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5008,"log_dups_size":0,"ondisk_log_size":5008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10208,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1697}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":152,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325182+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:43:59.325182+0000","last_peered":"2023-06-10T21:43:59.325182+0000","last_clean":"2023-06-10T21:43:59.325182+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:43:59.325182+0000","last_undegraded":"2023-06-10T21:43:59.325182+0000","last_fullsized":"2023-06-10T21:43:59.325182+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":266,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.326165+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:43:59.326165+0000","last_peered":"2023-06-10T21:43:59.326165+0000","last_clean":"2023-06-10T21:43:59.326165+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:43:59.326165+0000","last_undegraded":"2023-06-10T21:43:59.326165+0000","last_fullsized":"2023-06-10T21:43:59.326165+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":59,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214574+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:44:08.214574+0000","last_peered":"2023-06-10T21:44:08.214574+0000","last_clean":"2023-06-10T21:44:08.214574+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:44:08.214574+0000","last_undegraded":"2023-06-10T21:44:08.214574+0000","last_fullsized":"2023-06-10T21:44:08.214574+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.321672+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:43:59.321672+0000","last_peered":"2023-06-10T21:43:59.321672+0000","last_clean":"2023-06-10T21:43:59.321672+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:43:59.321672+0000","last_undegraded":"2023-06-10T21:43:59.321672+0000","last_fullsized":"2023-06-10T21:43:59.321672+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":300,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.319735+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:43:59.319735+0000","last_peered":"2023-06-10T21:43:59.319735+0000","last_clean":"2023-06-10T21:43:59.319735+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:43:59.319735+0000","last_undegraded":"2023-06-10T21:43:59.319735+0000","last_fullsized":"2023-06-10T21:43:59.319735+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":67,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214516+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:44:08.214516+0000","last_peered":"2023-06-10T21:44:08.214516+0000","last_clean":"2023-06-10T21:44:08.214516+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:44:08.214516+0000","last_undegraded":"2023-06-10T21:44:08.214516+0000","last_fullsized":"2023-06-10T21:44:08.214516+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":308,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325229+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:43:59.325229+0000","last_peered":"2023-06-10T21:43:59.325229+0000","last_clean":"2023-06-10T21:43:59.325229+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:43:59.325229+0000","last_undegraded":"2023-06-10T21:43:59.325229+0000","last_fullsized":"2023-06-10T21:43:59.325229+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":90,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.214527+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:44:08.214527+0000","last_peered":"2023-06-10T21:44:08.214527+0000","last_clean":"2023-06-10T21:44:08.214527+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:44:08.214527+0000","last_undegraded":"2023-06-10T21:44:08.214527+0000","last_fullsized":"2023-06-10T21:44:08.214527+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":379,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.325334+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:43:59.325334+0000","last_peered":"2023-06-10T21:43:59.325334+0000","last_clean":"2023-06-10T21:43:59.325334+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:43:59.325334+0000","last_undegraded":"2023-06-10T21:43:59.325334+0000","last_fullsized":"2023-06-10T21:43:59.325334+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":57,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324653+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:43:59.324653+0000","last_peered":"2023-06-10T21:43:59.324653+0000","last_clean":"2023-06-10T21:43:59.324653+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:43:59.324653+0000","last_undegraded":"2023-06-10T21:43:59.324653+0000","last_fullsized":"2023-06-10T21:43:59.324653+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.323802+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:43:59.323802+0000","last_peered":"2023-06-10T21:43:59.323802+0000","last_clean":"2023-06-10T21:43:59.323802+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:43:59.323802+0000","last_undegraded":"2023-06-10T21:43:59.323802+0000","last_fullsized":"2023-06-10T21:43:59.323802+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":326,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-06-10T21:44:08.215176+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:44:08.215176+0000","last_peered":"2023-06-10T21:44:08.215176+0000","last_clean":"2023-06-10T21:44:08.215176+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:44:08.215176+0000","last_undegraded":"2023-06-10T21:44:08.215176+0000","last_fullsized":"2023-06-10T21:44:08.215176+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":403,"state":"active+clean","last_fresh":"2023-06-10T21:43:59.324147+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:43:59.324147+0000","last_peered":"2023-06-10T21:43:59.324147+0000","last_clean":"2023-06-10T21:43:59.324147+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:43:59.324147+0000","last_undegraded":"2023-06-10T21:43:59.324147+0000","last_fullsized":"2023-06-10T21:43:59.324147+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3766026264,"num_objects":57466,"num_object_clones":0,"num_object_copies":172398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57466,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162534,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5154,"num_bytes_recovered":337772544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9783226368,"data_stored":9783214152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75720,"ondisk_log_size":75720,"up":48,"acting":36,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430767,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3407440,"kb_used_data":2475464,"kb_used_omap":0,"kb_used_meta":931968,"kb_avail":90337712,"statfs":{"total":95995035648,"available":92505817088,"internally_reserved":0,"allocated":2534875136,"data_stored":2534167781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":954335232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.3089999999999999,"15min":3.1179999999999999},"min":{"1min":0.28599999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.2699999999999996,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.244,"5min":1.552,"15min":3.371},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":3.0960000000000001,"5min":22.462,"15min":185.58000000000001},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.1309999999999998,"15min":2.6059999999999999},"min":{"1min":0.33400000000000002,"5min":0.25600000000000001,"15min":0.24199999999999999},"max":{"1min":4.3540000000000001,"5min":38.078000000000003,"15min":87.417000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.7210000000000001,"5min":2.6469999999999998,"15min":2.851},"min":{"1min":0.33200000000000002,"5min":0.28599999999999998,"15min":0.27100000000000002},"max":{"1min":7.3710000000000004,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:43:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":2.9649999999999999,"15min":4.8200000000000003},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.7959999999999998,"5min":75.219999999999999,"15min":172.33600000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":2,"5min":2.6970000000000001,"15min":4.7949999999999999},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":4.8799999999999999,"5min":75.108999999999995,"15min":172.185},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:43:30 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":3.1800000000000002,"15min":4.4429999999999996},"min":{"1min":0.443,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":7.5019999999999998,"5min":89.504999999999995,"15min":147.00700000000001},"last":8.0449999999999999},{"interface":"front","average":{"1min":2.2480000000000002,"5min":2.786,"15min":4.1669999999999998},"min":{"1min":0.47399999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":8.2889999999999997,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.439}]},{"osd":4,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":0.88900000000000001,"15min":0.59599999999999997},"min":{"1min":0.35499999999999998,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":13.429,"5min":13.429,"15min":13.429},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1120000000000001,"15min":0.71699999999999997},"min":{"1min":0.34200000000000003,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":4.6760000000000002,"5min":22.77,"15min":22.77},"last":0.498}]},{"osd":5,"last update":"Sat Jun 10 21:44:04 2023","interfaces":[{"interface":"back","average":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":131.619,"5min":131.619,"15min":131.619},"last":0.64100000000000001},{"interface":"front","average":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":134.928,"5min":134.928,"15min":134.928},"last":0.61799999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":0.89700000000000002,"15min":0.67000000000000004},"min":{"1min":0.41199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8280000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.911,"5min":0.99099999999999999,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":4.3739999999999997,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.58699999999999997}]}]},{"osd":6,"up_from":0,"seq":180388626995,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566437,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020414,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075825,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3782020,"kb_used_data":2806908,"kb_used_omap":7,"kb_used_meta":975096,"kb_avail":89963132,"statfs":{"total":95995035648,"available":92122247168,"internally_reserved":0,"allocated":2874273792,"data_stored":2872941468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":998498385},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.903,"15min":4.4560000000000004},"min":{"1min":0.27500000000000002,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":8.5899999999999999,"5min":83.733000000000004,"15min":171.33699999999999},"last":6.1130000000000004},{"interface":"front","average":{"1min":1.673,"5min":2.4100000000000001,"15min":4.3440000000000003},"min":{"1min":0.255,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.9400000000000004,"5min":82.322999999999993,"15min":171.56200000000001},"last":1.1479999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":4.4500000000000002,"15min":5.3940000000000001},"min":{"1min":0.23499999999999999,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":4.1449999999999996,"5min":85.709999999999994,"15min":171.286},"last":0.375},{"interface":"front","average":{"1min":2.4020000000000001,"5min":4.109,"15min":3.9550000000000001},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.0500000000000007,"5min":85.501999999999995,"15min":147.81299999999999},"last":6.1379999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:43:24 2023","interfaces":[{"interface":"back","average":{"1min":11.114000000000001,"5min":9.2509999999999994,"15min":6.2389999999999999},"min":{"1min":0.41599999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":145.07599999999999,"5min":171.78299999999999,"15min":171.78299999999999},"last":6.6319999999999997},{"interface":"front","average":{"1min":11.140000000000001,"5min":8.5540000000000003,"15min":6.2969999999999997},"min":{"1min":0.39300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":144.99000000000001,"5min":171.65100000000001,"15min":171.65100000000001},"last":3.0710000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.343,"15min":2.7839999999999998},"min":{"1min":0.46700000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.681,"5min":9.9390000000000001,"15min":171.48699999999999},"last":2.9510000000000001},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.1859999999999999,"15min":3.6680000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":4.673,"5min":15.544,"15min":171.18100000000001},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:44:02 2023","interfaces":[{"interface":"back","average":{"1min":6.9820000000000002,"5min":6.9820000000000002,"15min":6.9820000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":75.896000000000001,"5min":75.896000000000001,"15min":75.896000000000001},"last":6.0389999999999997},{"interface":"front","average":{"1min":6.3419999999999996,"5min":6.3419999999999996,"15min":6.3419999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":76.337000000000003,"5min":76.337000000000003,"15min":76.337000000000003},"last":2.996}]},{"osd":6,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.5449999999999999,"15min":2.472},"min":{"1min":0.496,"5min":0.307,"15min":0.307},"max":{"1min":5.7220000000000004,"5min":9.9049999999999994,"15min":171.441},"last":3.024},{"interface":"front","average":{"1min":1.8200000000000001,"5min":2.343,"15min":3.3370000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.1600000000000001,"5min":15.590999999999999,"15min":171.24600000000001},"last":0.63}]},{"osd":7,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.9370000000000001,"15min":4.4699999999999998},"min":{"1min":0.46500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.6560000000000006,"5min":18.16,"15min":219.131},"last":6.0819999999999999},{"interface":"front","average":{"1min":1.835,"5min":1.4419999999999999,"15min":4.4370000000000003},"min":{"1min":0.503,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5789999999999997,"5min":9.8719999999999999,"15min":219.19},"last":3.0489999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824185,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3285188,"kb_used_data":2579324,"kb_used_omap":7,"kb_used_meta":705848,"kb_avail":90459964,"statfs":{"total":95995035648,"available":92631003136,"internally_reserved":0,"allocated":2641227776,"data_stored":2639774781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":722788371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":2.1589999999999998,"15min":3.5760000000000001},"min":{"1min":0.23599999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.1949999999999998,"5min":30.744,"15min":105.09},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.8460000000000001,"5min":2.4580000000000002,"15min":3.6379999999999999},"min":{"1min":0.26000000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.6970000000000001,"5min":30.805,"15min":99.852999999999994},"last":1.1519999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5509999999999999,"5min":2.5310000000000001,"15min":3.5099999999999998},"min":{"1min":0.309,"5min":0.23200000000000001,"15min":0.20499999999999999},"max":{"1min":6.6459999999999999,"5min":21.704999999999998,"15min":94.766000000000005},"last":1.2549999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.484,"15min":3.4199999999999999},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.23000000000000001},"max":{"1min":3.9399999999999999,"5min":28.695,"15min":94.691000000000003},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":4.6109999999999998,"15min":5.0359999999999996},"min":{"1min":0.28699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.6609999999999996,"5min":231.07900000000001,"15min":231.07900000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.7050000000000001,"5min":5.3769999999999998,"15min":6.3760000000000003},"min":{"1min":0.35899999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":8.6489999999999991,"5min":231.58600000000001,"15min":231.58600000000001},"last":1.446}]},{"osd":4,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":1.5249999999999999,"15min":1.4410000000000001},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.7300000000000004,"5min":10.368,"15min":13.417},"last":3.0209999999999999},{"interface":"front","average":{"1min":1.514,"5min":2.085,"15min":2.3199999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":6.5359999999999996,"5min":19.244,"15min":19.728000000000002},"last":0.61199999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:44:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.532,"5min":6.532,"15min":6.532},"last":0.76000000000000001},{"interface":"front","average":{"1min":2.7040000000000002,"5min":2.7040000000000002,"15min":2.7040000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":1.627}]},{"osd":6,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.1240000000000001,"15min":2.3679999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":6.6150000000000002,"5min":19.164999999999999,"15min":19.805},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.508,"15min":1.514},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7610000000000001,"5min":9.9719999999999995,"15min":17.327000000000002},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:43:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.2839999999999998,"15min":2.6339999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33900000000000002},"max":{"1min":6.5830000000000002,"5min":19.193000000000001,"15min":79.984999999999999},"last":2.8260000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.603,"15min":1.802},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.1749999999999998,"5min":9.9380000000000006,"15min":80.272999999999996},"last":0.72699999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019451,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245457,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1961864,"kb_used_data":1776192,"kb_used_omap":11,"kb_used_meta":185652,"kb_avail":91783288,"statfs":{"total":95995035648,"available":93986086912,"internally_reserved":0,"allocated":1818820608,"data_stored":1818121050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":190108150},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":2.0800000000000001,"15min":2.0800000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.266,"5min":4.266,"15min":4.266},"last":0.995}]},{"osd":1,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.54,"5min":7.54,"15min":7.54},"last":2.4900000000000002},{"interface":"front","average":{"1min":2.282,"5min":2.282,"15min":2.282},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.7989999999999995,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":2.4980000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":1.8620000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":1.026},{"interface":"front","average":{"1min":2.306,"5min":2.306,"15min":2.306},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.641,"5min":12.641,"15min":12.641},"last":1.1359999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":8.2330000000000005,"5min":8.2330000000000005,"15min":8.2330000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":133.07499999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":1.8149999999999999},{"interface":"front","average":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":7.8949999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":133.03800000000001,"5min":133.03800000000001,"15min":133.03800000000001},"last":2.4500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.173,"5min":5.173,"15min":5.173},"last":0.438},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.3079999999999998,"15min":2.3079999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.476,"5min":7.476,"15min":7.476},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.8879999999999999,"15min":1.8879999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.345,"5min":1.345,"15min":1.345},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.891,"5min":2.891,"15min":2.891},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.4160000000000004,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":4.8979999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2863333376,"data_stored":2863333376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2628653056,"data_stored":2628648984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1783500800,"data_stored":1783496728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2507739136,"data_stored":2507735064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:44:25.108 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:44:25.129 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:25.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:25.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:25.253 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:25.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:25.364 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:25.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:25.472 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:25.510 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:25.575 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:25.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:25.692 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:25.756 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:25.819 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:25.873 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:25.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:25.955 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:44:25.985 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:25.997 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:44:26.029 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:26.042 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:44:26.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:44:26.130 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:26.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:26.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:26.141 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:44:26.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:44:26.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2855315213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:26.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:44:26.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2855315213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:26.192 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:44:26.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:44:26.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2855315213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:26.239 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:44:26.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:26.290 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:44:26.291 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:26.298 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.299 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.300 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:44:26.388 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:26.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:26.503 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:26.580 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:26.740 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:26.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:26.847 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:26.901 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:27.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:26 smithi152 ceph-mon[156819]: pgmap v1499: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 901 op/s; 20730/153804 objects degraded (13.478%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:44:27.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 21134/164253 objects degraded (12.867%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:27.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:26 smithi064 ceph-mon[186706]: pgmap v1499: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 901 op/s; 20730/153804 objects degraded (13.478%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:44:27.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 21134/164253 objects degraded (12.867%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:26 smithi064 ceph-mon[180297]: pgmap v1499: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 901 op/s; 20730/153804 objects degraded (13.478%); 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:44:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 21134/164253 objects degraded (12.867%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:27.729 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:27.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:27.839 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:27.917 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:27.921 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:28.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:28 smithi064 ceph-mon[180297]: pgmap v1500: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 777 op/s; 20730/153804 objects degraded (13.478%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:44:28.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:28 smithi064 ceph-mon[186706]: pgmap v1500: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 777 op/s; 20730/153804 objects degraded (13.478%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:44:28.520 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:28.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:28.630 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:28.690 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:28 smithi152 ceph-mon[156819]: pgmap v1500: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.1 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 777 op/s; 20730/153804 objects degraded (13.478%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:44:28.698 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:28.809 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:28.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:28.918 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:28.945 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:28.990 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:28.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:29.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:29.160 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:29.286 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:29.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:29.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:29.348 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:44:29.401 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:29.479 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:29.631 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:29.634 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:44:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:29.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:29.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:44:29.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:44:29.736 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:29.786 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:30.011 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:44:30.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:30.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:44:30.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: osdmap e408: 8 total, 8 up, 4 in 2023-06-10T21:44:30.314 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[180297]: pgmap v1502: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 20730/149379 objects degraded (13.877%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:44:30.462 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:30.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:30.574 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:30.638 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:30.638 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:44:30.638 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: osdmap e408: 8 total, 8 up, 4 in 2023-06-10T21:44:30.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:30 smithi152 ceph-mon[156819]: pgmap v1502: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 20730/149379 objects degraded (13.877%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:44:30.644 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:30.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:30.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4200132704' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:44:30.655 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: osdmap e408: 8 total, 8 up, 4 in 2023-06-10T21:44:30.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: failed to encode map e408 with expected crc 2023-06-10T21:44:30.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:30 smithi064 ceph-mon[186706]: pgmap v1502: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 3.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 629 op/s; 20730/149379 objects degraded (13.877%); 3.2 MiB/s, 51 objects/s recovering 2023-06-10T21:44:30.912 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:30.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:31.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:31.103 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:31.225 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:31.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:31.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:31.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:31 smithi064 ceph-mon[186706]: failed to encode map e408 with expected crc 2023-06-10T21:44:31.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20730/149379 objects degraded (13.877%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:31.341 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:31.403 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:31.456 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:31.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:31.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:31.585 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:44:31.586 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:44:31.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:31 smithi064 ceph-mon[180297]: failed to encode map e408 with expected crc 2023-06-10T21:44:31.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20730/149379 objects degraded (13.877%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:31.604 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:31.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:31 smithi152 ceph-mon[156819]: failed to encode map e408 with expected crc 2023-06-10T21:44:31.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20730/149379 objects degraded (13.877%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:32.018 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:32.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:32.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:32.124 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:32.179 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:32.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:44:32.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:32.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:32.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[180297]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 844 op/s; 20415/141759 objects degraded (14.401%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:32.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:32.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:32.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:32 smithi064 ceph-mon[186706]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 844 op/s; 20415/141759 objects degraded (14.401%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:32.964 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:44:32.990 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:33.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:33.110 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:33.163 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:32] "GET /metrics HTTP/1.1" 200 224011 "" "Prometheus/2.33.4" 2023-06-10T21:44:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:32 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:33.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:32 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:33.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:32 smithi152 ceph-mon[156819]: pgmap v1503: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 844 op/s; 20415/141759 objects degraded (14.401%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:33.276 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:33.277 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:33.534 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:33.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:33.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:33.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:33.701 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:33.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:34.124 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:34.132 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:34.132 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:34.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:34.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:34.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:34.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:34.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:34.507 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:34.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:34.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:34.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:34.651 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:34.757 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:34.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:34.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:34.890 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:34 smithi064 ceph-mon[180297]: pgmap v1504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 749 op/s; 20415/141759 objects degraded (14.401%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:44:34.890 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:34 smithi064 ceph-mon[186706]: pgmap v1504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 749 op/s; 20415/141759 objects degraded (14.401%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:44:34.932 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:34 smithi152 ceph-mon[156819]: pgmap v1504: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.9 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 749 op/s; 20415/141759 objects degraded (14.401%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:44:34.980 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:34.980 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 20415/141759 objects degraded (14.401%), 11 pgs degraded, 11 pgs undersized","count":22},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2148,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":408,"num_osds":8,"num_up_osds":8,"osd_up_since":1686433380,"num_in_osds":4,"osd_in_since":1686433189,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":77},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":10},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":47253,"data_bytes":3095730086,"bytes_used":10899779584,"bytes_avail":373080363008,"bytes_total":383980142592,"degraded_objects":20415,"degraded_total":141759,"degraded_ratio":0.14401202040082112,"recovering_objects_per_sec":71,"recovering_bytes_per_sec":4710375,"recovering_keys_per_sec":0,"num_objects_recovered":720,"num_bytes_recovered":47120384,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":749},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":29,"modified":"2023-06-10T21:44:00.179441+0000","services":{"osd":{"daemons":{"summary":"","3":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true},"9ab7ee0e-4ecd-48a4-8ef6-12d592a043b5":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 18s)","progress":0.86516851186752319,"add_to_ceph_s":true}}} 2023-06-10T21:44:35.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:35.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:35.846 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:35.910 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:36.158 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/307151954' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:36.159 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/307151954' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:36.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/307151954' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:44:37.067 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:37.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:36 smithi152 ceph-mon[156819]: pgmap v1505: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:37.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20415/141759 objects degraded (14.401%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:37.339 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:37.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:36 smithi064 ceph-mon[180297]: pgmap v1505: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:37.343 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20415/141759 objects degraded (14.401%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:37.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:36 smithi064 ceph-mon[186706]: pgmap v1505: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:37.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20415/141759 objects degraded (14.401%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:37.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:37.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:37.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:37.470 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:44:37.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:44:37.550 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:37.931 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:37.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:38.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:38.092 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:38.093 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:44:38.132 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:44:38.173 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:38.183 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:44:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:38.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:38.212 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3664825845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:38.212 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3664825845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:38.231 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:44:38.285 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:38.301 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:44:38.337 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:38.349 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:44:38.386 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:44:38.403 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:38.410 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3664825845' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:38.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:38.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:38.413 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:44:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.420 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.421 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:44:38.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:38.574 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:39.152 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:44:39.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[180297]: pgmap v1506: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:39.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/135803406' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[186706]: pgmap v1506: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/135803406' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:39 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:39 smithi152 ceph-mon[156819]: pgmap v1506: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.6 GiB data, 9.5 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 20036/129696 objects degraded (15.448%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:39.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/135803406' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:39 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:44:39.590 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:39.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:39.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:39.766 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:40.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: osdmap e409: 8 total, 8 up, 4 in 2023-06-10T21:44:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[186706]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:40.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: osdmap e409: 8 total, 8 up, 4 in 2023-06-10T21:44:40.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:40 smithi064 ceph-mon[180297]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.447 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:40.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:44:40.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: osdmap e409: 8 total, 8 up, 4 in 2023-06-10T21:44:40.448 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:40 smithi152 ceph-mon[156819]: failed to encode map e409 with expected crc 2023-06-10T21:44:40.749 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:40.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:40.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:40.897 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:40.939 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:40.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:41.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:41.121 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:41.398 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:41 smithi064 ceph-mon[186706]: pgmap v1508: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 9.1 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 20036/125097 objects degraded (16.016%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:41.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:41 smithi064 ceph-mon[180297]: pgmap v1508: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 9.1 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 20036/125097 objects degraded (16.016%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:41 smithi152 ceph-mon[156819]: pgmap v1508: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.5 GiB data, 9.1 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 20036/125097 objects degraded (16.016%); 4.3 MiB/s, 69 objects/s recovering 2023-06-10T21:44:41.474 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:41.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:41.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:41.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:41.737 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:41.844 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:41.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:181822 13.3 13:cd7a612b:::benchmark_data_smithi064_501823_object71820:head [delete] snapc 0=[] ondisk+write+known_if_redirected e409)", 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:44:41.871047+0000", 2023-06-10T21:44:41.878 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.002277425, 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023053399999999999, 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:44:41.879 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 181822 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871047+0000", 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:44:41.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871047+0000", 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:44:41.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871045+0000", 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871052+0000", 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6479999999999997e-06 2023-06-10T21:44:41.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871068+0000", 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5739999999999998e-05 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:44:41.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871076+0000", 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.439e-06 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871132+0000", 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6147000000000003e-05 2023-06-10T21:44:41.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871298+0000", 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016624200000000001 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:44:41.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:44:41.871478+0000", 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000179345 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 1 2023-06-10T21:44:41.886 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:44:41.956 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:42.025 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:42.120 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:42.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:42.234 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:42.285 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:42 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 20036/125097 objects degraded (16.016%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:42.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:42 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 20036/125097 objects degraded (16.016%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:42.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:42 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 20036/125097 objects degraded (16.016%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:43.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:42] "GET /metrics HTTP/1.1" 200 224092 "" "Prometheus/2.33.4" 2023-06-10T21:44:43.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:44:43.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:44:43.311 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:43.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:43.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:43.350 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:44:43.427 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:43.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:43 smithi152 ceph-mon[156819]: pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:43.540 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:43 smithi064 ceph-mon[180297]: pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:43.540 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:43 smithi064 ceph-mon[186706]: pgmap v1509: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:43.560 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:43.890 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:43.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:44.002 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:44.062 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:44.602 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:44.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:44 smithi064 ceph-mon[186706]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:44.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:44 smithi064 ceph-mon[180297]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:44 smithi152 ceph-mon[156819]: pgmap v1510: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.4 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 788 op/s; 19603/118107 objects degraded (16.598%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:44:44.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:44.772 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:44.863 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:44.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:44.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:44.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:45.042 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:45.061 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:45.149 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:45.150 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:45.227 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:45.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:45.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:45.409 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:45.487 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:45.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:45.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:45.594 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:45.627 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:45.683 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:45.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:45.791 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:45.818 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:45.855 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:45.859 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:44:45.860 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:45.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:45.968 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:46.019 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:46.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: osdmap e410: 8 total, 8 up, 4 in 2023-06-10T21:44:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:44:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:44:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: pgmap v1512: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 912 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:46.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 19603/118107 objects degraded (16.598%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:46.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: osdmap e410: 8 total, 8 up, 4 in 2023-06-10T21:44:46.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:46.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:44:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:44:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: pgmap v1512: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 912 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:46.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:46 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 19603/118107 objects degraded (16.598%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:46.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: osdmap e410: 8 total, 8 up, 4 in 2023-06-10T21:44:46.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: failed to encode map e410 with expected crc 2023-06-10T21:44:46.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='client.47039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:46.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:44:46.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:44:46.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:44:46.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: pgmap v1512: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 912 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:46.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:46 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 19603/118107 objects degraded (16.598%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:47.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: failed to encode map e410 with expected crc 2023-06-10T21:44:47.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: osdmap e411: 8 total, 8 up, 4 in 2023-06-10T21:44:47.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[186706]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: failed to encode map e410 with expected crc 2023-06-10T21:44:47.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: osdmap e411: 8 total, 8 up, 4 in 2023-06-10T21:44:47.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:47 smithi064 ceph-mon[180297]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: failed to encode map e410 with expected crc 2023-06-10T21:44:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: osdmap e411: 8 total, 8 up, 4 in 2023-06-10T21:44:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: failed to encode map e411 with expected crc 2023-06-10T21:44:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:47 smithi152 ceph-mon[156819]: failed to encode map e411 with expected crc 2023-06-10T21:44:48.120 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:48.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:48.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:48.229 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:48.293 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[186706]: failed to encode map e411 with expected crc 2023-06-10T21:44:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:48.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[186706]: pgmap v1514: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 720 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:48.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[180297]: failed to encode map e411 with expected crc 2023-06-10T21:44:48.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:48.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:48 smithi064 ceph-mon[180297]: pgmap v1514: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 720 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:48 smithi152 ceph-mon[156819]: failed to encode map e411 with expected crc 2023-06-10T21:44:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:44:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:48 smithi152 ceph-mon[156819]: pgmap v1514: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.2 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 720 op/s; 19518/107790 objects degraded (18.107%); 4.0 MiB/s, 64 objects/s recovering 2023-06-10T21:44:48.921 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:48.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:49.047 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:49.136 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:49.625 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:49.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:49.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:49.782 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:49.852 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1511,"stamp":"2023-06-10T21:44:45.317012+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2486507430,"num_objects":37957,"num_object_clones":0,"num_object_copies":113871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19603,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37957,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":182328,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7302,"num_bytes_recovered":475940878,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75545,"ondisk_log_size":75545,"up":267,"acting":256,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":8927492,"kb_used_data":6055076,"kb_used_omap":27,"kb_used_meta":2872164,"kb_avail":366053116,"statfs":{"total":383980142592,"available":374838390784,"internally_reserved":0,"allocated":6200397824,"data_stored":6196198934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2941096538},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,0,1,0,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-345702400,"num_objects":-5275,"num_object_clones":0,"num_object_copies":-15825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-433,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":433,"num_bytes_recovered":28377088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.122539"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":448,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.338711+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:44:29.338711+0000","last_peered":"2023-06-10T21:44:29.338711+0000","last_clean":"2023-06-10T21:44:29.338711+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:44:29.338711+0000","last_undegraded":"2023-06-10T21:44:29.338711+0000","last_fullsized":"2023-06-10T21:44:29.338711+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.350249+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:44:29.350249+0000","last_peered":"2023-06-10T21:44:29.350249+0000","last_clean":"2023-06-10T21:44:29.350249+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:44:29.350249+0000","last_undegraded":"2023-06-10T21:44:29.350249+0000","last_fullsized":"2023-06-10T21:44:29.350249+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":340,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341726+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:44:29.341726+0000","last_peered":"2023-06-10T21:44:29.341726+0000","last_clean":"2023-06-10T21:44:29.341726+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:44:29.341726+0000","last_undegraded":"2023-06-10T21:44:29.341726+0000","last_fullsized":"2023-06-10T21:44:29.341726+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":294,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344312+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:44:29.344312+0000","last_peered":"2023-06-10T21:44:29.344312+0000","last_clean":"2023-06-10T21:44:29.344312+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:44:29.344312+0000","last_undegraded":"2023-06-10T21:44:29.344312+0000","last_fullsized":"2023-06-10T21:44:29.344312+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":62,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341052+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:44:29.341052+0000","last_peered":"2023-06-10T21:44:29.341052+0000","last_clean":"2023-06-10T21:44:29.341052+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:44:29.341052+0000","last_undegraded":"2023-06-10T21:44:29.341052+0000","last_fullsized":"2023-06-10T21:44:29.341052+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335120+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:44:29.335120+0000","last_peered":"2023-06-10T21:44:29.335120+0000","last_clean":"2023-06-10T21:44:29.335120+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:44:29.335120+0000","last_undegraded":"2023-06-10T21:44:29.335120+0000","last_fullsized":"2023-06-10T21:44:29.335120+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":274,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334950+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:44:29.334950+0000","last_peered":"2023-06-10T21:44:29.334950+0000","last_clean":"2023-06-10T21:44:29.334950+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:44:29.334950+0000","last_undegraded":"2023-06-10T21:44:29.334950+0000","last_fullsized":"2023-06-10T21:44:29.334950+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341272+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:44:29.341272+0000","last_peered":"2023-06-10T21:44:29.341272+0000","last_clean":"2023-06-10T21:44:29.341272+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:44:29.341272+0000","last_undegraded":"2023-06-10T21:44:29.341272+0000","last_fullsized":"2023-06-10T21:44:29.341272+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":82,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335188+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:44:29.335188+0000","last_peered":"2023-06-10T21:44:29.335188+0000","last_clean":"2023-06-10T21:44:29.335188+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:44:29.335188+0000","last_undegraded":"2023-06-10T21:44:29.335188+0000","last_fullsized":"2023-06-10T21:44:29.335188+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":160,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:35.942576+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:44:35.942576+0000","last_peered":"2023-06-10T21:44:35.942576+0000","last_clean":"2023-06-10T21:44:35.942576+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:44:35.942576+0000","last_undegraded":"2023-06-10T21:44:35.942576+0000","last_fullsized":"2023-06-10T21:44:35.942576+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":166,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342405+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:44:29.342405+0000","last_peered":"2023-06-10T21:44:29.342405+0000","last_clean":"2023-06-10T21:44:29.342405+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:44:29.342405+0000","last_undegraded":"2023-06-10T21:44:29.342405+0000","last_fullsized":"2023-06-10T21:44:29.342405+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":408,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.338783+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:44:29.338783+0000","last_peered":"2023-06-10T21:44:29.338783+0000","last_clean":"2023-06-10T21:44:29.338783+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:44:29.338783+0000","last_undegraded":"2023-06-10T21:44:29.338783+0000","last_fullsized":"2023-06-10T21:44:29.338783+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":332,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341760+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:44:29.341760+0000","last_peered":"2023-06-10T21:44:29.341760+0000","last_clean":"2023-06-10T21:44:29.341760+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:44:29.341760+0000","last_undegraded":"2023-06-10T21:44:29.341760+0000","last_fullsized":"2023-06-10T21:44:29.341760+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335332+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:44:29.335332+0000","last_peered":"2023-06-10T21:44:29.335332+0000","last_clean":"2023-06-10T21:44:29.335332+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:44:29.335332+0000","last_undegraded":"2023-06-10T21:44:29.335332+0000","last_fullsized":"2023-06-10T21:44:29.335332+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":468,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.345295+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:44:29.345295+0000","last_peered":"2023-06-10T21:44:29.345295+0000","last_clean":"2023-06-10T21:44:29.345295+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:44:29.345295+0000","last_undegraded":"2023-06-10T21:44:29.345295+0000","last_fullsized":"2023-06-10T21:44:29.345295+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":356,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342132+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:44:29.342132+0000","last_peered":"2023-06-10T21:44:29.342132+0000","last_clean":"2023-06-10T21:44:29.342132+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:44:29.342132+0000","last_undegraded":"2023-06-10T21:44:29.342132+0000","last_fullsized":"2023-06-10T21:44:29.342132+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":279,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344205+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:44:29.344205+0000","last_peered":"2023-06-10T21:44:29.344205+0000","last_clean":"2023-06-10T21:44:29.344205+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:44:29.344205+0000","last_undegraded":"2023-06-10T21:44:29.344205+0000","last_fullsized":"2023-06-10T21:44:29.344205+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334869+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:44:29.334869+0000","last_peered":"2023-06-10T21:44:29.334869+0000","last_clean":"2023-06-10T21:44:29.334869+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:44:29.334869+0000","last_undegraded":"2023-06-10T21:44:29.334869+0000","last_fullsized":"2023-06-10T21:44:29.334869+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":421,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342591+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:44:29.342591+0000","last_peered":"2023-06-10T21:44:29.342591+0000","last_clean":"2023-06-10T21:44:29.342591+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:44:29.342591+0000","last_undegraded":"2023-06-10T21:44:29.342591+0000","last_fullsized":"2023-06-10T21:44:29.342591+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":62,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342708+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:44:29.342708+0000","last_peered":"2023-06-10T21:44:29.342708+0000","last_clean":"2023-06-10T21:44:29.342708+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:44:29.342708+0000","last_undegraded":"2023-06-10T21:44:29.342708+0000","last_fullsized":"2023-06-10T21:44:29.342708+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":433,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334627+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:44:29.334627+0000","last_peered":"2023-06-10T21:44:29.334627+0000","last_clean":"2023-06-10T21:44:29.334627+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:44:29.334627+0000","last_undegraded":"2023-06-10T21:44:29.334627+0000","last_fullsized":"2023-06-10T21:44:29.334627+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342063+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:44:29.342063+0000","last_peered":"2023-06-10T21:44:29.342063+0000","last_clean":"2023-06-10T21:44:29.342063+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:44:29.342063+0000","last_undegraded":"2023-06-10T21:44:29.342063+0000","last_fullsized":"2023-06-10T21:44:29.342063+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"409'11336","reported_seq":11382,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:41.124559+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:44:41.124559+0000","last_peered":"2023-06-10T21:44:41.124559+0000","last_clean":"2023-06-10T21:44:41.124559+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:44:41.124559+0000","last_undegraded":"2023-06-10T21:44:41.124559+0000","last_fullsized":"2023-06-10T21:44:41.124559+0000","mapping_epoch":373,"log_start":"388'6300","ondisk_log_start":"388'6300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155713536,"num_objects":2376,"num_object_clones":0,"num_object_copies":7128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11336,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":201,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.346601+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:44:29.346601+0000","last_peered":"2023-06-10T21:44:29.346601+0000","last_clean":"2023-06-10T21:44:29.346601+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:44:29.346601+0000","last_undegraded":"2023-06-10T21:44:29.346601+0000","last_fullsized":"2023-06-10T21:44:29.346601+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334419+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:44:29.334419+0000","last_peered":"2023-06-10T21:44:29.334419+0000","last_clean":"2023-06-10T21:44:29.334419+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:44:29.334419+0000","last_undegraded":"2023-06-10T21:44:29.334419+0000","last_fullsized":"2023-06-10T21:44:29.334419+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"409'11586","reported_seq":11648,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:44.097216+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:44:44.097216+0000","last_peered":"2023-06-10T21:44:44.097216+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:44:44.097216+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"393'7300","ondisk_log_start":"393'7300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11586,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":364}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":201,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344504+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:44:29.344504+0000","last_peered":"2023-06-10T21:44:29.344504+0000","last_clean":"2023-06-10T21:44:29.344504+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:44:29.344504+0000","last_undegraded":"2023-06-10T21:44:29.344504+0000","last_fullsized":"2023-06-10T21:44:29.344504+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":62,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341802+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:44:29.341802+0000","last_peered":"2023-06-10T21:44:29.341802+0000","last_clean":"2023-06-10T21:44:29.341802+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:44:29.341802+0000","last_undegraded":"2023-06-10T21:44:29.341802+0000","last_fullsized":"2023-06-10T21:44:29.341802+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"409'11368","reported_seq":11444,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:40.821308+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:44:40.821308+0000","last_peered":"2023-06-10T21:44:40.821308+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:44:40.821308+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"392'7000","ondisk_log_start":"392'7000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4368,"log_dups_size":0,"ondisk_log_size":4368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161873920,"num_objects":2470,"num_object_clones":0,"num_object_copies":7410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11368,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":542}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":488,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342456+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:44:29.342456+0000","last_peered":"2023-06-10T21:44:29.342456+0000","last_clean":"2023-06-10T21:44:29.342456+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:44:29.342456+0000","last_undegraded":"2023-06-10T21:44:29.342456+0000","last_fullsized":"2023-06-10T21:44:29.342456+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344083+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:44:29.344083+0000","last_peered":"2023-06-10T21:44:29.344083+0000","last_clean":"2023-06-10T21:44:29.344083+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:44:29.344083+0000","last_undegraded":"2023-06-10T21:44:29.344083+0000","last_fullsized":"2023-06-10T21:44:29.344083+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"409'11393","reported_seq":14422,"reported_epoch":409,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:40.844928+0000","last_change":"2023-06-10T21:44:21.632022+0000","last_active":"2023-06-10T21:44:40.844928+0000","last_peered":"2023-06-10T21:44:40.844928+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:43:02.514777+0000","last_became_peered":"2023-06-10T21:43:02.514777+0000","last_unstale":"2023-06-10T21:44:40.844928+0000","last_undegraded":"2023-06-10T21:43:02.215416+0000","last_fullsized":"2023-06-10T21:43:02.198507+0000","mapping_epoch":391,"log_start":"392'7084","ondisk_log_start":"392'7084","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4309,"log_dups_size":0,"ondisk_log_size":4309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:27:12.740821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158007296,"num_objects":2411,"num_object_clones":0,"num_object_copies":7233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":417,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11393,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1487,"num_bytes_recovered":97452032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1994}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334940+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:44:29.334940+0000","last_peered":"2023-06-10T21:44:29.334940+0000","last_clean":"2023-06-10T21:44:29.334940+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:44:29.334940+0000","last_undegraded":"2023-06-10T21:44:29.334940+0000","last_fullsized":"2023-06-10T21:44:29.334940+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.343314+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:44:29.343314+0000","last_peered":"2023-06-10T21:44:29.343314+0000","last_clean":"2023-06-10T21:44:29.343314+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:44:29.343314+0000","last_undegraded":"2023-06-10T21:44:29.343314+0000","last_fullsized":"2023-06-10T21:44:29.343314+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"409'11494","reported_seq":15350,"reported_epoch":409,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:44:44.679303+0000","last_change":"2023-06-10T21:44:23.399033+0000","last_active":"2023-06-10T21:44:44.679303+0000","last_peered":"2023-06-10T21:44:44.679303+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:44:23.337898+0000","last_became_peered":"2023-06-10T21:44:23.337898+0000","last_unstale":"2023-06-10T21:44:44.679303+0000","last_undegraded":"2023-06-10T21:44:44.679303+0000","last_fullsized":"2023-06-10T21:44:44.679303+0000","mapping_epoch":406,"log_start":"392'6940","ondisk_log_start":"392'6940","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11494,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":2234}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":66,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.343224+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:44:29.343224+0000","last_peered":"2023-06-10T21:44:29.343224+0000","last_clean":"2023-06-10T21:44:29.343224+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:44:29.343224+0000","last_undegraded":"2023-06-10T21:44:29.343224+0000","last_fullsized":"2023-06-10T21:44:29.343224+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":62,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342062+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:44:29.342062+0000","last_peered":"2023-06-10T21:44:29.342062+0000","last_clean":"2023-06-10T21:44:29.342062+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:44:29.342062+0000","last_undegraded":"2023-06-10T21:44:29.342062+0000","last_fullsized":"2023-06-10T21:44:29.342062+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"409'11287","reported_seq":15189,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:41.080360+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:44:41.080360+0000","last_peered":"2023-06-10T21:44:41.080360+0000","last_clean":"2023-06-10T21:44:41.080360+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:44:41.080360+0000","last_undegraded":"2023-06-10T21:44:41.080360+0000","last_fullsized":"2023-06-10T21:44:41.080360+0000","mapping_epoch":395,"log_start":"387'6200","ondisk_log_start":"387'6200","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158924800,"num_objects":2425,"num_object_clones":0,"num_object_copies":7275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":230,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344097+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:44:29.344097+0000","last_peered":"2023-06-10T21:44:29.344097+0000","last_clean":"2023-06-10T21:44:29.344097+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:44:29.344097+0000","last_undegraded":"2023-06-10T21:44:29.344097+0000","last_fullsized":"2023-06-10T21:44:29.344097+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":447,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341064+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:44:29.341064+0000","last_peered":"2023-06-10T21:44:29.341064+0000","last_clean":"2023-06-10T21:44:29.341064+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:44:29.341064+0000","last_undegraded":"2023-06-10T21:44:29.341064+0000","last_fullsized":"2023-06-10T21:44:29.341064+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"409'11338","reported_seq":11397,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:44.067907+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:44:44.067907+0000","last_peered":"2023-06-10T21:44:44.067907+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:44:44.067907+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"392'7000","ondisk_log_start":"392'7000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4338,"log_dups_size":0,"ondisk_log_size":4338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11338,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":381}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":424,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.338558+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:44:29.338558+0000","last_peered":"2023-06-10T21:44:29.338558+0000","last_clean":"2023-06-10T21:44:29.338558+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:44:29.338558+0000","last_undegraded":"2023-06-10T21:44:29.338558+0000","last_fullsized":"2023-06-10T21:44:29.338558+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342623+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:44:29.342623+0000","last_peered":"2023-06-10T21:44:29.342623+0000","last_clean":"2023-06-10T21:44:29.342623+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:44:29.342623+0000","last_undegraded":"2023-06-10T21:44:29.342623+0000","last_fullsized":"2023-06-10T21:44:29.342623+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"409'11242","reported_seq":11295,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:41.134742+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:44:41.134742+0000","last_peered":"2023-06-10T21:44:41.134742+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:44:41.134742+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"387'6174","ondisk_log_start":"387'6174","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383552,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11242,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":497}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341575+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:44:29.341575+0000","last_peered":"2023-06-10T21:44:29.341575+0000","last_clean":"2023-06-10T21:44:29.341575+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:44:29.341575+0000","last_undegraded":"2023-06-10T21:44:29.341575+0000","last_fullsized":"2023-06-10T21:44:29.341575+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":356,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342278+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:44:29.342278+0000","last_peered":"2023-06-10T21:44:29.342278+0000","last_clean":"2023-06-10T21:44:29.342278+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:44:29.342278+0000","last_undegraded":"2023-06-10T21:44:29.342278+0000","last_fullsized":"2023-06-10T21:44:29.342278+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"409'11138","reported_seq":11190,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:41.116896+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:44:41.116896+0000","last_peered":"2023-06-10T21:44:41.116896+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:44:41.116896+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"388'6119","ondisk_log_start":"388'6119","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5019,"log_dups_size":0,"ondisk_log_size":5019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161087488,"num_objects":2458,"num_object_clones":0,"num_object_copies":7374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11138,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":529}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":192,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341836+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:44:29.341836+0000","last_peered":"2023-06-10T21:44:29.341836+0000","last_clean":"2023-06-10T21:44:29.341836+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:44:29.341836+0000","last_undegraded":"2023-06-10T21:44:29.341836+0000","last_fullsized":"2023-06-10T21:44:29.341836+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":73,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334648+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:44:29.334648+0000","last_peered":"2023-06-10T21:44:29.334648+0000","last_clean":"2023-06-10T21:44:29.334648+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:44:29.334648+0000","last_undegraded":"2023-06-10T21:44:29.334648+0000","last_fullsized":"2023-06-10T21:44:29.334648+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":76,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334611+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:44:29.334611+0000","last_peered":"2023-06-10T21:44:29.334611+0000","last_clean":"2023-06-10T21:44:29.334611+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:44:29.334611+0000","last_undegraded":"2023-06-10T21:44:29.334611+0000","last_fullsized":"2023-06-10T21:44:29.334611+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"409'11420","reported_seq":11484,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:44.040831+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:44:44.040831+0000","last_peered":"2023-06-10T21:44:44.040831+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:44:44.040831+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"393'7100","ondisk_log_start":"393'7100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4320,"log_dups_size":0,"ondisk_log_size":4320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149618712,"num_objects":2284,"num_object_clones":0,"num_object_copies":6852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2284,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11420,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":367}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":92,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334531+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:44:29.334531+0000","last_peered":"2023-06-10T21:44:29.334531+0000","last_clean":"2023-06-10T21:44:29.334531+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:44:29.334531+0000","last_undegraded":"2023-06-10T21:44:29.334531+0000","last_fullsized":"2023-06-10T21:44:29.334531+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":370,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334965+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:44:29.334965+0000","last_peered":"2023-06-10T21:44:29.334965+0000","last_clean":"2023-06-10T21:44:29.334965+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:44:29.334965+0000","last_undegraded":"2023-06-10T21:44:29.334965+0000","last_fullsized":"2023-06-10T21:44:29.334965+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.346971+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:44:29.346971+0000","last_peered":"2023-06-10T21:44:29.346971+0000","last_clean":"2023-06-10T21:44:29.346971+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:44:29.346971+0000","last_undegraded":"2023-06-10T21:44:29.346971+0000","last_fullsized":"2023-06-10T21:44:29.346971+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"409'11426","reported_seq":11490,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:44.109639+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:44:44.109639+0000","last_peered":"2023-06-10T21:44:44.109639+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:44:44.109639+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"393'7100","ondisk_log_start":"393'7100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4326,"log_dups_size":0,"ondisk_log_size":4326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147587072,"num_objects":2252,"num_object_clones":0,"num_object_copies":6756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11426,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":379}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":294,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.338463+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:44:29.338463+0000","last_peered":"2023-06-10T21:44:29.338463+0000","last_clean":"2023-06-10T21:44:29.338463+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:44:29.338463+0000","last_undegraded":"2023-06-10T21:44:29.338463+0000","last_fullsized":"2023-06-10T21:44:29.338463+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":298,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.336856+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:44:29.336856+0000","last_peered":"2023-06-10T21:44:29.336856+0000","last_clean":"2023-06-10T21:44:29.336856+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:44:29.336856+0000","last_undegraded":"2023-06-10T21:44:29.336856+0000","last_fullsized":"2023-06-10T21:44:29.336856+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":91,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335036+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:44:29.335036+0000","last_peered":"2023-06-10T21:44:29.335036+0000","last_clean":"2023-06-10T21:44:29.335036+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:44:29.335036+0000","last_undegraded":"2023-06-10T21:44:29.335036+0000","last_fullsized":"2023-06-10T21:44:29.335036+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"409'11379","reported_seq":11432,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:41.131773+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:44:41.131773+0000","last_peered":"2023-06-10T21:44:41.131773+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:44:41.131773+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"388'6300","ondisk_log_start":"388'6300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5079,"log_dups_size":0,"ondisk_log_size":5079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164036608,"num_objects":2503,"num_object_clones":0,"num_object_copies":7509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11379,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":539}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":83,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335243+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:44:29.335243+0000","last_peered":"2023-06-10T21:44:29.335243+0000","last_clean":"2023-06-10T21:44:29.335243+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:44:29.335243+0000","last_undegraded":"2023-06-10T21:44:29.335243+0000","last_fullsized":"2023-06-10T21:44:29.335243+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":609,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.343842+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:44:29.343842+0000","last_peered":"2023-06-10T21:44:29.343842+0000","last_clean":"2023-06-10T21:44:29.343842+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:44:29.343842+0000","last_undegraded":"2023-06-10T21:44:29.343842+0000","last_fullsized":"2023-06-10T21:44:29.343842+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":78,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:39.149293+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:44:39.149293+0000","last_peered":"2023-06-10T21:44:39.149293+0000","last_clean":"2023-06-10T21:44:39.149293+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:44:39.149293+0000","last_undegraded":"2023-06-10T21:44:39.149293+0000","last_fullsized":"2023-06-10T21:44:39.149293+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":292,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342555+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:44:29.342555+0000","last_peered":"2023-06-10T21:44:29.342555+0000","last_clean":"2023-06-10T21:44:29.342555+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:44:29.342555+0000","last_undegraded":"2023-06-10T21:44:29.342555+0000","last_fullsized":"2023-06-10T21:44:29.342555+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"409'11357","reported_seq":15264,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:44.067888+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:44:44.067888+0000","last_peered":"2023-06-10T21:44:44.067888+0000","last_clean":"2023-06-10T21:44:44.067888+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:44:44.067888+0000","last_undegraded":"2023-06-10T21:44:44.067888+0000","last_fullsized":"2023-06-10T21:44:44.067888+0000","mapping_epoch":401,"log_start":"392'7000","ondisk_log_start":"392'7000","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4357,"log_dups_size":0,"ondisk_log_size":4357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147390464,"num_objects":2249,"num_object_clones":0,"num_object_copies":6747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11357,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":461,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.343938+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:44:29.343938+0000","last_peered":"2023-06-10T21:44:29.343938+0000","last_clean":"2023-06-10T21:44:29.343938+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:44:29.343938+0000","last_undegraded":"2023-06-10T21:44:29.343938+0000","last_fullsized":"2023-06-10T21:44:29.343938+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:39.148198+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:44:39.148198+0000","last_peered":"2023-06-10T21:44:39.148198+0000","last_clean":"2023-06-10T21:44:39.148198+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:44:39.148198+0000","last_undegraded":"2023-06-10T21:44:39.148198+0000","last_fullsized":"2023-06-10T21:44:39.148198+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":160,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342438+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:44:29.342438+0000","last_peered":"2023-06-10T21:44:29.342438+0000","last_clean":"2023-06-10T21:44:29.342438+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:44:29.342438+0000","last_undegraded":"2023-06-10T21:44:29.342438+0000","last_fullsized":"2023-06-10T21:44:29.342438+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"409'11325","reported_seq":11371,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-06-10T21:44:41.127776+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:44:41.127776+0000","last_peered":"2023-06-10T21:44:41.127776+0000","last_clean":"2023-06-10T21:44:41.127776+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:44:41.127776+0000","last_undegraded":"2023-06-10T21:44:41.127776+0000","last_fullsized":"2023-06-10T21:44:41.127776+0000","mapping_epoch":373,"log_start":"388'6300","ondisk_log_start":"388'6300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5025,"log_dups_size":0,"ondisk_log_size":5025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157876224,"num_objects":2409,"num_object_clones":0,"num_object_copies":7227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11325,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":440,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342509+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:44:29.342509+0000","last_peered":"2023-06-10T21:44:29.342509+0000","last_clean":"2023-06-10T21:44:29.342509+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:44:29.342509+0000","last_undegraded":"2023-06-10T21:44:29.342509+0000","last_fullsized":"2023-06-10T21:44:29.342509+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":289,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.347024+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:44:29.347024+0000","last_peered":"2023-06-10T21:44:29.347024+0000","last_clean":"2023-06-10T21:44:29.347024+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:44:29.347024+0000","last_undegraded":"2023-06-10T21:44:29.347024+0000","last_fullsized":"2023-06-10T21:44:29.347024+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":213,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335047+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:44:29.335047+0000","last_peered":"2023-06-10T21:44:29.335047+0000","last_clean":"2023-06-10T21:44:29.335047+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:44:29.335047+0000","last_undegraded":"2023-06-10T21:44:29.335047+0000","last_fullsized":"2023-06-10T21:44:29.335047+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"409'11598","reported_seq":11652,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:41.126505+0000","last_change":"2023-06-10T21:43:02.588633+0000","last_active":"2023-06-10T21:44:41.126505+0000","last_peered":"2023-06-10T21:44:41.126505+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:44:41.126505+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"388'6500","ondisk_log_start":"388'6500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1976,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11598,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":520}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":419,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342151+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:44:29.342151+0000","last_peered":"2023-06-10T21:44:29.342151+0000","last_clean":"2023-06-10T21:44:29.342151+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:44:29.342151+0000","last_undegraded":"2023-06-10T21:44:29.342151+0000","last_fullsized":"2023-06-10T21:44:29.342151+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":230,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.342940+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:44:29.342940+0000","last_peered":"2023-06-10T21:44:29.342940+0000","last_clean":"2023-06-10T21:44:29.342940+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:44:29.342940+0000","last_undegraded":"2023-06-10T21:44:29.342940+0000","last_fullsized":"2023-06-10T21:44:29.342940+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":78,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:35.942523+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:44:35.942523+0000","last_peered":"2023-06-10T21:44:35.942523+0000","last_clean":"2023-06-10T21:44:35.942523+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:44:35.942523+0000","last_undegraded":"2023-06-10T21:44:35.942523+0000","last_fullsized":"2023-06-10T21:44:35.942523+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"409'11380","reported_seq":11440,"reported_epoch":409,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:41.128131+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:44:41.128131+0000","last_peered":"2023-06-10T21:44:41.128131+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:44:41.128131+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"388'6300","ondisk_log_start":"388'6300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160301056,"num_objects":2446,"num_object_clones":0,"num_object_copies":7338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11380,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":525}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":157,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.346671+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:44:29.346671+0000","last_peered":"2023-06-10T21:44:29.346671+0000","last_clean":"2023-06-10T21:44:29.346671+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:44:29.346671+0000","last_undegraded":"2023-06-10T21:44:29.346671+0000","last_fullsized":"2023-06-10T21:44:29.346671+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":271,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344189+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:44:29.344189+0000","last_peered":"2023-06-10T21:44:29.344189+0000","last_clean":"2023-06-10T21:44:29.344189+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:44:29.344189+0000","last_undegraded":"2023-06-10T21:44:29.344189+0000","last_fullsized":"2023-06-10T21:44:29.344189+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":63,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335906+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:44:29.335906+0000","last_peered":"2023-06-10T21:44:29.335906+0000","last_clean":"2023-06-10T21:44:29.335906+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:44:29.335906+0000","last_undegraded":"2023-06-10T21:44:29.335906+0000","last_fullsized":"2023-06-10T21:44:29.335906+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":108,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.345392+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:44:29.345392+0000","last_peered":"2023-06-10T21:44:29.345392+0000","last_clean":"2023-06-10T21:44:29.345392+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:44:29.345392+0000","last_undegraded":"2023-06-10T21:44:29.345392+0000","last_fullsized":"2023-06-10T21:44:29.345392+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":305,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.347054+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:44:29.347054+0000","last_peered":"2023-06-10T21:44:29.347054+0000","last_clean":"2023-06-10T21:44:29.347054+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:44:29.347054+0000","last_undegraded":"2023-06-10T21:44:29.347054+0000","last_fullsized":"2023-06-10T21:44:29.347054+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334778+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:44:29.334778+0000","last_peered":"2023-06-10T21:44:29.334778+0000","last_clean":"2023-06-10T21:44:29.334778+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:44:29.334778+0000","last_undegraded":"2023-06-10T21:44:29.334778+0000","last_fullsized":"2023-06-10T21:44:29.334778+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":313,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.343596+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:44:29.343596+0000","last_peered":"2023-06-10T21:44:29.343596+0000","last_clean":"2023-06-10T21:44:29.343596+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:44:29.343596+0000","last_undegraded":"2023-06-10T21:44:29.343596+0000","last_fullsized":"2023-06-10T21:44:29.343596+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.334630+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:44:29.334630+0000","last_peered":"2023-06-10T21:44:29.334630+0000","last_clean":"2023-06-10T21:44:29.334630+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:44:29.334630+0000","last_undegraded":"2023-06-10T21:44:29.334630+0000","last_fullsized":"2023-06-10T21:44:29.334630+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":384,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.344188+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:44:29.344188+0000","last_peered":"2023-06-10T21:44:29.344188+0000","last_clean":"2023-06-10T21:44:29.344188+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:44:29.344188+0000","last_undegraded":"2023-06-10T21:44:29.344188+0000","last_fullsized":"2023-06-10T21:44:29.344188+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":63,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:35.943960+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:44:35.943960+0000","last_peered":"2023-06-10T21:44:35.943960+0000","last_clean":"2023-06-10T21:44:35.943960+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:44:35.943960+0000","last_undegraded":"2023-06-10T21:44:35.943960+0000","last_fullsized":"2023-06-10T21:44:35.943960+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341522+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:44:29.341522+0000","last_peered":"2023-06-10T21:44:29.341522+0000","last_clean":"2023-06-10T21:44:29.341522+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:44:29.341522+0000","last_undegraded":"2023-06-10T21:44:29.341522+0000","last_fullsized":"2023-06-10T21:44:29.341522+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":330,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.335364+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:44:29.335364+0000","last_peered":"2023-06-10T21:44:29.335364+0000","last_clean":"2023-06-10T21:44:29.335364+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:44:29.335364+0000","last_undegraded":"2023-06-10T21:44:29.335364+0000","last_fullsized":"2023-06-10T21:44:29.335364+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-06-10T21:44:29.341644+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:44:29.341644+0000","last_peered":"2023-06-10T21:44:29.341644+0000","last_clean":"2023-06-10T21:44:29.341644+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:44:29.341644+0000","last_undegraded":"2023-06-10T21:44:29.341644+0000","last_fullsized":"2023-06-10T21:44:29.341644+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2485911576,"num_objects":37933,"num_object_clones":0,"num_object_copies":113799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19603,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37933,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182067,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7227,"num_bytes_recovered":473563136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6109736960,"data_stored":6109724744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75350,"ondisk_log_size":75350,"up":48,"acting":37,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430772,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2575608,"kb_used_data":1621040,"kb_used_omap":0,"kb_used_meta":954560,"kb_avail":91169544,"statfs":{"total":95995035648,"available":93357613056,"internally_reserved":0,"allocated":1659944960,"data_stored":1659236889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":977469440},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.3089999999999999,"15min":3.1179999999999999},"min":{"1min":0.28599999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.2699999999999996,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.244,"5min":1.552,"15min":3.371},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":3.0960000000000001,"5min":22.462,"15min":185.58000000000001},"last":0.44}]},{"osd":1,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.8100000000000001,"15min":2.6560000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.1500000000000004,"5min":38.078000000000003,"15min":87.417000000000002},"last":0.93700000000000006},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.2509999999999999,"15min":2.9590000000000001},"min":{"1min":0.30299999999999999,"5min":0.28599999999999998,"15min":0.27300000000000002},"max":{"1min":10.484,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:44:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.1299999999999999,"15min":4.9729999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":36.627000000000002,"5min":75.219999999999999,"15min":172.33600000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":4.4880000000000004,"5min":3.1379999999999999,"15min":4.9610000000000003},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":36.533999999999999,"5min":75.108999999999995,"15min":172.185},"last":0.82799999999999996}]},{"osd":3,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":7.7990000000000004,"5min":4.0899999999999999,"15min":4.7960000000000003},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":71.852000000000004,"5min":89.504999999999995,"15min":147.00700000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":7.2359999999999998,"5min":3.7599999999999998,"15min":4.5270000000000001},"min":{"1min":0.439,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":72.813999999999993,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.54200000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":0.88900000000000001,"15min":0.59599999999999997},"min":{"1min":0.35499999999999998,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":13.429,"5min":13.429,"15min":13.429},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1120000000000001,"15min":0.71699999999999997},"min":{"1min":0.34200000000000003,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":4.6760000000000002,"5min":22.77,"15min":22.77},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:44:04 2023","interfaces":[{"interface":"back","average":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":131.619,"5min":131.619,"15min":131.619},"last":4.29},{"interface":"front","average":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":134.928,"5min":134.928,"15min":134.928},"last":0.498}]},{"osd":6,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":0.89700000000000002,"15min":0.67000000000000004},"min":{"1min":0.41199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8280000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.911,"5min":0.99099999999999999,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":4.3739999999999997,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.42399999999999999}]}]},{"osd":6,"up_from":0,"seq":180388627000,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566442,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020419,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075830,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2872880,"kb_used_data":1879720,"kb_used_omap":7,"kb_used_meta":993144,"kb_avail":90872272,"statfs":{"total":95995035648,"available":93053206528,"internally_reserved":0,"allocated":1924833280,"data_stored":1923496446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1016979537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.903,"15min":4.4560000000000004},"min":{"1min":0.27500000000000002,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":8.5899999999999999,"5min":83.733000000000004,"15min":171.33699999999999},"last":3.3969999999999998},{"interface":"front","average":{"1min":1.673,"5min":2.4100000000000001,"15min":4.3440000000000003},"min":{"1min":0.255,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.9400000000000004,"5min":82.322999999999993,"15min":171.56200000000001},"last":3.9300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":4.4500000000000002,"15min":5.3940000000000001},"min":{"1min":0.23499999999999999,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":4.1449999999999996,"5min":85.709999999999994,"15min":171.286},"last":2.8849999999999998},{"interface":"front","average":{"1min":2.4020000000000001,"5min":4.109,"15min":3.9550000000000001},"min":{"1min":0.252,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.0500000000000007,"5min":85.501999999999995,"15min":147.81299999999999},"last":3.5569999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:44:25 2023","interfaces":[{"interface":"back","average":{"1min":11.705,"5min":8.6259999999999994,"15min":6.6699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":100.786,"5min":145.07599999999999,"15min":171.78299999999999},"last":8.9329999999999998},{"interface":"front","average":{"1min":11.17,"5min":7.9969999999999999,"15min":6.6219999999999999},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":100.73999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":3.996}]},{"osd":4,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":1.9339999999999999,"5min":1.343,"15min":2.7839999999999998},"min":{"1min":0.46700000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.681,"5min":9.9390000000000001,"15min":171.48699999999999},"last":8.9629999999999992},{"interface":"front","average":{"1min":1.7010000000000001,"5min":2.1859999999999999,"15min":3.6680000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.27000000000000002},"max":{"1min":4.673,"5min":15.544,"15min":171.18100000000001},"last":2.944}]},{"osd":5,"last update":"Sat Jun 10 21:44:02 2023","interfaces":[{"interface":"back","average":{"1min":6.9820000000000002,"5min":6.9820000000000002,"15min":6.9820000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":75.896000000000001,"5min":75.896000000000001,"15min":75.896000000000001},"last":3.4340000000000002},{"interface":"front","average":{"1min":6.3419999999999996,"5min":6.3419999999999996,"15min":6.3419999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":76.337000000000003,"5min":76.337000000000003,"15min":76.337000000000003},"last":3.9660000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":1.5449999999999999,"15min":2.472},"min":{"1min":0.496,"5min":0.307,"15min":0.307},"max":{"1min":5.7220000000000004,"5min":9.9049999999999994,"15min":171.441},"last":9.0139999999999993},{"interface":"front","average":{"1min":1.8200000000000001,"5min":2.343,"15min":3.3370000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.1600000000000001,"5min":15.590999999999999,"15min":171.24600000000001},"last":2.9769999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:43:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4039999999999999,"5min":1.9370000000000001,"15min":4.4699999999999998},"min":{"1min":0.46500000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.6560000000000006,"5min":18.16,"15min":219.131},"last":8.9969999999999999},{"interface":"front","average":{"1min":1.835,"5min":1.4419999999999999,"15min":4.4370000000000003},"min":{"1min":0.503,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5789999999999997,"5min":9.8719999999999999,"15min":219.19},"last":8.9130000000000003}]}]},{"osd":3,"up_from":250,"seq":1073741824190,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2444204,"kb_used_data":1724196,"kb_used_omap":7,"kb_used_meta":719800,"kb_avail":91300948,"statfs":{"total":95995035648,"available":93492170752,"internally_reserved":0,"allocated":1765576704,"data_stored":1764122993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":737075219},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6319999999999999,"15min":3.2730000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.5880000000000001,"5min":30.744,"15min":105.09},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.375,"5min":2.0739999999999998,"15min":3.3740000000000001},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.20799999999999999},"max":{"1min":4.4450000000000003,"5min":30.805,"15min":99.852999999999994},"last":3.6549999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.8280000000000001,"15min":3.3660000000000001},"min":{"1min":0.315,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":4.5300000000000002,"5min":13.249000000000001,"15min":94.766000000000005},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.375,"5min":1.589,"15min":3.2789999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":5.4109999999999996,"5min":11.846,"15min":94.691000000000003},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.6499999999999999,"15min":4.319},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.9649999999999999,"5min":18.452000000000002,"15min":231.07900000000001},"last":3.7040000000000002},{"interface":"front","average":{"1min":2.1000000000000001,"5min":3.0259999999999998,"15min":5.5599999999999996},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":9.3930000000000007,"5min":18.733000000000001,"15min":231.58600000000001},"last":3.5510000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.5900000000000001,"15min":1.486},"min":{"1min":0.45600000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.8470000000000004,"5min":9.5150000000000006,"15min":13.417},"last":3.7309999999999999},{"interface":"front","average":{"1min":1.843,"5min":1.8300000000000001,"15min":2.2599999999999998},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":4.5049999999999999,"5min":10.722,"15min":19.728000000000002},"last":0.36399999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:44:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.532,"5min":6.532,"15min":6.532},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.7040000000000002,"5min":2.7040000000000002,"15min":2.7040000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":3.7549999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.903,"15min":2.323},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8519999999999999,"5min":10.691000000000001,"15min":19.805},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.484,"15min":1.5529999999999999},"min":{"1min":0.48199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.6299999999999999,"5min":9.9719999999999995,"15min":17.327000000000002},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.1179999999999999,"15min":2.601},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":5.4109999999999996,"5min":10.648999999999999,"15min":79.984999999999999},"last":3.5790000000000002},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.603,"15min":1.847},"min":{"1min":0.51300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":5.9320000000000004,"5min":9.9380000000000006,"15min":80.272999999999996},"last":0.58499999999999996}]}]},{"osd":4,"up_from":0,"seq":128849019456,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245462,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1034800,"kb_used_data":830120,"kb_used_omap":11,"kb_used_meta":204660,"kb_avail":92710352,"statfs":{"total":95995035648,"available":94935400448,"internally_reserved":0,"allocated":850042880,"data_stored":849342606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":209572342},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":2.0800000000000001,"15min":2.0800000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":8.2590000000000003},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.266,"5min":4.266,"15min":4.266},"last":3.0739999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.54,"5min":7.54,"15min":7.54},"last":8.2170000000000005},{"interface":"front","average":{"1min":2.282,"5min":2.282,"15min":2.282},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.7989999999999995,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":3.1360000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":1.8620000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":3.1070000000000002},{"interface":"front","average":{"1min":2.306,"5min":2.306,"15min":2.306},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.641,"5min":12.641,"15min":12.641},"last":3.0630000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":8.2330000000000005,"5min":8.2330000000000005,"15min":8.2330000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":133.07499999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":7.8949999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":133.03800000000001,"5min":133.03800000000001,"15min":133.03800000000001},"last":3.665}]},{"osd":4,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.173,"5min":5.173,"15min":5.173},"last":0.80000000000000004},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.3079999999999998,"15min":2.3079999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.476,"5min":7.476,"15min":7.476},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.8879999999999999,"15min":1.8879999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.345,"5min":1.345,"15min":1.345},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":5.2430000000000003},{"interface":"front","average":{"1min":2.891,"5min":2.891,"15min":2.891},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.4160000000000004,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":0.33900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1910964224,"data_stored":1910964224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1756434432,"data_stored":1756430360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":807407616,"data_stored":807403544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1634930688,"data_stored":1634926616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:44:49.865 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:49.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:49.979 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:50.050 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:50.159 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:50.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:50.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:50.270 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:50.301 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:50 smithi152 ceph-mon[156819]: pgmap v1515: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 645 op/s; 19101/102615 objects degraded (18.614%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:44:50.323 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:50.383 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:50.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:50.484 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:44:50.485 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:44:50.485 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:44:50.485 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:44:50.485 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:44:50.494 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:50.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:50 smithi064 ceph-mon[180297]: pgmap v1515: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 645 op/s; 19101/102615 objects degraded (18.614%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:44:50.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:50 smithi064 ceph-mon[186706]: pgmap v1515: 89 pgs: 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 2.1 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 645 op/s; 19101/102615 objects degraded (18.614%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:44:50.544 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:50.628 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:50.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:50.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:50.659 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:44:50.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:44:50.737 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:50.752 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:44:50.792 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:44:50.807 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:50.824 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:44:50.885 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:44:50.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:50.919 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:44:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:50.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:50.950 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.968 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.969 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:44:50.969 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:44:51.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:51.090 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:51.174 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:51.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:51.282 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:51.343 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:44:51.355 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4004879612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1263420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:51 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 19101/102615 objects degraded (18.614%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:51.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4004879612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1263420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 19101/102615 objects degraded (18.614%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4004879612' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:44:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1263420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:44:51.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:51 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 19101/102615 objects degraded (18.614%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:52.066 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:52.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:52.173 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:52.231 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:52.339 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:52.340 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:52.340 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:52.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: osdmap e412: 8 total, 8 up, 4 in 2023-06-10T21:44:52.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 872 op/s; 18773/95841 objects degraded (19.588%); 7.5 MiB/s, 120 objects/s recovering 2023-06-10T21:44:52.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[180297]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.343 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: osdmap e412: 8 total, 8 up, 4 in 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 872 op/s; 18773/95841 objects degraded (19.588%); 7.5 MiB/s, 120 objects/s recovering 2023-06-10T21:44:52.344 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:52 smithi064 ceph-mon[186706]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:52.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:52.447 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:52.473 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:52.514 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:52.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: osdmap e412: 8 total, 8 up, 4 in 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: pgmap v1517: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 872 op/s; 18773/95841 objects degraded (19.588%); 7.5 MiB/s, 120 objects/s recovering 2023-06-10T21:44:52.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:52 smithi152 ceph-mon[156819]: failed to encode map e412 with expected crc 2023-06-10T21:44:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:52.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:52.622 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:52.651 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:52.747 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:52.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:52.858 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:44:52.864 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:52.929 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:44:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:44:52] "GET /metrics HTTP/1.1" 200 224091 "" "Prometheus/2.33.4" 2023-06-10T21:44:52.932 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:53.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:44:53.575 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:53.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:53.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:53.745 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:53.836 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:53.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:53.906 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:53 smithi064 ceph-mon[186706]: failed to encode map e412 with expected crc 2023-06-10T21:44:53.909 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:53.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:53 smithi064 ceph-mon[180297]: failed to encode map e412 with expected crc 2023-06-10T21:44:53.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:53 smithi152 ceph-mon[156819]: failed to encode map e412 with expected crc 2023-06-10T21:44:53.947 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:53.989 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:54.102 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:54.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:54.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:54.229 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:54.286 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:54.381 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:54.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:54.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:54.569 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:54.663 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:54.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:54.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:54.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:54.838 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:54 smithi152 ceph-mon[156819]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 497 op/s; 18773/95841 objects degraded (19.588%); 5.8 MiB/s, 93 objects/s recovering 2023-06-10T21:44:54.843 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:54.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:54 smithi064 ceph-mon[186706]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 497 op/s; 18773/95841 objects degraded (19.588%); 5.8 MiB/s, 93 objects/s recovering 2023-06-10T21:44:54.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:54 smithi064 ceph-mon[180297]: pgmap v1518: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.9 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 497 op/s; 18773/95841 objects degraded (19.588%); 5.8 MiB/s, 93 objects/s recovering 2023-06-10T21:44:54.945 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:54.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:54.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:55.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:55.116 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:55.251 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:55.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:55.363 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:55.389 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:55.431 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:55.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:55.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:55.565 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:55.608 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:55.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:55.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:55.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:44:55.701 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:44:55.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:55.858 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:56.340 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:56.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:56.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:56.445 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:56.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:56 smithi064 ceph-mon[186706]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 611 op/s; 18773/89646 objects degraded (20.941%); 4.7 MiB/s, 75 objects/s recovering 2023-06-10T21:44:56.542 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:56 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18773/95841 objects degraded (19.588%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:56.542 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:56 smithi064 ceph-mon[180297]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 611 op/s; 18773/89646 objects degraded (20.941%); 4.7 MiB/s, 75 objects/s recovering 2023-06-10T21:44:56.543 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:56 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18773/95841 objects degraded (19.588%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:56.547 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:56.649 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:56.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:44:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:56 smithi152 ceph-mon[156819]: pgmap v1519: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.8 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 611 op/s; 18773/89646 objects degraded (20.941%); 4.7 MiB/s, 75 objects/s recovering 2023-06-10T21:44:56.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:56 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18773/95841 objects degraded (19.588%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:44:56.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:44:56.864 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:56.943 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:56.986 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:44:57.081 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:44:57.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:57.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:44:57.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:44:57.263 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:57.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:44:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:57.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:44:57.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:44:57.798 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:57.837 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:44:57.837 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:44:57.893 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:44:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:57.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:44:58.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:44:58.058 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:58.128 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:44:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:58.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:44:58.206 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:44:58.208 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:44:58.238 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:44:58.388 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:58.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:58 smithi152 ceph-mon[156819]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:58.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:44:58 smithi152 ceph-mon[156819]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 18415/83445 objects degraded (22.068%); 6.9 MiB/s, 110 objects/s recovering 2023-06-10T21:44:58.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:58 smithi064 ceph-mon[180297]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:58.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:44:58 smithi064 ceph-mon[180297]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 18415/83445 objects degraded (22.068%); 6.9 MiB/s, 110 objects/s recovering 2023-06-10T21:44:58.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:58 smithi064 ceph-mon[186706]: from='client.47051 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:44:58.520 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:44:58 smithi064 ceph-mon[186706]: pgmap v1520: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.7 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 811 op/s; 18415/83445 objects degraded (22.068%); 6.9 MiB/s, 110 objects/s recovering 2023-06-10T21:44:58.550 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:44:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:58.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:44:58.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:44:58.713 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:58.763 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:44:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:58.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:44:58.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:44:58.954 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:59.042 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:44:59.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:59.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:44:59.161 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:44:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:44:59.983 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:44:59.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:44:59.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:00.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:00.149 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:00.286 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:00.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:00.305 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:00 smithi064 ceph-mon[186706]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 18415/78372 objects degraded (23.497%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:45:00.305 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:00 smithi064 ceph-mon[180297]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 18415/78372 objects degraded (23.497%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:45:00.403 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:00.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:00 smithi152 ceph-mon[156819]: pgmap v1521: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.6 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 807 op/s; 18415/78372 objects degraded (23.497%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:45:00.460 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:01.398 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1519,"stamp":"2023-06-10T21:44:56.203406+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1957304230,"num_objects":29882,"num_object_clones":0,"num_object_copies":89646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29882,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":190403,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8133,"num_bytes_recovered":530335758,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75620,"ondisk_log_size":75620,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":7520616,"kb_used_data":4614408,"kb_used_omap":27,"kb_used_meta":2905956,"kb_avail":367459992,"statfs":{"total":383980142592,"available":376279031808,"internally_reserved":0,"allocated":4725153792,"data_stored":4720941512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":2975699546},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,1,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-396361728,"num_objects":-6048,"num_object_clones":0,"num_object_copies":-18144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-745,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6048,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":746,"num_bytes_recovered":48824320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.886003"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":450,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.325513+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:44:45.325513+0000","last_peered":"2023-06-10T21:44:45.325513+0000","last_clean":"2023-06-10T21:44:45.325513+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:44:45.325513+0000","last_undegraded":"2023-06-10T21:44:45.325513+0000","last_fullsized":"2023-06-10T21:44:45.325513+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.320022+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:44:46.320022+0000","last_peered":"2023-06-10T21:44:46.320022+0000","last_clean":"2023-06-10T21:44:46.320022+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:44:46.320022+0000","last_undegraded":"2023-06-10T21:44:46.320022+0000","last_fullsized":"2023-06-10T21:44:46.320022+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":343,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319042+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:44:46.319042+0000","last_peered":"2023-06-10T21:44:46.319042+0000","last_clean":"2023-06-10T21:44:46.319042+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:44:46.319042+0000","last_undegraded":"2023-06-10T21:44:46.319042+0000","last_fullsized":"2023-06-10T21:44:46.319042+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":297,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540444+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:44:46.540444+0000","last_peered":"2023-06-10T21:44:46.540444+0000","last_clean":"2023-06-10T21:44:46.540444+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:44:46.540444+0000","last_undegraded":"2023-06-10T21:44:46.540444+0000","last_fullsized":"2023-06-10T21:44:46.540444+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":65,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.318688+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:44:46.318688+0000","last_peered":"2023-06-10T21:44:46.318688+0000","last_clean":"2023-06-10T21:44:46.318688+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:44:46.318688+0000","last_undegraded":"2023-06-10T21:44:46.318688+0000","last_fullsized":"2023-06-10T21:44:46.318688+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":94,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.325214+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:44:46.325214+0000","last_peered":"2023-06-10T21:44:46.325214+0000","last_clean":"2023-06-10T21:44:46.325214+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:44:46.325214+0000","last_undegraded":"2023-06-10T21:44:46.325214+0000","last_fullsized":"2023-06-10T21:44:46.325214+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":277,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.325155+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:44:46.325155+0000","last_peered":"2023-06-10T21:44:46.325155+0000","last_clean":"2023-06-10T21:44:46.325155+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:44:46.325155+0000","last_undegraded":"2023-06-10T21:44:46.325155+0000","last_fullsized":"2023-06-10T21:44:46.325155+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.318788+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:44:46.318788+0000","last_peered":"2023-06-10T21:44:46.318788+0000","last_clean":"2023-06-10T21:44:46.318788+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:44:46.318788+0000","last_undegraded":"2023-06-10T21:44:46.318788+0000","last_fullsized":"2023-06-10T21:44:46.318788+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":85,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.325078+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:44:46.325078+0000","last_peered":"2023-06-10T21:44:46.325078+0000","last_clean":"2023-06-10T21:44:46.325078+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:44:46.325078+0000","last_undegraded":"2023-06-10T21:44:46.325078+0000","last_fullsized":"2023-06-10T21:44:46.325078+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":163,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319032+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:44:46.319032+0000","last_peered":"2023-06-10T21:44:46.319032+0000","last_clean":"2023-06-10T21:44:46.319032+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:44:46.319032+0000","last_undegraded":"2023-06-10T21:44:46.319032+0000","last_fullsized":"2023-06-10T21:44:46.319032+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":169,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537500+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:44:46.537500+0000","last_peered":"2023-06-10T21:44:46.537500+0000","last_clean":"2023-06-10T21:44:46.537500+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:44:46.537500+0000","last_undegraded":"2023-06-10T21:44:46.537500+0000","last_fullsized":"2023-06-10T21:44:46.537500+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":410,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.325594+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:44:45.325594+0000","last_peered":"2023-06-10T21:44:45.325594+0000","last_clean":"2023-06-10T21:44:45.325594+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:44:45.325594+0000","last_undegraded":"2023-06-10T21:44:45.325594+0000","last_fullsized":"2023-06-10T21:44:45.325594+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":335,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537311+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:44:46.537311+0000","last_peered":"2023-06-10T21:44:46.537311+0000","last_clean":"2023-06-10T21:44:46.537311+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:44:46.537311+0000","last_undegraded":"2023-06-10T21:44:46.537311+0000","last_fullsized":"2023-06-10T21:44:46.537311+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324794+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:44:46.324794+0000","last_peered":"2023-06-10T21:44:46.324794+0000","last_clean":"2023-06-10T21:44:46.324794+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:44:46.324794+0000","last_undegraded":"2023-06-10T21:44:46.324794+0000","last_fullsized":"2023-06-10T21:44:46.324794+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":471,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537099+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:44:46.537099+0000","last_peered":"2023-06-10T21:44:46.537099+0000","last_clean":"2023-06-10T21:44:46.537099+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:44:46.537099+0000","last_undegraded":"2023-06-10T21:44:46.537099+0000","last_fullsized":"2023-06-10T21:44:46.537099+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":359,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319379+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:44:46.319379+0000","last_peered":"2023-06-10T21:44:46.319379+0000","last_clean":"2023-06-10T21:44:46.319379+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:44:46.319379+0000","last_undegraded":"2023-06-10T21:44:46.319379+0000","last_fullsized":"2023-06-10T21:44:46.319379+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":282,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.536537+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:44:46.536537+0000","last_peered":"2023-06-10T21:44:46.536537+0000","last_clean":"2023-06-10T21:44:46.536537+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:44:46.536537+0000","last_undegraded":"2023-06-10T21:44:46.536537+0000","last_fullsized":"2023-06-10T21:44:46.536537+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324911+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:44:46.324911+0000","last_peered":"2023-06-10T21:44:46.324911+0000","last_clean":"2023-06-10T21:44:46.324911+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:44:46.324911+0000","last_undegraded":"2023-06-10T21:44:46.324911+0000","last_fullsized":"2023-06-10T21:44:46.324911+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":424,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537200+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:44:46.537200+0000","last_peered":"2023-06-10T21:44:46.537200+0000","last_clean":"2023-06-10T21:44:46.537200+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:44:46.537200+0000","last_undegraded":"2023-06-10T21:44:46.537200+0000","last_fullsized":"2023-06-10T21:44:46.537200+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":65,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319835+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:44:46.319835+0000","last_peered":"2023-06-10T21:44:46.319835+0000","last_clean":"2023-06-10T21:44:46.319835+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:44:46.319835+0000","last_undegraded":"2023-06-10T21:44:46.319835+0000","last_fullsized":"2023-06-10T21:44:46.319835+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":436,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324513+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:44:46.324513+0000","last_peered":"2023-06-10T21:44:46.324513+0000","last_clean":"2023-06-10T21:44:46.324513+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:44:46.324513+0000","last_undegraded":"2023-06-10T21:44:46.324513+0000","last_fullsized":"2023-06-10T21:44:46.324513+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319564+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:44:46.319564+0000","last_peered":"2023-06-10T21:44:46.319564+0000","last_clean":"2023-06-10T21:44:46.319564+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:44:46.319564+0000","last_undegraded":"2023-06-10T21:44:46.319564+0000","last_fullsized":"2023-06-10T21:44:46.319564+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"411'11768","reported_seq":11816,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.140266+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:44:51.140266+0000","last_peered":"2023-06-10T21:44:51.140266+0000","last_clean":"2023-06-10T21:44:51.140266+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:44:51.140266+0000","last_undegraded":"2023-06-10T21:44:51.140266+0000","last_fullsized":"2023-06-10T21:44:51.140266+0000","mapping_epoch":373,"log_start":"392'6700","ondisk_log_start":"392'6700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127401984,"num_objects":1944,"num_object_clones":0,"num_object_copies":5832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11768,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":204,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540296+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:44:46.540296+0000","last_peered":"2023-06-10T21:44:46.540296+0000","last_clean":"2023-06-10T21:44:46.540296+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:44:46.540296+0000","last_undegraded":"2023-06-10T21:44:46.540296+0000","last_fullsized":"2023-06-10T21:44:46.540296+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324366+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:44:46.324366+0000","last_peered":"2023-06-10T21:44:46.324366+0000","last_clean":"2023-06-10T21:44:46.324366+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:44:46.324366+0000","last_undegraded":"2023-06-10T21:44:46.324366+0000","last_fullsized":"2023-06-10T21:44:46.324366+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"412'12063","reported_seq":12128,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:54.187116+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:44:54.187116+0000","last_peered":"2023-06-10T21:44:54.187116+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:44:54.187116+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"393'7700","ondisk_log_start":"393'7700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4363,"log_dups_size":0,"ondisk_log_size":4363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116457472,"num_objects":1777,"num_object_clones":0,"num_object_copies":5331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12063,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-113}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":204,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540537+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:44:46.540537+0000","last_peered":"2023-06-10T21:44:46.540537+0000","last_clean":"2023-06-10T21:44:46.540537+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:44:46.540537+0000","last_undegraded":"2023-06-10T21:44:46.540537+0000","last_fullsized":"2023-06-10T21:44:46.540537+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":66,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.326055+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:44:51.326055+0000","last_peered":"2023-06-10T21:44:51.326055+0000","last_clean":"2023-06-10T21:44:51.326055+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:44:51.326055+0000","last_undegraded":"2023-06-10T21:44:51.326055+0000","last_fullsized":"2023-06-10T21:44:51.326055+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"412'12134","reported_seq":12213,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:55.940422+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:44:55.940422+0000","last_peered":"2023-06-10T21:44:55.940422+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:44:55.940422+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"394'7800","ondisk_log_start":"394'7800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4334,"log_dups_size":0,"ondisk_log_size":4334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12134,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-224}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":491,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537569+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:44:46.537569+0000","last_peered":"2023-06-10T21:44:46.537569+0000","last_clean":"2023-06-10T21:44:46.537569+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:44:46.537569+0000","last_undegraded":"2023-06-10T21:44:46.537569+0000","last_fullsized":"2023-06-10T21:44:46.537569+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":175,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.544186+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:44:46.544186+0000","last_peered":"2023-06-10T21:44:46.544186+0000","last_clean":"2023-06-10T21:44:46.544186+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:44:46.544186+0000","last_undegraded":"2023-06-10T21:44:46.544186+0000","last_fullsized":"2023-06-10T21:44:46.544186+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"412'12064","reported_seq":15941,"reported_epoch":412,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:44:54.699543+0000","last_change":"2023-06-10T21:44:46.591362+0000","last_active":"2023-06-10T21:44:54.699543+0000","last_peered":"2023-06-10T21:44:54.699543+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:44:46.544938+0000","last_became_peered":"2023-06-10T21:44:46.544938+0000","last_unstale":"2023-06-10T21:44:54.699543+0000","last_undegraded":"2023-06-10T21:44:54.699543+0000","last_fullsized":"2023-06-10T21:44:54.699543+0000","mapping_epoch":410,"log_start":"393'7584","ondisk_log_start":"393'7584","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4480,"log_dups_size":0,"ondisk_log_size":4480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114032640,"num_objects":1740,"num_object_clones":0,"num_object_copies":5220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12064,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1905,"num_bytes_recovered":124780544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":1740}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324964+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:44:46.324964+0000","last_peered":"2023-06-10T21:44:46.324964+0000","last_clean":"2023-06-10T21:44:46.324964+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:44:46.324964+0000","last_undegraded":"2023-06-10T21:44:46.324964+0000","last_fullsized":"2023-06-10T21:44:46.324964+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319211+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:44:46.319211+0000","last_peered":"2023-06-10T21:44:46.319211+0000","last_clean":"2023-06-10T21:44:46.319211+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:44:46.319211+0000","last_undegraded":"2023-06-10T21:44:46.319211+0000","last_fullsized":"2023-06-10T21:44:46.319211+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"412'11990","reported_seq":15849,"reported_epoch":412,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:44:54.697152+0000","last_change":"2023-06-10T21:44:23.399033+0000","last_active":"2023-06-10T21:44:54.697152+0000","last_peered":"2023-06-10T21:44:54.697152+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:44:23.337898+0000","last_became_peered":"2023-06-10T21:44:23.337898+0000","last_unstale":"2023-06-10T21:44:54.697152+0000","last_undegraded":"2023-06-10T21:44:54.697152+0000","last_fullsized":"2023-06-10T21:44:54.697152+0000","mapping_epoch":406,"log_start":"393'7440","ondisk_log_start":"393'7440","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113901568,"num_objects":1738,"num_object_clones":0,"num_object_copies":5214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11990,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1738}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":69,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319182+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:44:46.319182+0000","last_peered":"2023-06-10T21:44:46.319182+0000","last_clean":"2023-06-10T21:44:46.319182+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:44:46.319182+0000","last_undegraded":"2023-06-10T21:44:46.319182+0000","last_fullsized":"2023-06-10T21:44:46.319182+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319886+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:44:46.319886+0000","last_peered":"2023-06-10T21:44:46.319886+0000","last_clean":"2023-06-10T21:44:46.319886+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:44:46.319886+0000","last_undegraded":"2023-06-10T21:44:46.319886+0000","last_fullsized":"2023-06-10T21:44:46.319886+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"411'11788","reported_seq":15692,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.139409+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:44:51.139409+0000","last_peered":"2023-06-10T21:44:51.139409+0000","last_clean":"2023-06-10T21:44:51.139409+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:44:51.139409+0000","last_undegraded":"2023-06-10T21:44:51.139409+0000","last_fullsized":"2023-06-10T21:44:51.139409+0000","mapping_epoch":395,"log_start":"392'6700","ondisk_log_start":"392'6700","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126091264,"num_objects":1924,"num_object_clones":0,"num_object_copies":5772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11788,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":233,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319435+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:44:46.319435+0000","last_peered":"2023-06-10T21:44:46.319435+0000","last_clean":"2023-06-10T21:44:46.319435+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:44:46.319435+0000","last_undegraded":"2023-06-10T21:44:46.319435+0000","last_fullsized":"2023-06-10T21:44:46.319435+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":449,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.325675+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:44:45.325675+0000","last_peered":"2023-06-10T21:44:45.325675+0000","last_clean":"2023-06-10T21:44:45.325675+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:44:45.325675+0000","last_undegraded":"2023-06-10T21:44:45.325675+0000","last_fullsized":"2023-06-10T21:44:45.325675+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"412'11821","reported_seq":11883,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:54.215729+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:44:54.215729+0000","last_peered":"2023-06-10T21:44:54.215729+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:44:54.215729+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"393'7500","ondisk_log_start":"393'7500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4321,"log_dups_size":0,"ondisk_log_size":4321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114753536,"num_objects":1751,"num_object_clones":0,"num_object_copies":5253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11821,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-102}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":426,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.326931+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:44:45.326931+0000","last_peered":"2023-06-10T21:44:45.326931+0000","last_clean":"2023-06-10T21:44:45.326931+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:44:45.326931+0000","last_undegraded":"2023-06-10T21:44:45.326931+0000","last_fullsized":"2023-06-10T21:44:45.326931+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319752+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:44:46.319752+0000","last_peered":"2023-06-10T21:44:46.319752+0000","last_clean":"2023-06-10T21:44:46.319752+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:44:46.319752+0000","last_undegraded":"2023-06-10T21:44:46.319752+0000","last_fullsized":"2023-06-10T21:44:46.319752+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"411'11723","reported_seq":11779,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:51.110898+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:44:51.110898+0000","last_peered":"2023-06-10T21:44:51.110898+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:44:51.110898+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"392'6674","ondisk_log_start":"392'6674","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127860736,"num_objects":1951,"num_object_clones":0,"num_object_copies":5853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11723,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":16}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:50.960948+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:44:50.960948+0000","last_peered":"2023-06-10T21:44:50.960948+0000","last_clean":"2023-06-10T21:44:50.960948+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:44:50.960948+0000","last_undegraded":"2023-06-10T21:44:50.960948+0000","last_fullsized":"2023-06-10T21:44:50.960948+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":360,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319800+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:44:46.319800+0000","last_peered":"2023-06-10T21:44:46.319800+0000","last_clean":"2023-06-10T21:44:46.319800+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:44:46.319800+0000","last_undegraded":"2023-06-10T21:44:46.319800+0000","last_fullsized":"2023-06-10T21:44:46.319800+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"411'11597","reported_seq":11651,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:51.128018+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:44:51.128018+0000","last_peered":"2023-06-10T21:44:51.128018+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:44:51.128018+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"389'6519","ondisk_log_start":"389'6519","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5078,"log_dups_size":0,"ondisk_log_size":5078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11597,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":70}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":195,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537355+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:44:46.537355+0000","last_peered":"2023-06-10T21:44:46.537355+0000","last_clean":"2023-06-10T21:44:46.537355+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:44:46.537355+0000","last_undegraded":"2023-06-10T21:44:46.537355+0000","last_fullsized":"2023-06-10T21:44:46.537355+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":76,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324692+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:44:46.324692+0000","last_peered":"2023-06-10T21:44:46.324692+0000","last_clean":"2023-06-10T21:44:46.324692+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:44:46.324692+0000","last_undegraded":"2023-06-10T21:44:46.324692+0000","last_fullsized":"2023-06-10T21:44:46.324692+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324700+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:44:46.324700+0000","last_peered":"2023-06-10T21:44:46.324700+0000","last_clean":"2023-06-10T21:44:46.324700+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:44:46.324700+0000","last_undegraded":"2023-06-10T21:44:46.324700+0000","last_fullsized":"2023-06-10T21:44:46.324700+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"412'11888","reported_seq":11955,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:54.199605+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:44:54.199605+0000","last_peered":"2023-06-10T21:44:54.199605+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:44:54.199605+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"393'7600","ondisk_log_start":"393'7600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4288,"log_dups_size":0,"ondisk_log_size":4288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118947864,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11888,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-101}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":95,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324607+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:44:46.324607+0000","last_peered":"2023-06-10T21:44:46.324607+0000","last_clean":"2023-06-10T21:44:46.324607+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:44:46.324607+0000","last_undegraded":"2023-06-10T21:44:46.324607+0000","last_fullsized":"2023-06-10T21:44:46.324607+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":373,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324462+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:44:46.324462+0000","last_peered":"2023-06-10T21:44:46.324462+0000","last_clean":"2023-06-10T21:44:46.324462+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:44:46.324462+0000","last_undegraded":"2023-06-10T21:44:46.324462+0000","last_fullsized":"2023-06-10T21:44:46.324462+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":276,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.329848+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:44:45.329848+0000","last_peered":"2023-06-10T21:44:45.329848+0000","last_clean":"2023-06-10T21:44:45.329848+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:44:45.329848+0000","last_undegraded":"2023-06-10T21:44:45.329848+0000","last_fullsized":"2023-06-10T21:44:45.329848+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"412'11910","reported_seq":11977,"reported_epoch":412,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:54.243582+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:44:54.243582+0000","last_peered":"2023-06-10T21:44:54.243582+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:44:54.243582+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"393'7600","ondisk_log_start":"393'7600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4310,"log_dups_size":0,"ondisk_log_size":4310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115867648,"num_objects":1768,"num_object_clones":0,"num_object_copies":5304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11910,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-105}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":296,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.326847+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:44:45.326847+0000","last_peered":"2023-06-10T21:44:45.326847+0000","last_clean":"2023-06-10T21:44:45.326847+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:44:45.326847+0000","last_undegraded":"2023-06-10T21:44:45.326847+0000","last_fullsized":"2023-06-10T21:44:45.326847+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":300,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.324789+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:44:45.324789+0000","last_peered":"2023-06-10T21:44:45.324789+0000","last_clean":"2023-06-10T21:44:45.324789+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:44:45.324789+0000","last_undegraded":"2023-06-10T21:44:45.324789+0000","last_fullsized":"2023-06-10T21:44:45.324789+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.325185+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:44:46.325185+0000","last_peered":"2023-06-10T21:44:46.325185+0000","last_clean":"2023-06-10T21:44:46.325185+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:44:46.325185+0000","last_undegraded":"2023-06-10T21:44:46.325185+0000","last_fullsized":"2023-06-10T21:44:46.325185+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"411'11856","reported_seq":11912,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:51.120756+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:44:51.120756+0000","last_peered":"2023-06-10T21:44:51.120756+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:44:51.120756+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"392'6800","ondisk_log_start":"392'6800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5056,"log_dups_size":0,"ondisk_log_size":5056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132775936,"num_objects":2026,"num_object_clones":0,"num_object_copies":6078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11856,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":62}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":86,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324701+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:44:46.324701+0000","last_peered":"2023-06-10T21:44:46.324701+0000","last_clean":"2023-06-10T21:44:46.324701+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:44:46.324701+0000","last_undegraded":"2023-06-10T21:44:46.324701+0000","last_fullsized":"2023-06-10T21:44:46.324701+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":612,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.536901+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:44:46.536901+0000","last_peered":"2023-06-10T21:44:46.536901+0000","last_clean":"2023-06-10T21:44:46.536901+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:44:46.536901+0000","last_undegraded":"2023-06-10T21:44:46.536901+0000","last_fullsized":"2023-06-10T21:44:46.536901+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":81,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319246+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:44:46.319246+0000","last_peered":"2023-06-10T21:44:46.319246+0000","last_clean":"2023-06-10T21:44:46.319246+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:44:46.319246+0000","last_undegraded":"2023-06-10T21:44:46.319246+0000","last_fullsized":"2023-06-10T21:44:46.319246+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":295,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540384+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:44:46.540384+0000","last_peered":"2023-06-10T21:44:46.540384+0000","last_clean":"2023-06-10T21:44:46.540384+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:44:46.540384+0000","last_undegraded":"2023-06-10T21:44:46.540384+0000","last_fullsized":"2023-06-10T21:44:46.540384+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"412'11806","reported_seq":15716,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:54.174415+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:44:54.174415+0000","last_peered":"2023-06-10T21:44:54.174415+0000","last_clean":"2023-06-10T21:44:54.174415+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:44:54.174415+0000","last_undegraded":"2023-06-10T21:44:54.174415+0000","last_fullsized":"2023-06-10T21:44:54.174415+0000","mapping_epoch":401,"log_start":"393'7500","ondisk_log_start":"393'7500","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4306,"log_dups_size":0,"ondisk_log_size":4306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117964800,"num_objects":1800,"num_object_clones":0,"num_object_copies":5400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11806,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":464,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.536970+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:44:46.536970+0000","last_peered":"2023-06-10T21:44:46.536970+0000","last_clean":"2023-06-10T21:44:46.536970+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:44:46.536970+0000","last_undegraded":"2023-06-10T21:44:46.536970+0000","last_fullsized":"2023-06-10T21:44:46.536970+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":81,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.318836+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:44:46.318836+0000","last_peered":"2023-06-10T21:44:46.318836+0000","last_clean":"2023-06-10T21:44:46.318836+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:44:46.318836+0000","last_undegraded":"2023-06-10T21:44:46.318836+0000","last_fullsized":"2023-06-10T21:44:46.318836+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":162,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.327906+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:44:45.327906+0000","last_peered":"2023-06-10T21:44:45.327906+0000","last_clean":"2023-06-10T21:44:45.327906+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:44:45.327906+0000","last_undegraded":"2023-06-10T21:44:45.327906+0000","last_fullsized":"2023-06-10T21:44:45.327906+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"411'11806","reported_seq":11854,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.125188+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:44:51.125188+0000","last_peered":"2023-06-10T21:44:51.125188+0000","last_clean":"2023-06-10T21:44:51.125188+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:44:51.125188+0000","last_undegraded":"2023-06-10T21:44:51.125188+0000","last_fullsized":"2023-06-10T21:44:51.125188+0000","mapping_epoch":373,"log_start":"392'6800","ondisk_log_start":"392'6800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126353408,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11806,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":443,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537594+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:44:46.537594+0000","last_peered":"2023-06-10T21:44:46.537594+0000","last_clean":"2023-06-10T21:44:46.537594+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:44:46.537594+0000","last_undegraded":"2023-06-10T21:44:46.537594+0000","last_fullsized":"2023-06-10T21:44:46.537594+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":291,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.329921+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:44:45.329921+0000","last_peered":"2023-06-10T21:44:45.329921+0000","last_clean":"2023-06-10T21:44:45.329921+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:44:45.329921+0000","last_undegraded":"2023-06-10T21:44:45.329921+0000","last_fullsized":"2023-06-10T21:44:45.329921+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":216,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324550+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:44:46.324550+0000","last_peered":"2023-06-10T21:44:46.324550+0000","last_clean":"2023-06-10T21:44:46.324550+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:44:46.324550+0000","last_undegraded":"2023-06-10T21:44:46.324550+0000","last_fullsized":"2023-06-10T21:44:46.324550+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"411'12080","reported_seq":12964,"reported_epoch":411,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:51.146277+0000","last_change":"2023-06-10T21:44:45.105645+0000","last_active":"2023-06-10T21:44:51.146277+0000","last_peered":"2023-06-10T21:44:51.146277+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:44:51.146277+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"392'7000","ondisk_log_start":"392'7000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131989504,"num_objects":2014,"num_object_clones":0,"num_object_copies":6042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1563,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12080,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":413,"num_bytes_recovered":27066368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":451}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":422,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319656+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:44:46.319656+0000","last_peered":"2023-06-10T21:44:46.319656+0000","last_clean":"2023-06-10T21:44:46.319656+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:44:46.319656+0000","last_undegraded":"2023-06-10T21:44:46.319656+0000","last_fullsized":"2023-06-10T21:44:46.319656+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":233,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319979+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:44:46.319979+0000","last_peered":"2023-06-10T21:44:46.319979+0000","last_clean":"2023-06-10T21:44:46.319979+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:44:46.319979+0000","last_undegraded":"2023-06-10T21:44:46.319979+0000","last_fullsized":"2023-06-10T21:44:46.319979+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319156+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:44:46.319156+0000","last_peered":"2023-06-10T21:44:46.319156+0000","last_clean":"2023-06-10T21:44:46.319156+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:44:46.319156+0000","last_undegraded":"2023-06-10T21:44:46.319156+0000","last_fullsized":"2023-06-10T21:44:46.319156+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"411'11848","reported_seq":11911,"reported_epoch":411,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:44:51.150293+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:44:51.150293+0000","last_peered":"2023-06-10T21:44:51.150293+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:44:51.150293+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"392'6800","ondisk_log_start":"392'6800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5048,"log_dups_size":0,"ondisk_log_size":5048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129630208,"num_objects":1978,"num_object_clones":0,"num_object_copies":5934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11848,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":57}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":160,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540381+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:44:46.540381+0000","last_peered":"2023-06-10T21:44:46.540381+0000","last_clean":"2023-06-10T21:44:46.540381+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:44:46.540381+0000","last_undegraded":"2023-06-10T21:44:46.540381+0000","last_fullsized":"2023-06-10T21:44:46.540381+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":274,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.540267+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:44:46.540267+0000","last_peered":"2023-06-10T21:44:46.540267+0000","last_clean":"2023-06-10T21:44:46.540267+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:44:46.540267+0000","last_undegraded":"2023-06-10T21:44:46.540267+0000","last_fullsized":"2023-06-10T21:44:46.540267+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":66,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324436+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:44:46.324436+0000","last_peered":"2023-06-10T21:44:46.324436+0000","last_clean":"2023-06-10T21:44:46.324436+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:44:46.324436+0000","last_undegraded":"2023-06-10T21:44:46.324436+0000","last_fullsized":"2023-06-10T21:44:46.324436+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":111,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.537185+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:44:46.537185+0000","last_peered":"2023-06-10T21:44:46.537185+0000","last_clean":"2023-06-10T21:44:46.537185+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:44:46.537185+0000","last_undegraded":"2023-06-10T21:44:46.537185+0000","last_fullsized":"2023-06-10T21:44:46.537185+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":307,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-06-10T21:44:45.329988+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:44:45.329988+0000","last_peered":"2023-06-10T21:44:45.329988+0000","last_clean":"2023-06-10T21:44:45.329988+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:44:45.329988+0000","last_undegraded":"2023-06-10T21:44:45.329988+0000","last_fullsized":"2023-06-10T21:44:45.329988+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":74,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324551+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:44:46.324551+0000","last_peered":"2023-06-10T21:44:46.324551+0000","last_clean":"2023-06-10T21:44:46.324551+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:44:46.324551+0000","last_undegraded":"2023-06-10T21:44:46.324551+0000","last_fullsized":"2023-06-10T21:44:46.324551+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":316,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319307+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:44:46.319307+0000","last_peered":"2023-06-10T21:44:46.319307+0000","last_clean":"2023-06-10T21:44:46.319307+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:44:46.319307+0000","last_undegraded":"2023-06-10T21:44:46.319307+0000","last_fullsized":"2023-06-10T21:44:46.319307+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":97,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.324602+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:44:46.324602+0000","last_peered":"2023-06-10T21:44:46.324602+0000","last_clean":"2023-06-10T21:44:46.324602+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:44:46.324602+0000","last_undegraded":"2023-06-10T21:44:46.324602+0000","last_fullsized":"2023-06-10T21:44:46.324602+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":387,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319515+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:44:46.319515+0000","last_peered":"2023-06-10T21:44:46.319515+0000","last_clean":"2023-06-10T21:44:46.319515+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:44:46.319515+0000","last_undegraded":"2023-06-10T21:44:46.319515+0000","last_fullsized":"2023-06-10T21:44:46.319515+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":66,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319908+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:44:46.319908+0000","last_peered":"2023-06-10T21:44:46.319908+0000","last_clean":"2023-06-10T21:44:46.319908+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:44:46.319908+0000","last_undegraded":"2023-06-10T21:44:46.319908+0000","last_fullsized":"2023-06-10T21:44:46.319908+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.318945+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:44:46.318945+0000","last_peered":"2023-06-10T21:44:46.318945+0000","last_clean":"2023-06-10T21:44:46.318945+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:44:46.318945+0000","last_undegraded":"2023-06-10T21:44:46.318945+0000","last_fullsized":"2023-06-10T21:44:46.318945+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":333,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.325289+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:44:46.325289+0000","last_peered":"2023-06-10T21:44:46.325289+0000","last_clean":"2023-06-10T21:44:46.325289+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:44:46.325289+0000","last_undegraded":"2023-06-10T21:44:46.325289+0000","last_fullsized":"2023-06-10T21:44:46.325289+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":80,"reported_epoch":411,"state":"active+clean","last_fresh":"2023-06-10T21:44:46.319367+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:44:46.319367+0000","last_peered":"2023-06-10T21:44:46.319367+0000","last_clean":"2023-06-10T21:44:46.319367+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:44:46.319367+0000","last_undegraded":"2023-06-10T21:44:46.319367+0000","last_fullsized":"2023-06-10T21:44:46.319367+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1956708376,"num_objects":29858,"num_object_clones":0,"num_object_copies":89574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18773,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29858,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":190142,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8058,"num_bytes_recovered":527958016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4671746048,"data_stored":4671733832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75425,"ondisk_log_size":75425,"up":48,"acting":38,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430774,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2247760,"kb_used_data":1287624,"kb_used_omap":0,"kb_used_meta":960128,"kb_avail":91497392,"statfs":{"total":95995035648,"available":93693329408,"internally_reserved":0,"allocated":1318526976,"data_stored":1317818371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":983171072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.3089999999999999,"15min":3.1179999999999999},"min":{"1min":0.28599999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":4.2699999999999996,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.307},{"interface":"front","average":{"1min":1.244,"5min":1.552,"15min":3.371},"min":{"1min":0.28199999999999997,"5min":0.25900000000000001,"15min":0.223},"max":{"1min":3.0960000000000001,"5min":22.462,"15min":185.58000000000001},"last":1.3720000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.8100000000000001,"15min":2.6560000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.1500000000000004,"5min":38.078000000000003,"15min":87.417000000000002},"last":3.2530000000000001},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.2509999999999999,"15min":2.9590000000000001},"min":{"1min":0.30299999999999999,"5min":0.28599999999999998,"15min":0.27300000000000002},"max":{"1min":10.484,"5min":38.341000000000001,"15min":87.438000000000002},"last":1.907}]},{"osd":2,"last update":"Sat Jun 10 21:44:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.1299999999999999,"15min":4.9729999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":36.627000000000002,"5min":75.219999999999999,"15min":172.33600000000001},"last":1.3500000000000001},{"interface":"front","average":{"1min":4.4880000000000004,"5min":3.1379999999999999,"15min":4.9610000000000003},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":36.533999999999999,"5min":75.108999999999995,"15min":172.185},"last":1.9650000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":7.7990000000000004,"5min":4.0899999999999999,"15min":4.7960000000000003},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":71.852000000000004,"5min":89.504999999999995,"15min":147.00700000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":7.2359999999999998,"5min":3.7599999999999998,"15min":4.5270000000000001},"min":{"1min":0.439,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":72.813999999999993,"5min":90.215999999999994,"15min":146.93899999999999},"last":3.3740000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":0.88900000000000001,"15min":0.59599999999999997},"min":{"1min":0.35499999999999998,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":13.429,"5min":13.429,"15min":13.429},"last":1.9339999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1120000000000001,"15min":0.71699999999999997},"min":{"1min":0.34200000000000003,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":4.6760000000000002,"5min":22.77,"15min":22.77},"last":1.284}]},{"osd":5,"last update":"Sat Jun 10 21:44:04 2023","interfaces":[{"interface":"back","average":{"1min":12.178000000000001,"5min":12.178000000000001,"15min":12.178000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":131.619,"5min":131.619,"15min":131.619},"last":1.8700000000000001},{"interface":"front","average":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":134.928,"5min":134.928,"15min":134.928},"last":1.2569999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":0.89700000000000002,"15min":0.67000000000000004},"min":{"1min":0.41199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8280000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.911,"5min":0.99099999999999999,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":4.3739999999999997,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":0.60199999999999998}]}]},{"osd":6,"up_from":0,"seq":180388627002,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566445,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020421,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075832,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2525632,"kb_used_data":1514360,"kb_used_omap":7,"kb_used_meta":1011192,"kb_avail":91219520,"statfs":{"total":95995035648,"available":93408788480,"internally_reserved":0,"allocated":1550704640,"data_stored":1549367450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1035460689},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":2.903,"15min":4.4560000000000004},"min":{"1min":0.27500000000000002,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":8.5899999999999999,"5min":83.733000000000004,"15min":171.33699999999999},"last":0.247},{"interface":"front","average":{"1min":1.673,"5min":2.4100000000000001,"15min":4.3440000000000003},"min":{"1min":0.255,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.9400000000000004,"5min":82.322999999999993,"15min":171.56200000000001},"last":0.35099999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.653,"15min":5.3120000000000003},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.2370000000000001,"5min":85.709999999999994,"15min":171.286},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.542,"5min":3.548,"15min":3.891},"min":{"1min":0.27800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1379999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:44:25 2023","interfaces":[{"interface":"back","average":{"1min":11.705,"5min":8.6259999999999994,"15min":6.6699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":100.786,"5min":145.07599999999999,"15min":171.78299999999999},"last":5.2679999999999998},{"interface":"front","average":{"1min":11.17,"5min":7.9969999999999999,"15min":6.6219999999999999},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":100.73999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":5.9299999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.266,"15min":2.3130000000000002},"min":{"1min":0.33500000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.9629999999999992,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.903,"5min":1.7669999999999999,"15min":3.2370000000000001},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.2539999999999996,"5min":14.858000000000001,"15min":171.18100000000001},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:44:02 2023","interfaces":[{"interface":"back","average":{"1min":6.9820000000000002,"5min":6.9820000000000002,"15min":6.9820000000000002},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":75.896000000000001,"5min":75.896000000000001,"15min":75.896000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":6.3419999999999996,"5min":6.3419999999999996,"15min":6.3419999999999996},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":76.337000000000003,"5min":76.337000000000003,"15min":76.337000000000003},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.476,"15min":2.4300000000000002},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.0139999999999993,"5min":9.0139999999999993,"15min":171.441},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.948,"5min":1.931,"15min":3.3029999999999999},"min":{"1min":0.51900000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.9199999999999999,"5min":14.893000000000001,"15min":171.24600000000001},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.9430000000000001,"15min":4.0979999999999999},"min":{"1min":0.503,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":219.131},"last":3.0699999999999998},{"interface":"front","average":{"1min":2.9279999999999999,"5min":1.585,"15min":4.0250000000000004},"min":{"1min":0.48099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":12.15,"5min":12.15,"15min":219.19},"last":1.4359999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824193,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1919120,"kb_used_data":1207496,"kb_used_omap":7,"kb_used_meta":711608,"kb_avail":91826032,"statfs":{"total":95995035648,"available":94029856768,"internally_reserved":0,"allocated":1236475904,"data_stored":1235010227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":728686611},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6319999999999999,"15min":3.2730000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.5880000000000001,"5min":30.744,"15min":105.09},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.375,"5min":2.0739999999999998,"15min":3.3740000000000001},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.20799999999999999},"max":{"1min":4.4450000000000003,"5min":30.805,"15min":99.852999999999994},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.8280000000000001,"15min":3.3660000000000001},"min":{"1min":0.315,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":4.5300000000000002,"5min":13.249000000000001,"15min":94.766000000000005},"last":0.249},{"interface":"front","average":{"1min":1.375,"5min":1.589,"15min":3.2789999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":5.4109999999999996,"5min":11.846,"15min":94.691000000000003},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.6499999999999999,"15min":4.319},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.9649999999999999,"5min":18.452000000000002,"15min":231.07900000000001},"last":0.439},{"interface":"front","average":{"1min":2.1000000000000001,"5min":3.0259999999999998,"15min":5.5599999999999996},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":9.3930000000000007,"5min":18.733000000000001,"15min":231.58600000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.5900000000000001,"15min":1.486},"min":{"1min":0.45600000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.8470000000000004,"5min":9.5150000000000006,"15min":13.417},"last":1.47},{"interface":"front","average":{"1min":1.843,"5min":1.8300000000000001,"15min":2.2599999999999998},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":4.5049999999999999,"5min":10.722,"15min":19.728000000000002},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:44:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.532,"5min":6.532,"15min":6.532},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.7040000000000002,"5min":2.7040000000000002,"15min":2.7040000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":1.262}]},{"osd":6,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.903,"15min":2.323},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8519999999999999,"5min":10.691000000000001,"15min":19.805},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.484,"15min":1.5529999999999999},"min":{"1min":0.48199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.6299999999999999,"5min":9.9719999999999995,"15min":17.327000000000002},"last":3.6160000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.1179999999999999,"15min":2.601},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":5.4109999999999996,"5min":10.648999999999999,"15min":79.984999999999999},"last":2.597},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.603,"15min":1.847},"min":{"1min":0.51300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":5.9320000000000004,"5min":9.9380000000000006,"15min":80.272999999999996},"last":0.58199999999999996}]}]},{"osd":4,"up_from":0,"seq":128849019458,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245464,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":828104,"kb_used_data":604928,"kb_used_omap":11,"kb_used_meta":223028,"kb_avail":92917048,"statfs":{"total":95995035648,"available":95147057152,"internally_reserved":0,"allocated":619446272,"data_stored":618745464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":228381174},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":2.0800000000000001,"15min":2.0800000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.5199999999999996,"5min":7.5199999999999996,"15min":7.5199999999999996},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.266,"5min":4.266,"15min":4.266},"last":0.96199999999999997}]},{"osd":1,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.54,"5min":7.54,"15min":7.54},"last":0.379},{"interface":"front","average":{"1min":2.282,"5min":2.282,"15min":2.282},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":9.7989999999999995,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":1}]},{"osd":2,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8620000000000001,"5min":1.8620000000000001,"15min":1.8620000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":7.9809999999999999},"last":1.0980000000000001},{"interface":"front","average":{"1min":2.306,"5min":2.306,"15min":2.306},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":12.641,"5min":12.641,"15min":12.641},"last":2.2690000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":8.2330000000000005,"5min":8.2330000000000005,"15min":8.2330000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":133.07499999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":7.8949999999999996,"5min":7.8949999999999996,"15min":7.8949999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":133.03800000000001,"5min":133.03800000000001,"15min":133.03800000000001},"last":1.0720000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.173,"5min":5.173,"15min":5.173},"last":4.4329999999999998},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.3079999999999998,"15min":2.3079999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.476,"5min":7.476,"15min":7.476},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.8879999999999999,"15min":1.8879999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":1.048},{"interface":"front","average":{"1min":1.345,"5min":1.345,"15min":1.345},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":5.4509999999999996,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":2.3069999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:44:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.5670000000000002,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":5.0110000000000001},{"interface":"front","average":{"1min":2.891,"5min":2.891,"15min":2.891},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.4160000000000004,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":3.8959999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1539571712,"data_stored":1539571712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1229459456,"data_stored":1229455384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":615518208,"data_stored":615514136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1287196672,"data_stored":1287192600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:45:01.404 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:45:01.542 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:01.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:01.650 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:01 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18415/78372 objects degraded (23.497%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:01.650 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:01 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18415/78372 objects degraded (23.497%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:01.651 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:01.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:45:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:01 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18415/78372 objects degraded (23.497%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:01.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:02.119 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:02.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:02.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:02.227 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:02.297 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:02.326 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:45:02.390 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:02.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:02.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:02.501 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:02.517 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:02.533 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:02.543 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:02.581 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:02.617 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:02 smithi152 ceph-mon[156819]: pgmap v1522: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 18108/71532 objects degraded (25.315%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:45:02.625 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:02.638 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:02.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:02.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:02.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:02 smithi064 ceph-mon[180297]: pgmap v1522: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 18108/71532 objects degraded (25.315%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:45:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:02 smithi064 ceph-mon[186706]: pgmap v1522: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 18108/71532 objects degraded (25.315%); 3.8 MiB/s, 61 objects/s recovering 2023-06-10T21:45:02.666 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:02.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:02.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:02.754 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:02.797 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:02.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.807 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.808 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.808 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.808 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:45:02.808 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:03.114 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:03.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:02] "GET /metrics HTTP/1.1" 200 224091 "" "Prometheus/2.33.4" 2023-06-10T21:45:03.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:03.283 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:03.365 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:45:03.372 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:03.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:03.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2116391887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:03.537 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:03.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2116391887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:03.538 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:03.539 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:03.542 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2116391887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:04.551 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:04.556 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:04.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[186706]: failed to encode map e413 with expected crc 2023-06-10T21:45:04.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:04.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[186706]: osdmap e413: 8 total, 8 up, 4 in 2023-06-10T21:45:04.557 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[186706]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 18108/71532 objects degraded (25.315%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:04.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:04.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:04.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[180297]: failed to encode map e413 with expected crc 2023-06-10T21:45:04.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:04.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[180297]: osdmap e413: 8 total, 8 up, 4 in 2023-06-10T21:45:04.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:04 smithi064 ceph-mon[180297]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 18108/71532 objects degraded (25.315%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:04.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:04 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:04 smithi152 ceph-mon[156819]: failed to encode map e413 with expected crc 2023-06-10T21:45:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:04 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3932760793' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:04 smithi152 ceph-mon[156819]: osdmap e413: 8 total, 8 up, 4 in 2023-06-10T21:45:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:04 smithi152 ceph-mon[156819]: pgmap v1524: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.5 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 809 op/s; 18108/71532 objects degraded (25.315%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:04.709 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:05.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:05 smithi152 ceph-mon[156819]: failed to encode map e413 with expected crc 2023-06-10T21:45:05.635 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:05.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:05 smithi064 ceph-mon[180297]: failed to encode map e413 with expected crc 2023-06-10T21:45:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:05.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:05.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:05 smithi064 ceph-mon[186706]: failed to encode map e413 with expected crc 2023-06-10T21:45:05.739 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:05.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:05.816 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:45:05.816 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:45:05.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:200889 13.2 13:42b4a743:::benchmark_data_smithi064_501823_object90887:head [delete] snapc 0=[] ondisk+write+known_if_redirected e413)", 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.807440+0000", 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0028766680000000002, 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029123170000000002, 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:45:05.817 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200889 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807440+0000", 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807440+0000", 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:45:05.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807438+0000", 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1750000000000005e-06 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807447+0000", 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7561999999999999e-05 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807464+0000", 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5490000000000001e-06 2023-06-10T21:45:05.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807472+0000", 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013086400000000001 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807603+0000", 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015107200000000001 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807754+0000", 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000150832 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807904+0000", 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00046444200000000002 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:200890 13.5 13.e4fa0e65 (undecoded) ondisk+write+known_if_redirected e413)", 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.807489+0000", 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0028277049999999998, 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029565250000000002, 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:45:05.825 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200890 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.826 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807489+0000", 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807489+0000", 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807488+0000", 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9310000000000001e-06 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807492+0000", 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0638999999999999e-05 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807503+0000", 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8640000000000004e-06 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807508+0000", 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8844000000000001e-05 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:200892 13.5 13.dd7cdbc5 (undecoded) ondisk+write+known_if_redirected e413)", 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.807521+0000", 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.002795769, 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029883679999999999, 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:45:05.831 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200892 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807521+0000", 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807521+0000", 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807520+0000", 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.585e-06 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807524+0000", 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2180000000000002e-06 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807531+0000", 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5539999999999999e-06 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807536+0000", 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4797000000000001e-05 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.836 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:200896 13.2 13:43f44a98:::benchmark_data_smithi064_501823_object90894:head [delete] snapc 0=[] ondisk+write+known_if_redirected e413)", 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.807541+0000", 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0027760630000000001, 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0030342709999999998, 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:45:05.837 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 200896 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.838 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807541+0000", 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807541+0000", 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.839 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807540+0000", 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5179999999999999e-06 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807543+0000", 2023-06-10T21:45:05.840 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0140000000000004e-06 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807549+0000", 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.296e-06 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.841 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.807553+0000", 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028334250000000001 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810386+0000", 2023-06-10T21:45:05.842 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013331700000000001 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810520+0000", 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029791700000000002 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.843 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200882 13.8 e413/391 13:1da916d5:::benchmark_data_smithi064_501823_object90880:head v 413'12617, mlcod=383'4779)", 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808004+0000", 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0023124500000000002, 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002650384, 2023-06-10T21:45:05.844 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808004+0000", 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.845 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808004+0000", 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808003+0000", 2023-06-10T21:45:05.846 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0114e-05 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808014+0000", 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1219e-05 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.847 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808025+0000", 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4609999999999996e-06 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808031+0000", 2023-06-10T21:45:05.848 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022220030000000002 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810253+0000", 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4793000000000001e-05 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.849 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810278+0000", 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0022737719999999999 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.850 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200883 13.4 e413/395 13:2ade6e75:::benchmark_data_smithi064_501823_object90881:head v 413'12535, mlcod=413'12533)", 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808047+0000", 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0022698969999999999, 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026743600000000002, 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.851 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808047+0000", 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808047+0000", 2023-06-10T21:45:05.852 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808046+0000", 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7430000000000002e-06 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.853 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808051+0000", 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9689999999999997e-06 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808058+0000", 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4109999999999998e-06 2023-06-10T21:45:05.854 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808062+0000", 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8869999999999999e-05 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.855 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808101+0000", 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9890000000000001e-05 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808121+0000", 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4459e-05 2023-06-10T21:45:05.856 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200891 13.f e413/401 13:f108e3b1:::benchmark_data_smithi064_501823_object90889:head v 413'12424, mlcod=413'12422)", 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808091+0000", 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0022258389999999999, 2023-06-10T21:45:05.857 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027039630000000002, 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808091+0000", 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.858 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808091+0000", 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.859 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808090+0000", 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808096+0000", 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7549999999999997e-06 2023-06-10T21:45:05.860 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808103+0000", 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3939999999999998e-06 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.861 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808107+0000", 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012917199999999999 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808236+0000", 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6996000000000001e-05 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.862 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808253+0000", 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016243599999999999 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.863 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200881 13.7 e413/391 13:ec574907:::benchmark_data_smithi064_501823_object90879:head v 413'12445, mlcod=383'4734)", 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808120+0000", 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0021969670000000002, 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027396040000000001, 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:45:05.864 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808120+0000", 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.865 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808120+0000", 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808119+0000", 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4860000000000002e-06 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.866 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808123+0000", 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4190000000000001e-06 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808128+0000", 2023-06-10T21:45:05.867 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.912e-06 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808132+0000", 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00243684 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.868 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810569+0000", 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0315000000000001e-05 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.810589+0000", 2023-06-10T21:45:05.869 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024696869999999999 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200893 13.d e413/391)", 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808136+0000", 2023-06-10T21:45:05.870 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0021805269999999998, 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0027838099999999998, 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.871 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808136+0000", 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808136+0000", 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967296 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.872 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808136+0000", 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8549999999999996e-06 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808143+0000", 2023-06-10T21:45:05.873 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.485e-06 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808148+0000", 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6229999999999999e-06 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.874 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808152+0000", 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5758999999999999e-05 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.875 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.46829.0:200894 13.0 e413/391)", 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:45:05.808156+0000", 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0021605249999999999, 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028108550000000001, 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:45:05.876 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808156+0000", 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808156+0000", 2023-06-10T21:45:05.877 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808156+0000", 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0350000000000003e-06 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.878 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808160+0000", 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5609999999999998e-06 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808165+0000", 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.653e-06 2023-06-10T21:45:05.879 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:45:05.808169+0000", 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3003000000000001e-05 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:45:05.880 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.881 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:45:05.881 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:45:05.881 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 10 2023-06-10T21:45:05.881 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:45:05.906 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:05.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:06.012 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:06.082 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:06.183 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:06.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:06.290 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:06.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:06 smithi064 ceph-mon[180297]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 818 op/s; 18108/65085 objects degraded (27.822%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:06.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:06 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 18108/71532 objects degraded (25.315%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:06.374 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:06 smithi152 ceph-mon[156819]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 818 op/s; 18108/65085 objects degraded (27.822%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:06.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:06 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 18108/71532 objects degraded (25.315%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:06.480 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:06.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:06.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:06.601 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:06.602 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18108/65085 objects degraded (27.822%), 10 pgs degraded, 10 pgs undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2179,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":413,"num_osds":8,"num_up_osds":8,"osd_up_since":1686433380,"num_in_osds":4,"osd_in_since":1686433189,"num_remapped_pgs":10},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":77},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+recovery_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":21695,"data_bytes":1420760998,"bytes_used":6476087296,"bytes_avail":377504055296,"bytes_total":383980142592,"degraded_objects":18108,"degraded_total":65085,"degraded_ratio":0.27822078820004609,"recovering_objects_per_sec":66,"recovering_bytes_per_sec":4356481,"recovering_keys_per_sec":0,"num_objects_recovered":665,"num_bytes_recovered":43581440,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":818},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":30,"modified":"2023-06-10T21:44:56.203656+0000","services":{"osd":{"daemons":{"summary":"","1":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true},"9ab7ee0e-4ecd-48a4-8ef6-12d592a043b5":{"message":"Global Recovery Event (2m)\n [========================....] (remaining: 23s)","progress":0.86516851186752319,"add_to_ceph_s":true}}} 2023-06-10T21:45:06.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:06 smithi064 ceph-mon[186706]: pgmap v1525: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.3 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 818 op/s; 18108/65085 objects degraded (27.822%); 4.2 MiB/s, 66 objects/s recovering 2023-06-10T21:45:06.644 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:06 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 18108/71532 objects degraded (25.315%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:06.665 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:06.799 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:45:06.934 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:06.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:45:06.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:06.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:07.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:07.120 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:07.247 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:07.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:07.355 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:07.383 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:07.412 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:45:07.641 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/376204663' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:07.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/376204663' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:07.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/376204663' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:07.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:45:07.747 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:07.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:07.857 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:07.880 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-10T21:45:07.881 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:45:07.923 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:08.080 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:08.087 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:08.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:08.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:08.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:08.287 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:08.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:08.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:08.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:08.497 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:08.568 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:08.613 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:08.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:08 smithi064 ceph-mon[180297]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 831 op/s; 17744/58476 objects degraded (30.344%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:08.635 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:08 smithi064 ceph-mon[186706]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 831 op/s; 17744/58476 objects degraded (30.344%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:08.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:08 smithi152 ceph-mon[156819]: pgmap v1526: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.2 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 831 op/s; 17744/58476 objects degraded (30.344%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:08.902 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:08.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:09.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:09.046 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:09.091 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:09.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:09.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:09.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:09.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:09.500 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:45:09.501 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:45:09.501 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:45:09.501 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:45:09.501 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:45:09.588 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:09.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:09.595 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:09.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:09.697 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:09.768 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:10.185 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:10.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:10.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:10.278 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:10.295 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:10.312 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:10.348 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:10.395 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:45:10.397 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:10.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:10.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:10.414 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:10.454 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:10.495 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:10.502 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:10.530 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:10.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:10 smithi064 ceph-mon[180297]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 17744/53217 objects degraded (33.343%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:10.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/941395518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:10.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:10 smithi064 ceph-mon[186706]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 17744/53217 objects degraded (33.343%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:10.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/941395518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:10.557 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:10.568 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:10.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:10.624 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.624 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.625 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:45:10.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:10 smithi152 ceph-mon[156819]: pgmap v1527: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 1.1 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 837 op/s; 17744/53217 objects degraded (33.343%); 4.2 MiB/s, 67 objects/s recovering 2023-06-10T21:45:10.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/941395518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:11.058 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:45:11.554 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:11 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 17744/53217 objects degraded (33.343%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:11.555 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:11 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 17744/53217 objects degraded (33.343%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:11.614 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:11 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 17744/53217 objects degraded (33.343%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:12.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:12 smithi064 ceph-mon[180297]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:12 smithi064 ceph-mon[186706]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:12.616 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:12 smithi152 ceph-mon[156819]: pgmap v1528: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:12.654 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:12.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:12.766 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:12.835 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:12.903 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:12] "GET /metrics HTTP/1.1" 200 224086 "" "Prometheus/2.33.4" 2023-06-10T21:45:12.932 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:12.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:13.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:13.110 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:13.247 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:45:13.248 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:45:13.285 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:13.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:13.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:13.392 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:13.452 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:13.553 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:13.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:13.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:13.696 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:13.738 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:13.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:13.745 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:45:13.800 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:13.801 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:45:13.844 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:13.881 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:13.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2024311146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:13.987 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2024311146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:13.987 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:13.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2024311146' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:13.988 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:14.172 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:45:14.679 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:45:14.976 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:14.980 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:14.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:14.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: osdmap e414: 8 total, 8 up, 4 in 2023-06-10T21:45:14.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.981 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: from='client.47069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:14.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.982 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[180297]: pgmap v1530: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:14.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: osdmap e414: 8 total, 8 up, 4 in 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.984 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: from='client.47069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:14.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: failed to encode map e414 with expected crc 2023-06-10T21:45:14.985 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:14 smithi064 ceph-mon[186706]: pgmap v1530: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:15.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:15.164 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:15.166 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: osdmap e414: 8 total, 8 up, 4 in 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: from='client.47069 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.167 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: failed to encode map e414 with expected crc 2023-06-10T21:45:15.168 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:14 smithi152 ceph-mon[156819]: pgmap v1530: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 9 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 971 MiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 17400/46677 objects degraded (37.277%); 4.4 MiB/s, 70 objects/s recovering 2023-06-10T21:45:15.260 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:15.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:15.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:15.414 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:15.457 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:15.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:15.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:15.563 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:15.596 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:15.684 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:15.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:15.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:15.863 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:15.951 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:15.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:15.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:15 smithi152 ceph-mon[156819]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:15 smithi152 ceph-mon[156819]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:15 smithi152 ceph-mon[156819]: osdmap e415: 8 total, 8 up, 4 in 2023-06-10T21:45:15.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:15 smithi152 ceph-mon[156819]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.963 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:15 smithi152 ceph-mon[156819]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[186706]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[186706]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[186706]: osdmap e415: 8 total, 8 up, 4 in 2023-06-10T21:45:15.998 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[186706]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.999 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[186706]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[180297]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[180297]: failed to encode map e415 with expected crc 2023-06-10T21:45:15.999 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[180297]: osdmap e415: 8 total, 8 up, 4 in 2023-06-10T21:45:16.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[180297]: failed to encode map e415 with expected crc 2023-06-10T21:45:16.000 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:15 smithi064 ceph-mon[180297]: failed to encode map e415 with expected crc 2023-06-10T21:45:16.041 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1529,"stamp":"2023-06-10T21:45:13.734329+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1018632102,"num_objects":15559,"num_object_clones":0,"num_object_copies":46677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17400,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15559,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":204726,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9506,"num_bytes_recovered":620316686,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75543,"ondisk_log_size":75543,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":5540568,"kb_used_data":2577208,"kb_used_omap":27,"kb_used_meta":2962980,"kb_avail":369440040,"statfs":{"total":383980142592,"available":378306600960,"internally_reserved":0,"allocated":2639060992,"data_stored":2634840176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3034092122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,4,2,8],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-542965760,"num_objects":-8285,"num_object_clones":0,"num_object_copies":-24855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-708,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8285,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":708,"num_bytes_recovered":46399488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.402292"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":452,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.314349+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:44:51.314349+0000","last_peered":"2023-06-10T21:44:51.314349+0000","last_clean":"2023-06-10T21:44:51.314349+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:44:51.314349+0000","last_undegraded":"2023-06-10T21:44:51.314349+0000","last_fullsized":"2023-06-10T21:44:51.314349+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":81,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325027+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:44:51.325027+0000","last_peered":"2023-06-10T21:44:51.325027+0000","last_clean":"2023-06-10T21:44:51.325027+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:44:51.325027+0000","last_undegraded":"2023-06-10T21:44:51.325027+0000","last_fullsized":"2023-06-10T21:44:51.325027+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":344,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325985+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:44:51.325985+0000","last_peered":"2023-06-10T21:44:51.325985+0000","last_clean":"2023-06-10T21:44:51.325985+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:44:51.325985+0000","last_undegraded":"2023-06-10T21:44:51.325985+0000","last_fullsized":"2023-06-10T21:44:51.325985+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.318419+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:44:51.318419+0000","last_peered":"2023-06-10T21:44:51.318419+0000","last_clean":"2023-06-10T21:44:51.318419+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:44:51.318419+0000","last_undegraded":"2023-06-10T21:44:51.318419+0000","last_fullsized":"2023-06-10T21:44:51.318419+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":66,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.321279+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:44:51.321279+0000","last_peered":"2023-06-10T21:44:51.321279+0000","last_clean":"2023-06-10T21:44:51.321279+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:44:51.321279+0000","last_undegraded":"2023-06-10T21:44:51.321279+0000","last_fullsized":"2023-06-10T21:44:51.321279+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":95,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313351+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:44:51.313351+0000","last_peered":"2023-06-10T21:44:51.313351+0000","last_clean":"2023-06-10T21:44:51.313351+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:44:51.313351+0000","last_undegraded":"2023-06-10T21:44:51.313351+0000","last_fullsized":"2023-06-10T21:44:51.313351+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":278,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313149+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:44:51.313149+0000","last_peered":"2023-06-10T21:44:51.313149+0000","last_clean":"2023-06-10T21:44:51.313149+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:44:51.313149+0000","last_undegraded":"2023-06-10T21:44:51.313149+0000","last_fullsized":"2023-06-10T21:44:51.313149+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":82,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:03.346025+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:45:03.346025+0000","last_peered":"2023-06-10T21:45:03.346025+0000","last_clean":"2023-06-10T21:45:03.346025+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:45:03.346025+0000","last_undegraded":"2023-06-10T21:45:03.346025+0000","last_fullsized":"2023-06-10T21:45:03.346025+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":86,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313576+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:44:51.313576+0000","last_peered":"2023-06-10T21:44:51.313576+0000","last_clean":"2023-06-10T21:44:51.313576+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:44:51.313576+0000","last_undegraded":"2023-06-10T21:44:51.313576+0000","last_fullsized":"2023-06-10T21:44:51.313576+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325911+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:44:51.325911+0000","last_peered":"2023-06-10T21:44:51.325911+0000","last_clean":"2023-06-10T21:44:51.325911+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:44:51.325911+0000","last_undegraded":"2023-06-10T21:44:51.325911+0000","last_fullsized":"2023-06-10T21:44:51.325911+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":170,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316061+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:44:51.316061+0000","last_peered":"2023-06-10T21:44:51.316061+0000","last_clean":"2023-06-10T21:44:51.316061+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:44:51.316061+0000","last_undegraded":"2023-06-10T21:44:51.316061+0000","last_fullsized":"2023-06-10T21:44:51.316061+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":413,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:45:01.227086+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:45:01.227086+0000","last_peered":"2023-06-10T21:45:01.227086+0000","last_clean":"2023-06-10T21:45:01.227086+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:45:01.227086+0000","last_undegraded":"2023-06-10T21:45:01.227086+0000","last_fullsized":"2023-06-10T21:45:01.227086+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":336,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316310+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:44:51.316310+0000","last_peered":"2023-06-10T21:44:51.316310+0000","last_clean":"2023-06-10T21:44:51.316310+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:44:51.316310+0000","last_undegraded":"2023-06-10T21:44:51.316310+0000","last_fullsized":"2023-06-10T21:44:51.316310+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313205+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:44:51.313205+0000","last_peered":"2023-06-10T21:44:51.313205+0000","last_clean":"2023-06-10T21:44:51.313205+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:44:51.313205+0000","last_undegraded":"2023-06-10T21:44:51.313205+0000","last_fullsized":"2023-06-10T21:44:51.313205+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":472,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316025+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:44:51.316025+0000","last_peered":"2023-06-10T21:44:51.316025+0000","last_clean":"2023-06-10T21:44:51.316025+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:44:51.316025+0000","last_undegraded":"2023-06-10T21:44:51.316025+0000","last_fullsized":"2023-06-10T21:44:51.316025+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":360,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.322384+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:44:51.322384+0000","last_peered":"2023-06-10T21:44:51.322384+0000","last_clean":"2023-06-10T21:44:51.322384+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:44:51.322384+0000","last_undegraded":"2023-06-10T21:44:51.322384+0000","last_fullsized":"2023-06-10T21:44:51.322384+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":283,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.317078+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:44:51.317078+0000","last_peered":"2023-06-10T21:44:51.317078+0000","last_clean":"2023-06-10T21:44:51.317078+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:44:51.317078+0000","last_undegraded":"2023-06-10T21:44:51.317078+0000","last_fullsized":"2023-06-10T21:44:51.317078+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":95,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313499+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:44:51.313499+0000","last_peered":"2023-06-10T21:44:51.313499+0000","last_clean":"2023-06-10T21:44:51.313499+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:44:51.313499+0000","last_undegraded":"2023-06-10T21:44:51.313499+0000","last_fullsized":"2023-06-10T21:44:51.313499+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":425,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.315914+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:44:51.315914+0000","last_peered":"2023-06-10T21:44:51.315914+0000","last_clean":"2023-06-10T21:44:51.315914+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:44:51.315914+0000","last_undegraded":"2023-06-10T21:44:51.315914+0000","last_fullsized":"2023-06-10T21:44:51.315914+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":66,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.332030+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:44:51.332030+0000","last_peered":"2023-06-10T21:44:51.332030+0000","last_clean":"2023-06-10T21:44:51.332030+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:44:51.332030+0000","last_undegraded":"2023-06-10T21:44:51.332030+0000","last_fullsized":"2023-06-10T21:44:51.332030+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":437,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312769+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:44:51.312769+0000","last_peered":"2023-06-10T21:44:51.312769+0000","last_clean":"2023-06-10T21:44:51.312769+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:44:51.312769+0000","last_undegraded":"2023-06-10T21:44:51.312769+0000","last_fullsized":"2023-06-10T21:44:51.312769+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.335739+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:44:51.335739+0000","last_peered":"2023-06-10T21:44:51.335739+0000","last_clean":"2023-06-10T21:44:51.335739+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:44:51.335739+0000","last_undegraded":"2023-06-10T21:44:51.335739+0000","last_fullsized":"2023-06-10T21:44:51.335739+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"413'12746","reported_seq":12796,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:11.115014+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:45:11.115014+0000","last_peered":"2023-06-10T21:45:11.115014+0000","last_clean":"2023-06-10T21:45:11.115014+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:45:11.115014+0000","last_undegraded":"2023-06-10T21:45:11.115014+0000","last_fullsized":"2023-06-10T21:45:11.115014+0000","mapping_epoch":373,"log_start":"393'7700","ondisk_log_start":"393'7700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5046,"log_dups_size":0,"ondisk_log_size":5046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63307776,"num_objects":966,"num_object_clones":0,"num_object_copies":2898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12746,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":205,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.318553+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:44:51.318553+0000","last_peered":"2023-06-10T21:44:51.318553+0000","last_clean":"2023-06-10T21:44:51.318553+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:44:51.318553+0000","last_undegraded":"2023-06-10T21:44:51.318553+0000","last_fullsized":"2023-06-10T21:44:51.318553+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":86,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312622+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:44:51.312622+0000","last_peered":"2023-06-10T21:44:51.312622+0000","last_clean":"2023-06-10T21:44:51.312622+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:44:51.312622+0000","last_undegraded":"2023-06-10T21:44:51.312622+0000","last_fullsized":"2023-06-10T21:44:51.312622+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"413'12831","reported_seq":12897,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:09.256673+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:45:09.256673+0000","last_peered":"2023-06-10T21:45:09.256673+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:45:09.256673+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"397'8500","ondisk_log_start":"397'8500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4331,"log_dups_size":0,"ondisk_log_size":4331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66125824,"num_objects":1009,"num_object_clones":0,"num_object_copies":3027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12831,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-881}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":205,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.319868+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:44:51.319868+0000","last_peered":"2023-06-10T21:44:51.319868+0000","last_clean":"2023-06-10T21:44:51.319868+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:44:51.319868+0000","last_undegraded":"2023-06-10T21:44:51.319868+0000","last_fullsized":"2023-06-10T21:44:51.319868+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":67,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:55.962760+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:44:55.962760+0000","last_peered":"2023-06-10T21:44:55.962760+0000","last_clean":"2023-06-10T21:44:55.962760+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:44:55.962760+0000","last_undegraded":"2023-06-10T21:44:55.962760+0000","last_fullsized":"2023-06-10T21:44:55.962760+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"413'12889","reported_seq":12969,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:10.919302+0000","last_change":"2023-06-10T21:43:02.639566+0000","last_active":"2023-06-10T21:45:10.919302+0000","last_peered":"2023-06-10T21:45:10.919302+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:45:10.919302+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"398'8600","ondisk_log_start":"398'8600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4289,"log_dups_size":0,"ondisk_log_size":4289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1928,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12889,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-979}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":492,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316156+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:44:51.316156+0000","last_peered":"2023-06-10T21:44:51.316156+0000","last_clean":"2023-06-10T21:44:51.316156+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:44:51.316156+0000","last_undegraded":"2023-06-10T21:44:51.316156+0000","last_fullsized":"2023-06-10T21:44:51.316156+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":176,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316948+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:44:51.316948+0000","last_peered":"2023-06-10T21:44:51.316948+0000","last_clean":"2023-06-10T21:44:51.316948+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:44:51.316948+0000","last_undegraded":"2023-06-10T21:44:51.316948+0000","last_fullsized":"2023-06-10T21:44:51.316948+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"413'12806","reported_seq":16684,"reported_epoch":413,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:45:09.744149+0000","last_change":"2023-06-10T21:44:46.591362+0000","last_active":"2023-06-10T21:45:09.744149+0000","last_peered":"2023-06-10T21:45:09.744149+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:44:46.544938+0000","last_became_peered":"2023-06-10T21:44:46.544938+0000","last_unstale":"2023-06-10T21:45:09.744149+0000","last_undegraded":"2023-06-10T21:45:09.744149+0000","last_fullsized":"2023-06-10T21:45:09.744149+0000","mapping_epoch":410,"log_start":"396'8284","ondisk_log_start":"396'8284","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4522,"log_dups_size":0,"ondisk_log_size":4522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65404928,"num_objects":998,"num_object_clones":0,"num_object_copies":2994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":998,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12806,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1905,"num_bytes_recovered":124780544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":998}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":96,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312944+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:44:51.312944+0000","last_peered":"2023-06-10T21:44:51.312944+0000","last_clean":"2023-06-10T21:44:51.312944+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:44:51.312944+0000","last_undegraded":"2023-06-10T21:44:51.312944+0000","last_fullsized":"2023-06-10T21:44:51.312944+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:45:00.964475+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:45:00.964475+0000","last_peered":"2023-06-10T21:45:00.964475+0000","last_clean":"2023-06-10T21:45:00.964475+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:45:00.964475+0000","last_undegraded":"2023-06-10T21:45:00.964475+0000","last_fullsized":"2023-06-10T21:45:00.964475+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"413'12737","reported_seq":16597,"reported_epoch":413,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:45:09.743650+0000","last_change":"2023-06-10T21:44:23.399033+0000","last_active":"2023-06-10T21:45:09.743650+0000","last_peered":"2023-06-10T21:45:09.743650+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:44:23.337898+0000","last_became_peered":"2023-06-10T21:44:23.337898+0000","last_unstale":"2023-06-10T21:45:09.743650+0000","last_undegraded":"2023-06-10T21:45:09.743650+0000","last_fullsized":"2023-06-10T21:45:09.743650+0000","mapping_epoch":406,"log_start":"397'8240","ondisk_log_start":"397'8240","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4497,"log_dups_size":0,"ondisk_log_size":4497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12737,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":991}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":71,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:45:00.964463+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:45:00.964463+0000","last_peered":"2023-06-10T21:45:00.964463+0000","last_clean":"2023-06-10T21:45:00.964463+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:45:00.964463+0000","last_undegraded":"2023-06-10T21:45:00.964463+0000","last_fullsized":"2023-06-10T21:45:00.964463+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:45:00.964499+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:45:00.964499+0000","last_peered":"2023-06-10T21:45:00.964499+0000","last_clean":"2023-06-10T21:45:00.964499+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:45:00.964499+0000","last_undegraded":"2023-06-10T21:45:00.964499+0000","last_fullsized":"2023-06-10T21:45:00.964499+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"413'12789","reported_seq":16695,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:11.176374+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:45:11.176374+0000","last_peered":"2023-06-10T21:45:11.176374+0000","last_clean":"2023-06-10T21:45:11.176374+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:45:11.176374+0000","last_undegraded":"2023-06-10T21:45:11.176374+0000","last_fullsized":"2023-06-10T21:45:11.176374+0000","mapping_epoch":395,"log_start":"394'7700","ondisk_log_start":"394'7700","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60489728,"num_objects":923,"num_object_clones":0,"num_object_copies":2769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12789,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":234,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.322490+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:44:51.322490+0000","last_peered":"2023-06-10T21:44:51.322490+0000","last_clean":"2023-06-10T21:44:51.322490+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:44:51.322490+0000","last_undegraded":"2023-06-10T21:44:51.322490+0000","last_fullsized":"2023-06-10T21:44:51.322490+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":451,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.314612+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:44:51.314612+0000","last_peered":"2023-06-10T21:44:51.314612+0000","last_clean":"2023-06-10T21:44:51.314612+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:44:51.314612+0000","last_undegraded":"2023-06-10T21:44:51.314612+0000","last_fullsized":"2023-06-10T21:44:51.314612+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"413'12601","reported_seq":12664,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:09.254698+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:45:09.254698+0000","last_peered":"2023-06-10T21:45:09.254698+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:45:09.254698+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"398'8300","ondisk_log_start":"398'8300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4301,"log_dups_size":0,"ondisk_log_size":4301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63635456,"num_objects":971,"num_object_clones":0,"num_object_copies":2913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12601,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-882}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":428,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.317260+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:44:51.317260+0000","last_peered":"2023-06-10T21:44:51.317260+0000","last_clean":"2023-06-10T21:44:51.317260+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:44:51.317260+0000","last_undegraded":"2023-06-10T21:44:51.317260+0000","last_fullsized":"2023-06-10T21:44:51.317260+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":82,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:03.352896+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:45:03.352896+0000","last_peered":"2023-06-10T21:45:03.352896+0000","last_clean":"2023-06-10T21:45:03.352896+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:45:03.352896+0000","last_undegraded":"2023-06-10T21:45:03.352896+0000","last_fullsized":"2023-06-10T21:45:03.352896+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"413'12756","reported_seq":12813,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:11.161411+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:45:11.161411+0000","last_peered":"2023-06-10T21:45:11.161411+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:45:11.161411+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"393'7674","ondisk_log_start":"393'7674","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5082,"log_dups_size":0,"ondisk_log_size":5082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12756,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1017}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.321845+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:44:51.321845+0000","last_peered":"2023-06-10T21:44:51.321845+0000","last_clean":"2023-06-10T21:44:51.321845+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:44:51.321845+0000","last_undegraded":"2023-06-10T21:44:51.321845+0000","last_fullsized":"2023-06-10T21:44:51.321845+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":361,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.335997+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:44:51.335997+0000","last_peered":"2023-06-10T21:44:51.335997+0000","last_clean":"2023-06-10T21:44:51.335997+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:44:51.335997+0000","last_undegraded":"2023-06-10T21:44:51.335997+0000","last_fullsized":"2023-06-10T21:44:51.335997+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"413'12662","reported_seq":12718,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:11.142128+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:45:11.142128+0000","last_peered":"2023-06-10T21:45:11.142128+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:45:11.142128+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"393'7619","ondisk_log_start":"393'7619","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5043,"log_dups_size":0,"ondisk_log_size":5043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12662,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-995}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":196,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.318655+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:44:51.318655+0000","last_peered":"2023-06-10T21:44:51.318655+0000","last_clean":"2023-06-10T21:44:51.318655+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:44:51.318655+0000","last_undegraded":"2023-06-10T21:44:51.318655+0000","last_fullsized":"2023-06-10T21:44:51.318655+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":77,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313039+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:44:51.313039+0000","last_peered":"2023-06-10T21:44:51.313039+0000","last_clean":"2023-06-10T21:44:51.313039+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:44:51.313039+0000","last_undegraded":"2023-06-10T21:44:51.313039+0000","last_fullsized":"2023-06-10T21:44:51.313039+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312712+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:44:51.312712+0000","last_peered":"2023-06-10T21:44:51.312712+0000","last_clean":"2023-06-10T21:44:51.312712+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:44:51.312712+0000","last_undegraded":"2023-06-10T21:44:51.312712+0000","last_fullsized":"2023-06-10T21:44:51.312712+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"413'12664","reported_seq":12732,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:09.228037+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:45:09.228037+0000","last_peered":"2023-06-10T21:45:09.228037+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:45:09.228037+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"397'8300","ondisk_log_start":"397'8300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4364,"log_dups_size":0,"ondisk_log_size":4364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68091928,"num_objects":1040,"num_object_clones":0,"num_object_copies":3120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1040,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12664,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-877}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312689+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:44:51.312689+0000","last_peered":"2023-06-10T21:44:51.312689+0000","last_clean":"2023-06-10T21:44:51.312689+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:44:51.312689+0000","last_undegraded":"2023-06-10T21:44:51.312689+0000","last_fullsized":"2023-06-10T21:44:51.312689+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":374,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312840+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:44:51.312840+0000","last_peered":"2023-06-10T21:44:51.312840+0000","last_clean":"2023-06-10T21:44:51.312840+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:44:51.312840+0000","last_undegraded":"2023-06-10T21:44:51.312840+0000","last_fullsized":"2023-06-10T21:44:51.312840+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":278,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.319142+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:44:51.319142+0000","last_peered":"2023-06-10T21:44:51.319142+0000","last_clean":"2023-06-10T21:44:51.319142+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:44:51.319142+0000","last_undegraded":"2023-06-10T21:44:51.319142+0000","last_fullsized":"2023-06-10T21:44:51.319142+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"413'12622","reported_seq":12690,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:09.230675+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:45:09.230675+0000","last_peered":"2023-06-10T21:45:09.230675+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:45:09.230675+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"397'8300","ondisk_log_start":"397'8300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12622,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-817}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":298,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.317182+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:44:51.317182+0000","last_peered":"2023-06-10T21:44:51.317182+0000","last_clean":"2023-06-10T21:44:51.317182+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:44:51.317182+0000","last_undegraded":"2023-06-10T21:44:51.317182+0000","last_fullsized":"2023-06-10T21:44:51.317182+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":302,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.319645+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:44:51.319645+0000","last_peered":"2023-06-10T21:44:51.319645+0000","last_clean":"2023-06-10T21:44:51.319645+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:44:51.319645+0000","last_undegraded":"2023-06-10T21:44:51.319645+0000","last_fullsized":"2023-06-10T21:44:51.319645+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313230+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:44:51.313230+0000","last_peered":"2023-06-10T21:44:51.313230+0000","last_clean":"2023-06-10T21:44:51.313230+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:44:51.313230+0000","last_undegraded":"2023-06-10T21:44:51.313230+0000","last_fullsized":"2023-06-10T21:44:51.313230+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"413'12932","reported_seq":12990,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:11.178459+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:45:11.178459+0000","last_peered":"2023-06-10T21:45:11.178459+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:45:11.178459+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"394'7900","ondisk_log_start":"394'7900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259200,"num_objects":950,"num_object_clones":0,"num_object_copies":2850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12932,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-1014}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":87,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313099+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:44:51.313099+0000","last_peered":"2023-06-10T21:44:51.313099+0000","last_clean":"2023-06-10T21:44:51.313099+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:44:51.313099+0000","last_undegraded":"2023-06-10T21:44:51.313099+0000","last_fullsized":"2023-06-10T21:44:51.313099+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":613,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.315664+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:44:51.315664+0000","last_peered":"2023-06-10T21:44:51.315664+0000","last_clean":"2023-06-10T21:44:51.315664+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:44:51.315664+0000","last_undegraded":"2023-06-10T21:44:51.315664+0000","last_fullsized":"2023-06-10T21:44:51.315664+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.322349+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:44:51.322349+0000","last_peered":"2023-06-10T21:44:51.322349+0000","last_clean":"2023-06-10T21:44:51.322349+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:44:51.322349+0000","last_undegraded":"2023-06-10T21:44:51.322349+0000","last_fullsized":"2023-06-10T21:44:51.322349+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316327+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:44:51.316327+0000","last_peered":"2023-06-10T21:44:51.316327+0000","last_clean":"2023-06-10T21:44:51.316327+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:44:51.316327+0000","last_undegraded":"2023-06-10T21:44:51.316327+0000","last_fullsized":"2023-06-10T21:44:51.316327+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"413'12592","reported_seq":16503,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:09.222022+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:45:09.222022+0000","last_peered":"2023-06-10T21:45:09.222022+0000","last_clean":"2023-06-10T21:45:09.222022+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:45:09.222022+0000","last_undegraded":"2023-06-10T21:45:09.222022+0000","last_fullsized":"2023-06-10T21:45:09.222022+0000","mapping_epoch":401,"log_start":"397'8300","ondisk_log_start":"397'8300","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4292,"log_dups_size":0,"ondisk_log_size":4292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12592,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":465,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.315748+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:44:51.315748+0000","last_peered":"2023-06-10T21:44:51.315748+0000","last_clean":"2023-06-10T21:44:51.315748+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:44:51.315748+0000","last_undegraded":"2023-06-10T21:44:51.315748+0000","last_fullsized":"2023-06-10T21:44:51.315748+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325599+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:44:51.325599+0000","last_peered":"2023-06-10T21:44:51.325599+0000","last_clean":"2023-06-10T21:44:51.325599+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:44:51.325599+0000","last_undegraded":"2023-06-10T21:44:51.325599+0000","last_fullsized":"2023-06-10T21:44:51.325599+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":164,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.317904+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:44:51.317904+0000","last_peered":"2023-06-10T21:44:51.317904+0000","last_clean":"2023-06-10T21:44:51.317904+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:44:51.317904+0000","last_undegraded":"2023-06-10T21:44:51.317904+0000","last_fullsized":"2023-06-10T21:44:51.317904+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"413'12832","reported_seq":12882,"reported_epoch":413,"state":"active+clean","last_fresh":"2023-06-10T21:45:11.130505+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:45:11.130505+0000","last_peered":"2023-06-10T21:45:11.130505+0000","last_clean":"2023-06-10T21:45:11.130505+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:45:11.130505+0000","last_undegraded":"2023-06-10T21:45:11.130505+0000","last_fullsized":"2023-06-10T21:45:11.130505+0000","mapping_epoch":373,"log_start":"394'7800","ondisk_log_start":"394'7800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59113472,"num_objects":902,"num_object_clones":0,"num_object_copies":2706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12832,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":444,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316245+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:44:51.316245+0000","last_peered":"2023-06-10T21:44:51.316245+0000","last_clean":"2023-06-10T21:44:51.316245+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:44:51.316245+0000","last_undegraded":"2023-06-10T21:44:51.316245+0000","last_fullsized":"2023-06-10T21:44:51.316245+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.319229+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:44:51.319229+0000","last_peered":"2023-06-10T21:44:51.319229+0000","last_clean":"2023-06-10T21:44:51.319229+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:44:51.319229+0000","last_undegraded":"2023-06-10T21:44:51.319229+0000","last_fullsized":"2023-06-10T21:44:51.319229+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":217,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312948+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:44:51.312948+0000","last_peered":"2023-06-10T21:44:51.312948+0000","last_clean":"2023-06-10T21:44:51.312948+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:44:51.312948+0000","last_undegraded":"2023-06-10T21:44:51.312948+0000","last_fullsized":"2023-06-10T21:44:51.312948+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"413'13141","reported_seq":16773,"reported_epoch":413,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:11.237223+0000","last_change":"2023-06-10T21:44:45.105645+0000","last_active":"2023-06-10T21:45:11.237223+0000","last_peered":"2023-06-10T21:45:11.237223+0000","last_clean":"2023-06-10T21:42:30.052021+0000","last_became_active":"2023-06-10T21:43:02.580304+0000","last_became_peered":"2023-06-10T21:43:02.580304+0000","last_unstale":"2023-06-10T21:45:11.237223+0000","last_undegraded":"2023-06-10T21:43:02.226709+0000","last_fullsized":"2023-06-10T21:43:02.197351+0000","mapping_epoch":391,"log_start":"394'8100","ondisk_log_start":"394'8100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5041,"log_dups_size":0,"ondisk_log_size":5041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:10:10.860920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62455808,"num_objects":953,"num_object_clones":0,"num_object_copies":2859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":190,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13141,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1786,"num_bytes_recovered":117047296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":763}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":423,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.335841+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:44:51.335841+0000","last_peered":"2023-06-10T21:44:51.335841+0000","last_clean":"2023-06-10T21:44:51.335841+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:44:51.335841+0000","last_undegraded":"2023-06-10T21:44:51.335841+0000","last_fullsized":"2023-06-10T21:44:51.335841+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":234,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.332244+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:44:51.332244+0000","last_peered":"2023-06-10T21:44:51.332244+0000","last_clean":"2023-06-10T21:44:51.332244+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:44:51.332244+0000","last_undegraded":"2023-06-10T21:44:51.332244+0000","last_fullsized":"2023-06-10T21:44:51.332244+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":82,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.321760+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:44:51.321760+0000","last_peered":"2023-06-10T21:44:51.321760+0000","last_clean":"2023-06-10T21:44:51.321760+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:44:51.321760+0000","last_undegraded":"2023-06-10T21:44:51.321760+0000","last_fullsized":"2023-06-10T21:44:51.321760+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"413'12865","reported_seq":12930,"reported_epoch":413,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:11.151305+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:45:11.151305+0000","last_peered":"2023-06-10T21:45:11.151305+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:45:11.151305+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"394'7800","ondisk_log_start":"394'7800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5065,"log_dups_size":0,"ondisk_log_size":5065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62980096,"num_objects":961,"num_object_clones":0,"num_object_copies":2883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12865,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-960}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":161,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.318634+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:44:51.318634+0000","last_peered":"2023-06-10T21:44:51.318634+0000","last_clean":"2023-06-10T21:44:51.318634+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:44:51.318634+0000","last_undegraded":"2023-06-10T21:44:51.318634+0000","last_fullsized":"2023-06-10T21:44:51.318634+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":275,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.318233+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:44:51.318233+0000","last_peered":"2023-06-10T21:44:51.318233+0000","last_clean":"2023-06-10T21:44:51.318233+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:44:51.318233+0000","last_undegraded":"2023-06-10T21:44:51.318233+0000","last_fullsized":"2023-06-10T21:44:51.318233+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":67,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312698+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:44:51.312698+0000","last_peered":"2023-06-10T21:44:51.312698+0000","last_clean":"2023-06-10T21:44:51.312698+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:44:51.312698+0000","last_undegraded":"2023-06-10T21:44:51.312698+0000","last_fullsized":"2023-06-10T21:44:51.312698+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":112,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.316170+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:44:51.316170+0000","last_peered":"2023-06-10T21:44:51.316170+0000","last_clean":"2023-06-10T21:44:51.316170+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:44:51.316170+0000","last_undegraded":"2023-06-10T21:44:51.316170+0000","last_fullsized":"2023-06-10T21:44:51.316170+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":309,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.319316+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:44:51.319316+0000","last_peered":"2023-06-10T21:44:51.319316+0000","last_clean":"2023-06-10T21:44:51.319316+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:44:51.319316+0000","last_undegraded":"2023-06-10T21:44:51.319316+0000","last_fullsized":"2023-06-10T21:44:51.319316+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":75,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312859+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:44:51.312859+0000","last_peered":"2023-06-10T21:44:51.312859+0000","last_clean":"2023-06-10T21:44:51.312859+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:44:51.312859+0000","last_undegraded":"2023-06-10T21:44:51.312859+0000","last_fullsized":"2023-06-10T21:44:51.312859+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":317,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325250+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:44:51.325250+0000","last_peered":"2023-06-10T21:44:51.325250+0000","last_clean":"2023-06-10T21:44:51.325250+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:44:51.325250+0000","last_undegraded":"2023-06-10T21:44:51.325250+0000","last_fullsized":"2023-06-10T21:44:51.325250+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":98,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.312924+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:44:51.312924+0000","last_peered":"2023-06-10T21:44:51.312924+0000","last_clean":"2023-06-10T21:44:51.312924+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:44:51.312924+0000","last_undegraded":"2023-06-10T21:44:51.312924+0000","last_fullsized":"2023-06-10T21:44:51.312924+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":388,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.322565+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:44:51.322565+0000","last_peered":"2023-06-10T21:44:51.322565+0000","last_clean":"2023-06-10T21:44:51.322565+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:44:51.322565+0000","last_undegraded":"2023-06-10T21:44:51.322565+0000","last_fullsized":"2023-06-10T21:44:51.322565+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":67,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.332113+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:44:51.332113+0000","last_peered":"2023-06-10T21:44:51.332113+0000","last_clean":"2023-06-10T21:44:51.332113+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:44:51.332113+0000","last_undegraded":"2023-06-10T21:44:51.332113+0000","last_fullsized":"2023-06-10T21:44:51.332113+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":81,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.325780+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:44:51.325780+0000","last_peered":"2023-06-10T21:44:51.325780+0000","last_clean":"2023-06-10T21:44:51.325780+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:44:51.325780+0000","last_undegraded":"2023-06-10T21:44:51.325780+0000","last_fullsized":"2023-06-10T21:44:51.325780+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":334,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.313453+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:44:51.313453+0000","last_peered":"2023-06-10T21:44:51.313453+0000","last_clean":"2023-06-10T21:44:51.313453+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:44:51.313453+0000","last_undegraded":"2023-06-10T21:44:51.313453+0000","last_fullsized":"2023-06-10T21:44:51.313453+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":412,"state":"active+clean","last_fresh":"2023-06-10T21:44:51.321921+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:44:51.321921+0000","last_peered":"2023-06-10T21:44:51.321921+0000","last_clean":"2023-06-10T21:44:51.321921+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:44:51.321921+0000","last_undegraded":"2023-06-10T21:44:51.321921+0000","last_fullsized":"2023-06-10T21:44:51.321921+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":1018036248,"num_objects":15535,"num_object_clones":0,"num_object_copies":46605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17400,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15535,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":204465,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9431,"num_bytes_recovered":617938944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2584948736,"data_stored":2584936520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75348,"ondisk_log_size":75348,"up":48,"acting":38,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430777,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1713496,"kb_used_data":748112,"kb_used_omap":0,"kb_used_meta":965056,"kb_avail":92031656,"statfs":{"total":95995035648,"available":94240415744,"internally_reserved":0,"allocated":766066688,"data_stored":765357905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":988217344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.272,"15min":2.5510000000000002},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":8.5129999999999999,"5min":22.277000000000001,"15min":184.83699999999999},"last":2.9830000000000001},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.5820000000000001,"15min":2.8540000000000001},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":20.568999999999999,"5min":22.462,"15min":185.58000000000001},"last":0.53200000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.8100000000000001,"15min":2.6560000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.1500000000000004,"5min":38.078000000000003,"15min":87.417000000000002},"last":4.9859999999999998},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.2509999999999999,"15min":2.9590000000000001},"min":{"1min":0.30299999999999999,"5min":0.28599999999999998,"15min":0.27300000000000002},"max":{"1min":10.484,"5min":38.341000000000001,"15min":87.438000000000002},"last":4.8739999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:44:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.1299999999999999,"15min":4.9729999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":36.627000000000002,"5min":75.219999999999999,"15min":172.33600000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":4.4880000000000004,"5min":3.1379999999999999,"15min":4.9610000000000003},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":36.533999999999999,"5min":75.108999999999995,"15min":172.185},"last":4.8049999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":7.7990000000000004,"5min":4.0899999999999999,"15min":4.7960000000000003},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":71.852000000000004,"5min":89.504999999999995,"15min":147.00700000000001},"last":5.0199999999999996},{"interface":"front","average":{"1min":7.2359999999999998,"5min":3.7599999999999998,"15min":4.5270000000000001},"min":{"1min":0.439,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":72.813999999999993,"5min":90.215999999999994,"15min":146.93899999999999},"last":4.8959999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":0.88900000000000001,"15min":0.59599999999999997},"min":{"1min":0.35499999999999998,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":13.429,"5min":13.429,"15min":13.429},"last":4.8479999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.1120000000000001,"15min":0.71699999999999997},"min":{"1min":0.34200000000000003,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":4.6760000000000002,"5min":22.77,"15min":22.77},"last":5.2830000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:45:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":10.191000000000001,"15min":11.515000000000001},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.984,"5min":131.619,"15min":131.619},"last":5.3230000000000004},{"interface":"front","average":{"1min":2.036,"5min":10.006,"15min":11.334},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.9370000000000003,"5min":134.928,"15min":134.928},"last":8.5410000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:44:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":0.89700000000000002,"15min":0.67000000000000004},"min":{"1min":0.41199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8280000000000003,"5min":10.890000000000001,"15min":10.890000000000001},"last":5.0510000000000002},{"interface":"front","average":{"1min":1.911,"5min":0.99099999999999999,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":4.3739999999999997,"5min":5.8849999999999998,"15min":5.8849999999999998},"last":4.9240000000000004}]}]},{"osd":6,"up_from":0,"seq":180388627005,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566448,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020424,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075836,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1741268,"kb_used_data":721868,"kb_used_omap":7,"kb_used_meta":1019384,"kb_avail":92003884,"statfs":{"total":95995035648,"available":94211977216,"internally_reserved":0,"allocated":739192832,"data_stored":737851463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1043849297},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":2.375,"15min":4.2670000000000003},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":6.6429999999999998,"5min":83.733000000000004,"15min":171.33699999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.9690000000000001,"15min":4.0640000000000001},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9300000000000002,"5min":82.322999999999993,"15min":171.56200000000001},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.653,"15min":5.3120000000000003},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.2370000000000001,"5min":85.709999999999994,"15min":171.286},"last":0.316},{"interface":"front","average":{"1min":1.542,"5min":3.548,"15min":3.891},"min":{"1min":0.27800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1379999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.379}]},{"osd":3,"last update":"Sat Jun 10 21:44:25 2023","interfaces":[{"interface":"back","average":{"1min":11.705,"5min":8.6259999999999994,"15min":6.6699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":100.786,"5min":145.07599999999999,"15min":171.78299999999999},"last":22.437000000000001},{"interface":"front","average":{"1min":11.17,"5min":7.9969999999999999,"15min":6.6219999999999999},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":100.73999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":22.268000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.266,"15min":2.3130000000000002},"min":{"1min":0.33500000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.9629999999999992,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.903,"5min":1.7669999999999999,"15min":3.2370000000000001},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.2539999999999996,"5min":14.858000000000001,"15min":171.18100000000001},"last":0.52700000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.9989999999999997,"15min":6.6539999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.0389999999999997,"5min":75.896000000000001,"15min":75.896000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.976,"5min":5.468,"15min":6.0499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":7.3460000000000001,"5min":76.337000000000003,"15min":76.337000000000003},"last":0.503}]},{"osd":6,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.476,"15min":2.4300000000000002},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.0139999999999993,"5min":9.0139999999999993,"15min":171.441},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.948,"5min":1.931,"15min":3.3029999999999999},"min":{"1min":0.51900000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.9199999999999999,"5min":14.893000000000001,"15min":171.24600000000001},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.9430000000000001,"15min":4.0979999999999999},"min":{"1min":0.503,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":219.131},"last":0.63600000000000001},{"interface":"front","average":{"1min":2.9279999999999999,"5min":1.585,"15min":4.0250000000000004},"min":{"1min":0.48099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":12.15,"5min":12.15,"15min":219.19},"last":0.54100000000000004}]}]},{"osd":3,"up_from":250,"seq":1073741824196,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1414108,"kb_used_data":672148,"kb_used_omap":7,"kb_used_meta":741944,"kb_avail":92331044,"statfs":{"total":95995035648,"available":94546989056,"internally_reserved":0,"allocated":688279552,"data_stored":686809874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":759750675},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6319999999999999,"15min":3.2730000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.5880000000000001,"5min":30.744,"15min":105.09},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.375,"5min":2.0739999999999998,"15min":3.3740000000000001},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.20799999999999999},"max":{"1min":4.4450000000000003,"5min":30.805,"15min":99.852999999999994},"last":6.3479999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.8280000000000001,"15min":3.3660000000000001},"min":{"1min":0.315,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":4.5300000000000002,"5min":13.249000000000001,"15min":94.766000000000005},"last":2.0640000000000001},{"interface":"front","average":{"1min":1.375,"5min":1.589,"15min":3.2789999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":5.4109999999999996,"5min":11.846,"15min":94.691000000000003},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.6499999999999999,"15min":4.319},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.9649999999999999,"5min":18.452000000000002,"15min":231.07900000000001},"last":6.3070000000000004},{"interface":"front","average":{"1min":2.1000000000000001,"5min":3.0259999999999998,"15min":5.5599999999999996},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":9.3930000000000007,"5min":18.733000000000001,"15min":231.58600000000001},"last":2.335}]},{"osd":4,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.5900000000000001,"15min":1.486},"min":{"1min":0.45600000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.8470000000000004,"5min":9.5150000000000006,"15min":13.417},"last":3.4289999999999998},{"interface":"front","average":{"1min":1.843,"5min":1.8300000000000001,"15min":2.2599999999999998},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":4.5049999999999999,"5min":10.722,"15min":19.728000000000002},"last":5.6020000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:45:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.097,"15min":1.0920000000000001},"min":{"1min":0.37,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.6269999999999998,"5min":6.532,"15min":6.532},"last":0.443},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.6160000000000001,"15min":2.6739999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.1130000000000004,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":7.8120000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.903,"15min":2.323},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8519999999999999,"5min":10.691000000000001,"15min":19.805},"last":5.5069999999999997},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.484,"15min":1.5529999999999999},"min":{"1min":0.48199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.6299999999999999,"5min":9.9719999999999995,"15min":17.327000000000002},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.1179999999999999,"15min":2.601},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":5.4109999999999996,"5min":10.648999999999999,"15min":79.984999999999999},"last":5.5599999999999996},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.603,"15min":1.847},"min":{"1min":0.51300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":5.9320000000000004,"5min":9.9380000000000006,"15min":80.272999999999996},"last":1.0569999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019461,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245467,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671696,"kb_used_data":435080,"kb_used_omap":11,"kb_used_meta":236596,"kb_avail":93073456,"statfs":{"total":95995035648,"available":95307218944,"internally_reserved":0,"allocated":445521920,"data_stored":444820934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":242274806},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.0579999999999998,"15min":2.0720000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.5209999999999999,"5min":1.361,"15min":1.335},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3170000000000002,"5min":4.266,"15min":4.266},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.6269999999999998,"15min":2.6480000000000001},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":1.379},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2229999999999999,"15min":2.262},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.9500000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.315}]},{"osd":2,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":9.5090000000000003,"5min":3.391,"15min":2.371},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":159.94999999999999,"5min":159.94999999999999,"15min":159.94999999999999},"last":2.3980000000000001},{"interface":"front","average":{"1min":9.3230000000000004,"5min":3.7090000000000001,"15min":2.7730000000000001},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":159.851,"5min":159.851,"15min":159.851},"last":2.8279999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":7.569,"15min":8.0109999999999992},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":54.920000000000002,"5min":133.07499999999999,"15min":133.07499999999999},"last":1.3380000000000001},{"interface":"front","average":{"1min":5.3339999999999996,"5min":7.3819999999999997,"15min":7.7240000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":54.848999999999997,"5min":133.03800000000001,"15min":133.03800000000001},"last":1.4330000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.96199999999999997,"15min":0.92600000000000005},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.4329999999999998,"5min":5.173,"15min":5.173},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":2.1099999999999999,"15min":2.242},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.238,"5min":7.476,"15min":7.476},"last":1.304}]},{"osd":6,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.752,"15min":1.8420000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.2149999999999999,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.173,"5min":1.3100000000000001,"15min":1.333},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":4.2910000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.625,"15min":1.6160000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.2430000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.9630000000000001,"5min":2.9049999999999998,"15min":2.895},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.1920000000000002,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":4.0750000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":734134272,"data_stored":734134272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":676139008,"data_stored":676134936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":431362048,"data_stored":431357976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":743313408,"data_stored":743309336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:45:16.063 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:16.158 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:16.178 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:16.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:16.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:16.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:16.300 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:16.372 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:16.423 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:16.547 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:16.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:16.661 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:16.726 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:16.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:16 smithi064 ceph-mon[180297]: pgmap v1532: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 848 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 17002/40728 objects degraded (41.745%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:16.837 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:16 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 17400/46677 objects degraded (37.277%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:16.838 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:16 smithi064 ceph-mon[186706]: pgmap v1532: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 848 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 17002/40728 objects degraded (41.745%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:16.838 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:16 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 17400/46677 objects degraded (37.277%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:16.838 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:16.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:16.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:16.947 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:17.031 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:17.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:16 smithi152 ceph-mon[156819]: pgmap v1532: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 848 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 739 op/s; 17002/40728 objects degraded (41.745%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:17.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:16 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 17400/46677 objects degraded (37.277%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:18.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:45:18.179 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:18.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:18.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:18.312 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:18.312 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:18.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:18.728 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:18.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:18.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:18.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:18.876 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:18.967 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:18.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:19.048 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:45:19.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:19.143 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:19.212 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:19.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:19.319 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:19.388 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:19 smithi064 ceph-mon[186706]: pgmap v1533: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 749 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 17002/35994 objects degraded (47.236%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:19.389 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:19 smithi064 ceph-mon[180297]: pgmap v1533: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 749 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 17002/35994 objects degraded (47.236%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:19.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:19.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:19 smithi152 ceph-mon[156819]: pgmap v1533: 89 pgs: 1 active, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 749 MiB data, 4.8 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 717 op/s; 17002/35994 objects degraded (47.236%); 5.8 MiB/s, 92 objects/s recovering 2023-06-10T21:45:19.494 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:19.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:19.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:19.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:19.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:19.745 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:19.765 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:45:19.766 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:45:19.790 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:19.817 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:19.842 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:19.872 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:19.922 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:19.961 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:19.996 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:20.030 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.040 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.041 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:45:20.104 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:20.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:20.219 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:20.295 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:20.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:20.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:20.384 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/512190858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:20.385 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/512190858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:20.484 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:20.547 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/512190858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:20.553 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:20.642 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:20.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:20.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:20.806 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:21 smithi064 ceph-mon[180297]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 622 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 17002/29880 objects degraded (56.901%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:45:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:21 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 17002/29880 objects degraded (56.901%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:21 smithi064 ceph-mon[186706]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 622 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 17002/29880 objects degraded (56.901%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:45:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:21 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 17002/29880 objects degraded (56.901%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:21.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:21 smithi152 ceph-mon[156819]: pgmap v1534: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 622 MiB data, 4.5 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 17002/29880 objects degraded (56.901%); 3.1 MiB/s, 49 objects/s recovering 2023-06-10T21:45:21.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:21 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 17002/29880 objects degraded (56.901%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:22.144 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:22.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:22.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:22.312 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:22.374 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:22.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:22.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:22 smithi064 ceph-mon[186706]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 923 op/s; 16693/23187 objects degraded (71.993%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:45:22.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:22 smithi064 ceph-mon[180297]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 923 op/s; 16693/23187 objects degraded (71.993%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:45:22.483 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:22.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:22 smithi152 ceph-mon[156819]: pgmap v1535: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 923 op/s; 16693/23187 objects degraded (71.993%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:45:22.863 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:45:23.007 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:23.036 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:23.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:23.150 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:23.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:22] "GET /metrics HTTP/1.1" 200 224091 "" "Prometheus/2.33.4" 2023-06-10T21:45:23.245 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:23.456 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:23.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:23.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:23.573 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:23.636 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:23.720 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:23.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:23.832 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:23.948 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:24.018 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:24.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:24.133 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:24.204 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:24.211 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:45:24.295 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:24.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:24.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:24.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/494590407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.447 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:24 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.447 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/494590407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.448 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:24 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.459 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:24.553 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:24.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:24.658 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/494590407' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:24 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:24.715 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:24.921 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:45:25.290 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:25.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:25.296 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: osdmap e416: 8 total, 8 up, 4 in 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[186706]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 16693/23187 objects degraded (71.993%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:45:25.297 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:25.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:25.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: osdmap e416: 8 total, 8 up, 4 in 2023-06-10T21:45:25.298 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:25 smithi064 ceph-mon[180297]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 16693/23187 objects degraded (71.993%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:45:25.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:25.452 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:25.625 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: failed to encode map e416 with expected crc 2023-06-10T21:45:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: osdmap e416: 8 total, 8 up, 4 in 2023-06-10T21:45:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:25 smithi152 ceph-mon[156819]: pgmap v1537: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 482 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 699 op/s; 16693/23187 objects degraded (71.993%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:45:25.848 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:25.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:25.954 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:26.009 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:26.177 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:26.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:26.282 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:26.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:26 smithi064 ceph-mon[180297]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 329 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 16344/15840 objects degraded (103.182%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:26.318 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:26 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 16693/23187 objects degraded (71.993%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:26.331 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:26.406 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:26.424 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:26.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:26.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:26.542 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:26.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:26 smithi064 ceph-mon[186706]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 329 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 16344/15840 objects degraded (103.182%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:26.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:26 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 16693/23187 objects degraded (71.993%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:26.618 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:26 smithi152 ceph-mon[156819]: pgmap v1538: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 329 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 829 op/s; 16344/15840 objects degraded (103.182%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:26 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 16693/23187 objects degraded (71.993%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:26.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:26.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:26.835 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:26.909 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:27.016 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:27.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:27.063 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:27.074 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:45:27.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:27.196 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:27.308 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:27.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:27.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:27 smithi064 ceph-mon[186706]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:27.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:27 smithi064 ceph-mon[180297]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:27.422 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:27.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:27 smithi152 ceph-mon[156819]: from='client.37074 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:27.482 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:27.583 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:27.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:27.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:27.775 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:27.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:45:28.490 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:28.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:28.596 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:28.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:28 smithi064 ceph-mon[186706]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 208 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 864 op/s; 16344/10050 objects degraded (162.627%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:28.642 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:28 smithi064 ceph-mon[180297]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 208 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 864 op/s; 16344/10050 objects degraded (162.627%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:28.646 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:28 smithi152 ceph-mon[156819]: pgmap v1539: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 208 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 864 op/s; 16344/10050 objects degraded (162.627%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:29.050 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 109999 objects 2023-06-10T21:45:29.050 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :141.613 2023-06-10T21:45:29.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:29.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:29.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:29.188 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-06-10T21:45:29.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-06-10T21:45:29.278 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:29.306 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:29.482 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1538,"stamp":"2023-06-10T21:45:26.218734+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":344987558,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16344,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":215005,"num_write_kb":7042383,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10562,"num_bytes_recovered":672745486,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75244,"ondisk_log_size":75244,"up":267,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3855024,"kb_used_data":870032,"kb_used_omap":27,"kb_used_meta":2984612,"kb_avail":371125584,"statfs":{"total":383980142592,"available":380032598016,"internally_reserved":0,"allocated":890912768,"data_stored":886682294,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3056243290},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,7,10,9,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-543686656,"num_objects":-8296,"num_object_clones":0,"num_object_copies":-24888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-658,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8296,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":658,"num_bytes_recovered":26345472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005209"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":454,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.740231+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:45:13.740231+0000","last_peered":"2023-06-10T21:45:13.740231+0000","last_clean":"2023-06-10T21:45:13.740231+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:45:13.740231+0000","last_undegraded":"2023-06-10T21:45:13.740231+0000","last_fullsized":"2023-06-10T21:45:13.740231+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.750949+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:45:14.750949+0000","last_peered":"2023-06-10T21:45:14.750949+0000","last_clean":"2023-06-10T21:45:14.750949+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:45:14.750949+0000","last_undegraded":"2023-06-10T21:45:14.750949+0000","last_fullsized":"2023-06-10T21:45:14.750949+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746730+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:45:14.746730+0000","last_peered":"2023-06-10T21:45:14.746730+0000","last_clean":"2023-06-10T21:45:14.746730+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:45:14.746730+0000","last_undegraded":"2023-06-10T21:45:14.746730+0000","last_fullsized":"2023-06-10T21:45:14.746730+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":301,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801718+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:45:14.801718+0000","last_peered":"2023-06-10T21:45:14.801718+0000","last_clean":"2023-06-10T21:45:14.801718+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:45:14.801718+0000","last_undegraded":"2023-06-10T21:45:14.801718+0000","last_fullsized":"2023-06-10T21:45:14.801718+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":70,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-06-10T21:45:24.196267+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:45:24.196267+0000","last_peered":"2023-06-10T21:45:24.196267+0000","last_clean":"2023-06-10T21:45:24.196267+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:45:24.196267+0000","last_undegraded":"2023-06-10T21:45:24.196267+0000","last_fullsized":"2023-06-10T21:45:24.196267+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745361+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:45:14.745361+0000","last_peered":"2023-06-10T21:45:14.745361+0000","last_clean":"2023-06-10T21:45:14.745361+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:45:14.745361+0000","last_undegraded":"2023-06-10T21:45:14.745361+0000","last_fullsized":"2023-06-10T21:45:14.745361+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":281,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745272+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:45:14.745272+0000","last_peered":"2023-06-10T21:45:14.745272+0000","last_clean":"2023-06-10T21:45:14.745272+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:45:14.745272+0000","last_undegraded":"2023-06-10T21:45:14.745272+0000","last_fullsized":"2023-06-10T21:45:14.745272+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746630+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:45:14.746630+0000","last_peered":"2023-06-10T21:45:14.746630+0000","last_clean":"2023-06-10T21:45:14.746630+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:45:14.746630+0000","last_undegraded":"2023-06-10T21:45:14.746630+0000","last_fullsized":"2023-06-10T21:45:14.746630+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745302+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:45:14.745302+0000","last_peered":"2023-06-10T21:45:14.745302+0000","last_clean":"2023-06-10T21:45:14.745302+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:45:14.745302+0000","last_undegraded":"2023-06-10T21:45:14.745302+0000","last_fullsized":"2023-06-10T21:45:14.745302+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746644+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:45:14.746644+0000","last_peered":"2023-06-10T21:45:14.746644+0000","last_clean":"2023-06-10T21:45:14.746644+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:45:14.746644+0000","last_undegraded":"2023-06-10T21:45:14.746644+0000","last_fullsized":"2023-06-10T21:45:14.746644+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":173,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799841+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:45:14.799841+0000","last_peered":"2023-06-10T21:45:14.799841+0000","last_clean":"2023-06-10T21:45:14.799841+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:45:14.799841+0000","last_undegraded":"2023-06-10T21:45:14.799841+0000","last_fullsized":"2023-06-10T21:45:14.799841+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":415,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.740310+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:45:13.740310+0000","last_peered":"2023-06-10T21:45:13.740310+0000","last_clean":"2023-06-10T21:45:13.740310+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:45:13.740310+0000","last_undegraded":"2023-06-10T21:45:13.740310+0000","last_fullsized":"2023-06-10T21:45:13.740310+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":339,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799616+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:45:14.799616+0000","last_peered":"2023-06-10T21:45:14.799616+0000","last_clean":"2023-06-10T21:45:14.799616+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:45:14.799616+0000","last_undegraded":"2023-06-10T21:45:14.799616+0000","last_fullsized":"2023-06-10T21:45:14.799616+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":130,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744617+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:45:14.744617+0000","last_peered":"2023-06-10T21:45:14.744617+0000","last_clean":"2023-06-10T21:45:14.744617+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:45:14.744617+0000","last_undegraded":"2023-06-10T21:45:14.744617+0000","last_fullsized":"2023-06-10T21:45:14.744617+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":475,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.800029+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:45:14.800029+0000","last_peered":"2023-06-10T21:45:14.800029+0000","last_clean":"2023-06-10T21:45:14.800029+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:45:14.800029+0000","last_undegraded":"2023-06-10T21:45:14.800029+0000","last_fullsized":"2023-06-10T21:45:14.800029+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":364,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:20.977248+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:45:20.977248+0000","last_peered":"2023-06-10T21:45:20.977248+0000","last_clean":"2023-06-10T21:45:20.977248+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:45:20.977248+0000","last_undegraded":"2023-06-10T21:45:20.977248+0000","last_fullsized":"2023-06-10T21:45:20.977248+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":286,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799677+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:45:14.799677+0000","last_peered":"2023-06-10T21:45:14.799677+0000","last_clean":"2023-06-10T21:45:14.799677+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:45:14.799677+0000","last_undegraded":"2023-06-10T21:45:14.799677+0000","last_fullsized":"2023-06-10T21:45:14.799677+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745022+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:45:14.745022+0000","last_peered":"2023-06-10T21:45:14.745022+0000","last_clean":"2023-06-10T21:45:14.745022+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:45:14.745022+0000","last_undegraded":"2023-06-10T21:45:14.745022+0000","last_fullsized":"2023-06-10T21:45:14.745022+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":428,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799545+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:45:14.799545+0000","last_peered":"2023-06-10T21:45:14.799545+0000","last_clean":"2023-06-10T21:45:14.799545+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:45:14.799545+0000","last_undegraded":"2023-06-10T21:45:14.799545+0000","last_fullsized":"2023-06-10T21:45:14.799545+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:20.977282+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:45:20.977282+0000","last_peered":"2023-06-10T21:45:20.977282+0000","last_clean":"2023-06-10T21:45:20.977282+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:45:20.977282+0000","last_undegraded":"2023-06-10T21:45:20.977282+0000","last_fullsized":"2023-06-10T21:45:20.977282+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":440,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744401+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:45:14.744401+0000","last_peered":"2023-06-10T21:45:14.744401+0000","last_clean":"2023-06-10T21:45:14.744401+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:45:14.744401+0000","last_undegraded":"2023-06-10T21:45:14.744401+0000","last_fullsized":"2023-06-10T21:45:14.744401+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:20.977315+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:45:20.977315+0000","last_peered":"2023-06-10T21:45:20.977315+0000","last_clean":"2023-06-10T21:45:20.977315+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:45:20.977315+0000","last_undegraded":"2023-06-10T21:45:20.977315+0000","last_fullsized":"2023-06-10T21:45:20.977315+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"415'13268","reported_seq":13320,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:21.233763+0000","last_change":"2023-06-10T21:41:28.177869+0000","last_active":"2023-06-10T21:45:21.233763+0000","last_peered":"2023-06-10T21:45:21.233763+0000","last_clean":"2023-06-10T21:45:21.233763+0000","last_became_active":"2023-06-10T21:41:28.177549+0000","last_became_peered":"2023-06-10T21:41:28.177549+0000","last_unstale":"2023-06-10T21:45:21.233763+0000","last_undegraded":"2023-06-10T21:45:21.233763+0000","last_fullsized":"2023-06-10T21:45:21.233763+0000","mapping_epoch":373,"log_start":"396'8200","ondisk_log_start":"396'8200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5068,"log_dups_size":0,"ondisk_log_size":5068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:12:59.579407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13268,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":208,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801327+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:45:14.801327+0000","last_peered":"2023-06-10T21:45:14.801327+0000","last_clean":"2023-06-10T21:45:14.801327+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:45:14.801327+0000","last_undegraded":"2023-06-10T21:45:14.801327+0000","last_fullsized":"2023-06-10T21:45:14.801327+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":89,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744259+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:45:14.744259+0000","last_peered":"2023-06-10T21:45:14.744259+0000","last_clean":"2023-06-10T21:45:14.744259+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:45:14.744259+0000","last_undegraded":"2023-06-10T21:45:14.744259+0000","last_fullsized":"2023-06-10T21:45:14.744259+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"416'13564","reported_seq":13633,"reported_epoch":416,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:24.214057+0000","last_change":"2023-06-10T21:43:02.588872+0000","last_active":"2023-06-10T21:45:24.214057+0000","last_peered":"2023-06-10T21:45:24.214057+0000","last_clean":"2023-06-10T21:42:30.126704+0000","last_became_active":"2023-06-10T21:43:02.580456+0000","last_became_peered":"2023-06-10T21:43:02.580456+0000","last_unstale":"2023-06-10T21:45:24.214057+0000","last_undegraded":"2023-06-10T21:43:02.223384+0000","last_fullsized":"2023-06-10T21:43:02.197811+0000","mapping_epoch":391,"log_start":"402'9200","ondisk_log_start":"402'9200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4364,"log_dups_size":0,"ondisk_log_size":4364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13564,"num_write_kb":442880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-1614}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":208,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801807+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:45:14.801807+0000","last_peered":"2023-06-10T21:45:14.801807+0000","last_clean":"2023-06-10T21:45:14.801807+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:45:14.801807+0000","last_undegraded":"2023-06-10T21:45:14.801807+0000","last_fullsized":"2023-06-10T21:45:14.801807+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746813+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:45:14.746813+0000","last_peered":"2023-06-10T21:45:14.746813+0000","last_clean":"2023-06-10T21:45:14.746813+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:45:14.746813+0000","last_undegraded":"2023-06-10T21:45:14.746813+0000","last_fullsized":"2023-06-10T21:45:14.746813+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.3","version":"416'13678","reported_seq":15494,"reported_epoch":416,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:25.974142+0000","last_change":"2023-06-10T21:45:13.860584+0000","last_active":"2023-06-10T21:45:25.974142+0000","last_peered":"2023-06-10T21:45:25.974142+0000","last_clean":"2023-06-10T21:42:30.252233+0000","last_became_active":"2023-06-10T21:43:02.498890+0000","last_became_peered":"2023-06-10T21:43:02.498890+0000","last_unstale":"2023-06-10T21:45:25.974142+0000","last_undegraded":"2023-06-10T21:43:02.211226+0000","last_fullsized":"2023-06-10T21:43:02.196825+0000","mapping_epoch":391,"log_start":"402'9300","ondisk_log_start":"402'9300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4378,"log_dups_size":0,"ondisk_log_size":4378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:01.469753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10485760,"num_objects":160,"num_object_clones":0,"num_object_copies":480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13678,"num_write_kb":442816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":873,"num_bytes_recovered":40435712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-902}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":495,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799889+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:45:14.799889+0000","last_peered":"2023-06-10T21:45:14.799889+0000","last_clean":"2023-06-10T21:45:14.799889+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:45:14.799889+0000","last_undegraded":"2023-06-10T21:45:14.799889+0000","last_fullsized":"2023-06-10T21:45:14.799889+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799513+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:45:14.799513+0000","last_peered":"2023-06-10T21:45:14.799513+0000","last_clean":"2023-06-10T21:45:14.799513+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:45:14.799513+0000","last_undegraded":"2023-06-10T21:45:14.799513+0000","last_fullsized":"2023-06-10T21:45:14.799513+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.2","version":"416'13583","reported_seq":17464,"reported_epoch":416,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:45:24.776339+0000","last_change":"2023-06-10T21:44:46.591362+0000","last_active":"2023-06-10T21:45:24.776339+0000","last_peered":"2023-06-10T21:45:24.776339+0000","last_clean":"2023-06-10T21:42:30.091461+0000","last_became_active":"2023-06-10T21:44:46.544938+0000","last_became_peered":"2023-06-10T21:44:46.544938+0000","last_unstale":"2023-06-10T21:45:24.776339+0000","last_undegraded":"2023-06-10T21:45:24.776339+0000","last_fullsized":"2023-06-10T21:45:24.776339+0000","mapping_epoch":410,"log_start":"400'9084","ondisk_log_start":"400'9084","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13583,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1905,"num_bytes_recovered":124780544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[{"shards":"3,5,7","objects":221}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744670+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:45:14.744670+0000","last_peered":"2023-06-10T21:45:14.744670+0000","last_clean":"2023-06-10T21:45:14.744670+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:45:14.744670+0000","last_undegraded":"2023-06-10T21:45:14.744670+0000","last_fullsized":"2023-06-10T21:45:14.744670+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746543+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:45:14.746543+0000","last_peered":"2023-06-10T21:45:14.746543+0000","last_clean":"2023-06-10T21:45:14.746543+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:45:14.746543+0000","last_undegraded":"2023-06-10T21:45:14.746543+0000","last_fullsized":"2023-06-10T21:45:14.746543+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.5","version":"416'13494","reported_seq":17357,"reported_epoch":416,"state":"active+recovery_wait","last_fresh":"2023-06-10T21:45:24.753705+0000","last_change":"2023-06-10T21:44:23.399033+0000","last_active":"2023-06-10T21:45:24.753705+0000","last_peered":"2023-06-10T21:45:24.753705+0000","last_clean":"2023-06-10T21:42:30.110012+0000","last_became_active":"2023-06-10T21:44:23.337898+0000","last_became_peered":"2023-06-10T21:44:23.337898+0000","last_unstale":"2023-06-10T21:45:24.753705+0000","last_undegraded":"2023-06-10T21:45:24.753705+0000","last_fullsized":"2023-06-10T21:45:24.753705+0000","mapping_epoch":406,"log_start":"399'8940","ondisk_log_start":"399'8940","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15335424,"num_objects":234,"num_object_clones":0,"num_object_copies":702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13494,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":234}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":74,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746461+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:45:14.746461+0000","last_peered":"2023-06-10T21:45:14.746461+0000","last_clean":"2023-06-10T21:45:14.746461+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:45:14.746461+0000","last_undegraded":"2023-06-10T21:45:14.746461+0000","last_fullsized":"2023-06-10T21:45:14.746461+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747318+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:45:14.747318+0000","last_peered":"2023-06-10T21:45:14.747318+0000","last_clean":"2023-06-10T21:45:14.747318+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:45:14.747318+0000","last_undegraded":"2023-06-10T21:45:14.747318+0000","last_fullsized":"2023-06-10T21:45:14.747318+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"415'13299","reported_seq":17206,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:21.231055+0000","last_change":"2023-06-10T21:43:33.323092+0000","last_active":"2023-06-10T21:45:21.231055+0000","last_peered":"2023-06-10T21:45:21.231055+0000","last_clean":"2023-06-10T21:45:21.231055+0000","last_became_active":"2023-06-10T21:43:33.311968+0000","last_became_peered":"2023-06-10T21:43:33.311968+0000","last_unstale":"2023-06-10T21:45:21.231055+0000","last_undegraded":"2023-06-10T21:45:21.231055+0000","last_fullsized":"2023-06-10T21:45:21.231055+0000","mapping_epoch":395,"log_start":"397'8200","ondisk_log_start":"397'8200","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5099,"log_dups_size":0,"ondisk_log_size":5099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:14:14.238734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13299,"num_write_kb":438784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1915,"num_bytes_recovered":125501440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":237,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747580+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:45:14.747580+0000","last_peered":"2023-06-10T21:45:14.747580+0000","last_clean":"2023-06-10T21:45:14.747580+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:45:14.747580+0000","last_undegraded":"2023-06-10T21:45:14.747580+0000","last_fullsized":"2023-06-10T21:45:14.747580+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":453,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.739867+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:45:13.739867+0000","last_peered":"2023-06-10T21:45:13.739867+0000","last_clean":"2023-06-10T21:45:13.739867+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:45:13.739867+0000","last_undegraded":"2023-06-10T21:45:13.739867+0000","last_fullsized":"2023-06-10T21:45:13.739867+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"416'13300","reported_seq":13366,"reported_epoch":416,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:24.244900+0000","last_change":"2023-06-10T21:43:02.588592+0000","last_active":"2023-06-10T21:45:24.244900+0000","last_peered":"2023-06-10T21:45:24.244900+0000","last_clean":"2023-06-10T21:42:30.099205+0000","last_became_active":"2023-06-10T21:43:02.579246+0000","last_became_peered":"2023-06-10T21:43:02.579246+0000","last_unstale":"2023-06-10T21:45:24.244900+0000","last_undegraded":"2023-06-10T21:43:02.225218+0000","last_fullsized":"2023-06-10T21:43:02.199081+0000","mapping_epoch":391,"log_start":"402'9000","ondisk_log_start":"402'9000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4300,"log_dups_size":0,"ondisk_log_size":4300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1853,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13300,"num_write_kb":434304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-1581}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":430,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.739635+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:45:13.739635+0000","last_peered":"2023-06-10T21:45:13.739635+0000","last_clean":"2023-06-10T21:45:13.739635+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:45:13.739635+0000","last_undegraded":"2023-06-10T21:45:13.739635+0000","last_fullsized":"2023-06-10T21:45:13.739635+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747822+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:45:14.747822+0000","last_peered":"2023-06-10T21:45:14.747822+0000","last_clean":"2023-06-10T21:45:14.747822+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:45:14.747822+0000","last_undegraded":"2023-06-10T21:45:14.747822+0000","last_fullsized":"2023-06-10T21:45:14.747822+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.6","version":"415'13261","reported_seq":13322,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:21.222795+0000","last_change":"2023-06-10T21:43:02.605783+0000","last_active":"2023-06-10T21:45:21.222795+0000","last_peered":"2023-06-10T21:45:21.222795+0000","last_clean":"2023-06-10T21:42:30.093155+0000","last_became_active":"2023-06-10T21:43:02.594086+0000","last_became_peered":"2023-06-10T21:43:02.594086+0000","last_unstale":"2023-06-10T21:45:21.222795+0000","last_undegraded":"2023-06-10T21:43:02.235497+0000","last_fullsized":"2023-06-10T21:43:02.197934+0000","mapping_epoch":391,"log_start":"396'8174","ondisk_log_start":"396'8174","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:00:20.272805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1935,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13261,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1522}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746966+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:45:14.746966+0000","last_peered":"2023-06-10T21:45:14.746966+0000","last_clean":"2023-06-10T21:45:14.746966+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:45:14.746966+0000","last_undegraded":"2023-06-10T21:45:14.746966+0000","last_fullsized":"2023-06-10T21:45:14.746966+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":364,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.748368+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:45:14.748368+0000","last_peered":"2023-06-10T21:45:14.748368+0000","last_clean":"2023-06-10T21:45:14.748368+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:45:14.748368+0000","last_undegraded":"2023-06-10T21:45:14.748368+0000","last_fullsized":"2023-06-10T21:45:14.748368+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.b","version":"415'13156","reported_seq":13214,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:21.244354+0000","last_change":"2023-06-10T21:43:02.600275+0000","last_active":"2023-06-10T21:45:21.244354+0000","last_peered":"2023-06-10T21:45:21.244354+0000","last_clean":"2023-06-10T21:42:30.078160+0000","last_became_active":"2023-06-10T21:43:02.594116+0000","last_became_peered":"2023-06-10T21:43:02.594116+0000","last_unstale":"2023-06-10T21:45:21.244354+0000","last_undegraded":"2023-06-10T21:43:02.238410+0000","last_fullsized":"2023-06-10T21:43:02.197513+0000","mapping_epoch":391,"log_start":"396'8119","ondisk_log_start":"396'8119","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5037,"log_dups_size":0,"ondisk_log_size":5037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:58:16.500987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28835840,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13156,"num_write_kb":435072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1489}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":199,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799671+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:45:14.799671+0000","last_peered":"2023-06-10T21:45:14.799671+0000","last_clean":"2023-06-10T21:45:14.799671+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:45:14.799671+0000","last_undegraded":"2023-06-10T21:45:14.799671+0000","last_fullsized":"2023-06-10T21:45:14.799671+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":80,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744685+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:45:14.744685+0000","last_peered":"2023-06-10T21:45:14.744685+0000","last_clean":"2023-06-10T21:45:14.744685+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:45:14.744685+0000","last_undegraded":"2023-06-10T21:45:14.744685+0000","last_fullsized":"2023-06-10T21:45:14.744685+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744383+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:45:14.744383+0000","last_peered":"2023-06-10T21:45:14.744383+0000","last_clean":"2023-06-10T21:45:14.744383+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:45:14.744383+0000","last_undegraded":"2023-06-10T21:45:14.744383+0000","last_fullsized":"2023-06-10T21:45:14.744383+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"416'13417","reported_seq":13488,"reported_epoch":416,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:24.260324+0000","last_change":"2023-06-10T21:43:02.576109+0000","last_active":"2023-06-10T21:45:24.260324+0000","last_peered":"2023-06-10T21:45:24.260324+0000","last_clean":"2023-06-10T21:42:30.100288+0000","last_became_active":"2023-06-10T21:43:02.568003+0000","last_became_peered":"2023-06-10T21:43:02.568003+0000","last_unstale":"2023-06-10T21:45:24.260324+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198679+0000","mapping_epoch":391,"log_start":"402'9100","ondisk_log_start":"402'9100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4317,"log_dups_size":0,"ondisk_log_size":4317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18743320,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1917,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13417,"num_write_kb":438465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":-1630}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":99,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744328+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:45:14.744328+0000","last_peered":"2023-06-10T21:45:14.744328+0000","last_clean":"2023-06-10T21:45:14.744328+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:45:14.744328+0000","last_undegraded":"2023-06-10T21:45:14.744328+0000","last_fullsized":"2023-06-10T21:45:14.744328+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":377,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744430+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:45:14.744430+0000","last_peered":"2023-06-10T21:45:14.744430+0000","last_clean":"2023-06-10T21:45:14.744430+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:45:14.744430+0000","last_undegraded":"2023-06-10T21:45:14.744430+0000","last_fullsized":"2023-06-10T21:45:14.744430+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":280,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.741939+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:45:13.741939+0000","last_peered":"2023-06-10T21:45:13.741939+0000","last_clean":"2023-06-10T21:45:13.741939+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:45:13.741939+0000","last_undegraded":"2023-06-10T21:45:13.741939+0000","last_fullsized":"2023-06-10T21:45:13.741939+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"416'13404","reported_seq":13475,"reported_epoch":416,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:24.244855+0000","last_change":"2023-06-10T21:43:02.597623+0000","last_active":"2023-06-10T21:45:24.244855+0000","last_peered":"2023-06-10T21:45:24.244855+0000","last_clean":"2023-06-10T21:42:30.112881+0000","last_became_active":"2023-06-10T21:43:02.587027+0000","last_became_peered":"2023-06-10T21:43:02.587027+0000","last_unstale":"2023-06-10T21:45:24.244855+0000","last_undegraded":"2023-06-10T21:43:02.222578+0000","last_fullsized":"2023-06-10T21:43:02.198134+0000","mapping_epoch":391,"log_start":"402'9100","ondisk_log_start":"402'9100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4304,"log_dups_size":0,"ondisk_log_size":4304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17956864,"num_objects":274,"num_object_clones":0,"num_object_copies":822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13404,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-1599}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":300,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.739556+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:45:13.739556+0000","last_peered":"2023-06-10T21:45:13.739556+0000","last_clean":"2023-06-10T21:45:13.739556+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:45:13.739556+0000","last_undegraded":"2023-06-10T21:45:13.739556+0000","last_fullsized":"2023-06-10T21:45:13.739556+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":304,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.739487+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:45:13.739487+0000","last_peered":"2023-06-10T21:45:13.739487+0000","last_clean":"2023-06-10T21:45:13.739487+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:45:13.739487+0000","last_undegraded":"2023-06-10T21:45:13.739487+0000","last_fullsized":"2023-06-10T21:45:13.739487+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":98,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745295+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:45:14.745295+0000","last_peered":"2023-06-10T21:45:14.745295+0000","last_clean":"2023-06-10T21:45:14.745295+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:45:14.745295+0000","last_undegraded":"2023-06-10T21:45:14.745295+0000","last_fullsized":"2023-06-10T21:45:14.745295+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"415'13426","reported_seq":13486,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:21.212369+0000","last_change":"2023-06-10T21:43:02.609073+0000","last_active":"2023-06-10T21:45:21.212369+0000","last_peered":"2023-06-10T21:45:21.212369+0000","last_clean":"2023-06-10T21:42:30.091083+0000","last_became_active":"2023-06-10T21:43:02.598996+0000","last_became_peered":"2023-06-10T21:43:02.598996+0000","last_unstale":"2023-06-10T21:45:21.212369+0000","last_undegraded":"2023-06-10T21:43:02.230488+0000","last_fullsized":"2023-06-10T21:43:02.196078+0000","mapping_epoch":391,"log_start":"397'8400","ondisk_log_start":"397'8400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:19:51.944781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29884416,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1964,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13426,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-1508}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":90,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744600+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:45:14.744600+0000","last_peered":"2023-06-10T21:45:14.744600+0000","last_clean":"2023-06-10T21:45:14.744600+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:45:14.744600+0000","last_undegraded":"2023-06-10T21:45:14.744600+0000","last_fullsized":"2023-06-10T21:45:14.744600+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":616,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801232+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:45:14.801232+0000","last_peered":"2023-06-10T21:45:14.801232+0000","last_clean":"2023-06-10T21:45:14.801232+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:45:14.801232+0000","last_undegraded":"2023-06-10T21:45:14.801232+0000","last_fullsized":"2023-06-10T21:45:14.801232+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747435+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:45:14.747435+0000","last_peered":"2023-06-10T21:45:14.747435+0000","last_clean":"2023-06-10T21:45:14.747435+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:45:14.747435+0000","last_undegraded":"2023-06-10T21:45:14.747435+0000","last_fullsized":"2023-06-10T21:45:14.747435+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":299,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.800069+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:45:14.800069+0000","last_peered":"2023-06-10T21:45:14.800069+0000","last_clean":"2023-06-10T21:45:14.800069+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:45:14.800069+0000","last_undegraded":"2023-06-10T21:45:14.800069+0000","last_fullsized":"2023-06-10T21:45:14.800069+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"416'13336","reported_seq":17250,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-06-10T21:45:24.204083+0000","last_change":"2023-06-10T21:43:55.274866+0000","last_active":"2023-06-10T21:45:24.204083+0000","last_peered":"2023-06-10T21:45:24.204083+0000","last_clean":"2023-06-10T21:45:24.204083+0000","last_became_active":"2023-06-10T21:43:55.259895+0000","last_became_peered":"2023-06-10T21:43:55.259895+0000","last_unstale":"2023-06-10T21:45:24.204083+0000","last_undegraded":"2023-06-10T21:45:24.204083+0000","last_fullsized":"2023-06-10T21:45:24.204083+0000","mapping_epoch":401,"log_start":"402'9000","ondisk_log_start":"402'9000","created":373,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4336,"log_dups_size":0,"ondisk_log_size":4336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17694720,"num_objects":270,"num_object_clones":0,"num_object_copies":810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13336,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1919,"num_bytes_recovered":125763584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":468,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.800394+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:45:14.800394+0000","last_peered":"2023-06-10T21:45:14.800394+0000","last_clean":"2023-06-10T21:45:14.800394+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:45:14.800394+0000","last_undegraded":"2023-06-10T21:45:14.800394+0000","last_fullsized":"2023-06-10T21:45:14.800394+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746392+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:45:14.746392+0000","last_peered":"2023-06-10T21:45:14.746392+0000","last_clean":"2023-06-10T21:45:14.746392+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:45:14.746392+0000","last_undegraded":"2023-06-10T21:45:14.746392+0000","last_fullsized":"2023-06-10T21:45:14.746392+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":166,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.741282+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:45:13.741282+0000","last_peered":"2023-06-10T21:45:13.741282+0000","last_clean":"2023-06-10T21:45:13.741282+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:45:13.741282+0000","last_undegraded":"2023-06-10T21:45:13.741282+0000","last_fullsized":"2023-06-10T21:45:13.741282+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"415'13340","reported_seq":13392,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:21.211609+0000","last_change":"2023-06-10T21:41:28.176198+0000","last_active":"2023-06-10T21:45:21.211609+0000","last_peered":"2023-06-10T21:45:21.211609+0000","last_clean":"2023-06-10T21:45:21.211609+0000","last_became_active":"2023-06-10T21:41:28.175901+0000","last_became_peered":"2023-06-10T21:41:28.175901+0000","last_unstale":"2023-06-10T21:45:21.211609+0000","last_undegraded":"2023-06-10T21:45:21.211609+0000","last_fullsized":"2023-06-10T21:45:21.211609+0000","mapping_epoch":373,"log_start":"397'8300","ondisk_log_start":"397'8300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5040,"log_dups_size":0,"ondisk_log_size":5040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:51:00.605304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13340,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":447,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.799996+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:45:14.799996+0000","last_peered":"2023-06-10T21:45:14.799996+0000","last_clean":"2023-06-10T21:45:14.799996+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:45:14.799996+0000","last_undegraded":"2023-06-10T21:45:14.799996+0000","last_fullsized":"2023-06-10T21:45:14.799996+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":295,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.742029+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:45:13.742029+0000","last_peered":"2023-06-10T21:45:13.742029+0000","last_clean":"2023-06-10T21:45:13.742029+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:45:13.742029+0000","last_undegraded":"2023-06-10T21:45:13.742029+0000","last_fullsized":"2023-06-10T21:45:13.742029+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":220,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744431+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:45:14.744431+0000","last_peered":"2023-06-10T21:45:14.744431+0000","last_clean":"2023-06-10T21:45:14.744431+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:45:14.744431+0000","last_undegraded":"2023-06-10T21:45:14.744431+0000","last_fullsized":"2023-06-10T21:45:14.744431+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"416'13848","reported_seq":17874,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-06-10T21:45:24.784506+0000","last_change":"2023-06-10T21:45:14.796190+0000","last_active":"2023-06-10T21:45:24.784506+0000","last_peered":"2023-06-10T21:45:24.784506+0000","last_clean":"2023-06-10T21:45:24.784506+0000","last_became_active":"2023-06-10T21:45:14.749662+0000","last_became_peered":"2023-06-10T21:45:14.749662+0000","last_unstale":"2023-06-10T21:45:24.784506+0000","last_undegraded":"2023-06-10T21:45:24.784506+0000","last_fullsized":"2023-06-10T21:45:24.784506+0000","mapping_epoch":414,"log_start":"401'9278","ondisk_log_start":"401'9278","created":373,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":4570,"log_dups_size":0,"ondisk_log_size":4570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121856,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13848,"num_write_kb":451008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1976,"num_bytes_recovered":129499136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":426,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.748286+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:45:14.748286+0000","last_peered":"2023-06-10T21:45:14.748286+0000","last_clean":"2023-06-10T21:45:14.748286+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:45:14.748286+0000","last_undegraded":"2023-06-10T21:45:14.748286+0000","last_fullsized":"2023-06-10T21:45:14.748286+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":237,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747982+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:45:14.747982+0000","last_peered":"2023-06-10T21:45:14.747982+0000","last_clean":"2023-06-10T21:45:14.747982+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:45:14.747982+0000","last_undegraded":"2023-06-10T21:45:14.747982+0000","last_fullsized":"2023-06-10T21:45:14.747982+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":85,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.746887+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:45:14.746887+0000","last_peered":"2023-06-10T21:45:14.746887+0000","last_clean":"2023-06-10T21:45:14.746887+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:45:14.746887+0000","last_undegraded":"2023-06-10T21:45:14.746887+0000","last_fullsized":"2023-06-10T21:45:14.746887+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"415'13370","reported_seq":13437,"reported_epoch":415,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-10T21:45:21.233595+0000","last_change":"2023-06-10T21:43:02.603811+0000","last_active":"2023-06-10T21:45:21.233595+0000","last_peered":"2023-06-10T21:45:21.233595+0000","last_clean":"2023-06-10T21:42:30.251787+0000","last_became_active":"2023-06-10T21:43:02.594099+0000","last_became_peered":"2023-06-10T21:43:02.594099+0000","last_unstale":"2023-06-10T21:45:21.233595+0000","last_undegraded":"2023-06-10T21:43:02.235463+0000","last_fullsized":"2023-06-10T21:43:02.199614+0000","mapping_epoch":391,"log_start":"396'8300","ondisk_log_start":"396'8300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:41:27.150417+0000","last_clean_scrub_stamp":"2023-06-10T21:41:27.150417+0000","objects_scrubbed":0,"log_size":5070,"log_dups_size":0,"ondisk_log_size":5070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:12:57.304964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29884416,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1921,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13370,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-1465}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":164,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801325+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:45:14.801325+0000","last_peered":"2023-06-10T21:45:14.801325+0000","last_clean":"2023-06-10T21:45:14.801325+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:45:14.801325+0000","last_undegraded":"2023-06-10T21:45:14.801325+0000","last_fullsized":"2023-06-10T21:45:14.801325+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":278,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.801641+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:45:14.801641+0000","last_peered":"2023-06-10T21:45:14.801641+0000","last_clean":"2023-06-10T21:45:14.801641+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:45:14.801641+0000","last_undegraded":"2023-06-10T21:45:14.801641+0000","last_fullsized":"2023-06-10T21:45:14.801641+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744342+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:45:14.744342+0000","last_peered":"2023-06-10T21:45:14.744342+0000","last_clean":"2023-06-10T21:45:14.744342+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:45:14.744342+0000","last_undegraded":"2023-06-10T21:45:14.744342+0000","last_fullsized":"2023-06-10T21:45:14.744342+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":115,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.800182+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:45:14.800182+0000","last_peered":"2023-06-10T21:45:14.800182+0000","last_clean":"2023-06-10T21:45:14.800182+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:45:14.800182+0000","last_undegraded":"2023-06-10T21:45:14.800182+0000","last_fullsized":"2023-06-10T21:45:14.800182+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":311,"reported_epoch":414,"state":"active+clean","last_fresh":"2023-06-10T21:45:13.742048+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:45:13.742048+0000","last_peered":"2023-06-10T21:45:13.742048+0000","last_clean":"2023-06-10T21:45:13.742048+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:45:13.742048+0000","last_undegraded":"2023-06-10T21:45:13.742048+0000","last_fullsized":"2023-06-10T21:45:13.742048+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":78,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744397+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:45:14.744397+0000","last_peered":"2023-06-10T21:45:14.744397+0000","last_clean":"2023-06-10T21:45:14.744397+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:45:14.744397+0000","last_undegraded":"2023-06-10T21:45:14.744397+0000","last_fullsized":"2023-06-10T21:45:14.744397+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":320,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747388+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:45:14.747388+0000","last_peered":"2023-06-10T21:45:14.747388+0000","last_clean":"2023-06-10T21:45:14.747388+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:45:14.747388+0000","last_undegraded":"2023-06-10T21:45:14.747388+0000","last_fullsized":"2023-06-10T21:45:14.747388+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":101,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.744590+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:45:14.744590+0000","last_peered":"2023-06-10T21:45:14.744590+0000","last_clean":"2023-06-10T21:45:14.744590+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:45:14.744590+0000","last_undegraded":"2023-06-10T21:45:14.744590+0000","last_fullsized":"2023-06-10T21:45:14.744590+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":391,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747660+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:45:14.747660+0000","last_peered":"2023-06-10T21:45:14.747660+0000","last_clean":"2023-06-10T21:45:14.747660+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:45:14.747660+0000","last_undegraded":"2023-06-10T21:45:14.747660+0000","last_fullsized":"2023-06-10T21:45:14.747660+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":70,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.747909+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:45:14.747909+0000","last_peered":"2023-06-10T21:45:14.747909+0000","last_clean":"2023-06-10T21:45:14.747909+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:45:14.747909+0000","last_undegraded":"2023-06-10T21:45:14.747909+0000","last_fullsized":"2023-06-10T21:45:14.747909+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":85,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-06-10T21:45:24.197424+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:45:24.197424+0000","last_peered":"2023-06-10T21:45:24.197424+0000","last_clean":"2023-06-10T21:45:24.197424+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:45:24.197424+0000","last_undegraded":"2023-06-10T21:45:24.197424+0000","last_fullsized":"2023-06-10T21:45:24.197424+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":337,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-06-10T21:45:14.745461+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:45:14.745461+0000","last_peered":"2023-06-10T21:45:14.745461+0000","last_clean":"2023-06-10T21:45:14.745461+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:45:14.745461+0000","last_undegraded":"2023-06-10T21:45:14.745461+0000","last_fullsized":"2023-06-10T21:45:14.745461+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":85,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-06-10T21:45:24.196632+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:45:24.196632+0000","last_peered":"2023-06-10T21:45:24.196632+0000","last_clean":"2023-06-10T21:45:24.196632+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:45:24.196632+0000","last_undegraded":"2023-06-10T21:45:24.196632+0000","last_fullsized":"2023-06-10T21:45:24.196632+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":344391704,"num_objects":5256,"num_object_clones":0,"num_object_copies":15768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16344,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5256,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":214744,"num_write_kb":7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10487,"num_bytes_recovered":670367744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":856895488,"data_stored":856883272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75049,"ondisk_log_size":75049,"up":48,"acting":39,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430780,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191528,"kb_used_data":209184,"kb_used_omap":0,"kb_used_meta":982272,"kb_avail":92553624,"statfs":{"total":95995035648,"available":94774910976,"internally_reserved":0,"allocated":214204416,"data_stored":213495219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1005846528},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,5,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.272,"15min":2.5510000000000002},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":8.5129999999999999,"5min":22.277000000000001,"15min":184.83699999999999},"last":3.302},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.5820000000000001,"15min":2.8540000000000001},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":20.568999999999999,"5min":22.462,"15min":185.58000000000001},"last":3.7360000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.8100000000000001,"15min":2.6560000000000001},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.1500000000000004,"5min":38.078000000000003,"15min":87.417000000000002},"last":3.3290000000000002},{"interface":"front","average":{"1min":2.3119999999999998,"5min":2.2509999999999999,"15min":2.9590000000000001},"min":{"1min":0.30299999999999999,"5min":0.28599999999999998,"15min":0.27300000000000002},"max":{"1min":10.484,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:44:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.1299999999999999,"15min":4.9729999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":36.627000000000002,"5min":75.219999999999999,"15min":172.33600000000001},"last":3.7040000000000002},{"interface":"front","average":{"1min":4.4880000000000004,"5min":3.1379999999999999,"15min":4.9610000000000003},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":36.533999999999999,"5min":75.108999999999995,"15min":172.185},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:44:32 2023","interfaces":[{"interface":"back","average":{"1min":7.7990000000000004,"5min":4.0899999999999999,"15min":4.7960000000000003},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":71.852000000000004,"5min":89.504999999999995,"15min":147.00700000000001},"last":3.5089999999999999},{"interface":"front","average":{"1min":7.2359999999999998,"5min":3.7599999999999998,"15min":4.5270000000000001},"min":{"1min":0.439,"5min":0.35899999999999999,"15min":0.308},"max":{"1min":72.813999999999993,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.51400000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.1080000000000001,"15min":0.66600000000000004},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":4.8479999999999999,"5min":13.429,"15min":13.429},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.645,"5min":1.266,"15min":0.82399999999999995},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":12.279999999999999,"5min":12.279999999999999,"15min":22.77},"last":3.9580000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:45:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":10.191000000000001,"15min":11.515000000000001},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.984,"5min":131.619,"15min":131.619},"last":0.65700000000000003},{"interface":"front","average":{"1min":2.036,"5min":10.006,"15min":11.334},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.9370000000000003,"5min":134.928,"15min":134.928},"last":3.665}]},{"osd":6,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.0189999999999999,"15min":0.74399999999999999},"min":{"1min":0.371,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":10.890000000000001},"last":3.5569999999999999},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.2210000000000001,"15min":0.76600000000000001},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.55000000000000004}]}]},{"osd":6,"up_from":0,"seq":180388627008,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566451,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020427,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075838,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360040,"kb_used_data":334880,"kb_used_omap":7,"kb_used_meta":1025144,"kb_avail":92385112,"statfs":{"total":95995035648,"available":94602354688,"internally_reserved":0,"allocated":342917120,"data_stored":341571543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1049747537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":2.375,"15min":4.2670000000000003},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":6.6429999999999998,"5min":83.733000000000004,"15min":171.33699999999999},"last":5.8849999999999998},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.9690000000000001,"15min":4.0640000000000001},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9300000000000002,"5min":82.322999999999993,"15min":171.56200000000001},"last":6.415}]},{"osd":1,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.653,"15min":5.3120000000000003},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.2370000000000001,"5min":85.709999999999994,"15min":171.286},"last":11.398999999999999},{"interface":"front","average":{"1min":1.542,"5min":3.548,"15min":3.891},"min":{"1min":0.27800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1379999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":5.9119999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:44:25 2023","interfaces":[{"interface":"back","average":{"1min":11.705,"5min":8.6259999999999994,"15min":6.6699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":100.786,"5min":145.07599999999999,"15min":171.78299999999999},"last":5.9370000000000003},{"interface":"front","average":{"1min":11.17,"5min":7.9969999999999999,"15min":6.6219999999999999},"min":{"1min":0.36099999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":100.73999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":6.6239999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.266,"15min":2.3130000000000002},"min":{"1min":0.33500000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.9629999999999992,"5min":8.9629999999999992,"15min":171.48699999999999},"last":6.3259999999999996},{"interface":"front","average":{"1min":1.903,"5min":1.7669999999999999,"15min":3.2370000000000001},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.2539999999999996,"5min":14.858000000000001,"15min":171.18100000000001},"last":2.7999999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.9989999999999997,"15min":6.6539999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.0389999999999997,"5min":75.896000000000001,"15min":75.896000000000001},"last":5.8259999999999996},{"interface":"front","average":{"1min":1.976,"5min":5.468,"15min":6.0499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":7.3460000000000001,"5min":76.337000000000003,"15min":76.337000000000003},"last":6.282}]},{"osd":6,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.476,"15min":2.4300000000000002},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.0139999999999993,"5min":9.0139999999999993,"15min":171.441},"last":6.3879999999999999},{"interface":"front","average":{"1min":1.948,"5min":1.931,"15min":3.3029999999999999},"min":{"1min":0.51900000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.9199999999999999,"5min":14.893000000000001,"15min":171.24600000000001},"last":3.1819999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.9430000000000001,"15min":4.0979999999999999},"min":{"1min":0.503,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":219.131},"last":5.8570000000000002},{"interface":"front","average":{"1min":2.9279999999999999,"5min":1.585,"15min":4.0250000000000004},"min":{"1min":0.48099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":12.15,"5min":12.15,"15min":219.19},"last":6.3570000000000002}]}]},{"osd":3,"up_from":250,"seq":1073741824199,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861240,"kb_used_data":123184,"kb_used_omap":7,"kb_used_meta":737848,"kb_avail":92883912,"statfs":{"total":95995035648,"available":95113125888,"internally_reserved":0,"allocated":126140416,"data_stored":124666323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":755556371},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.6319999999999999,"15min":3.2730000000000001},"min":{"1min":0.22500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":3.5880000000000001,"5min":30.744,"15min":105.09},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.375,"5min":2.0739999999999998,"15min":3.3740000000000001},"min":{"1min":0.26100000000000001,"5min":0.23999999999999999,"15min":0.20799999999999999},"max":{"1min":4.4450000000000003,"5min":30.805,"15min":99.852999999999994},"last":2.0950000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":1.476,"5min":1.8280000000000001,"15min":3.3660000000000001},"min":{"1min":0.315,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":4.5300000000000002,"5min":13.249000000000001,"15min":94.766000000000005},"last":3.2240000000000002},{"interface":"front","average":{"1min":1.375,"5min":1.589,"15min":3.2789999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":5.4109999999999996,"5min":11.846,"15min":94.691000000000003},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.6499999999999999,"15min":4.319},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.9649999999999999,"5min":18.452000000000002,"15min":231.07900000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":2.1000000000000001,"5min":3.0259999999999998,"15min":5.5599999999999996},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":9.3930000000000007,"5min":18.733000000000001,"15min":231.58600000000001},"last":8.9559999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":1.5900000000000001,"15min":1.486},"min":{"1min":0.45600000000000002,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.8470000000000004,"5min":9.5150000000000006,"15min":13.417},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.843,"5min":1.8300000000000001,"15min":2.2599999999999998},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":4.5049999999999999,"5min":10.722,"15min":19.728000000000002},"last":8.8379999999999992}]},{"osd":5,"last update":"Sat Jun 10 21:45:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.097,"15min":1.0920000000000001},"min":{"1min":0.37,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.6269999999999998,"5min":6.532,"15min":6.532},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.6160000000000001,"15min":2.6739999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.1130000000000004,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":5.657}]},{"osd":6,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.903,"15min":2.323},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8519999999999999,"5min":10.691000000000001,"15min":19.805},"last":9.0250000000000004},{"interface":"front","average":{"1min":2.0249999999999999,"5min":1.484,"15min":1.5529999999999999},"min":{"1min":0.48199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.6299999999999999,"5min":9.9719999999999995,"15min":17.327000000000002},"last":0.52900000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:44:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.1179999999999999,"15min":2.601},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":5.4109999999999996,"5min":10.648999999999999,"15min":79.984999999999999},"last":8.9920000000000009},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.603,"15min":1.847},"min":{"1min":0.51300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":5.9320000000000004,"5min":9.9380000000000006,"15min":80.272999999999996},"last":2.609}]}]},{"osd":4,"up_from":0,"seq":128849019464,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245470,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":442216,"kb_used_data":202784,"kb_used_omap":11,"kb_used_meta":239348,"kb_avail":93302936,"statfs":{"total":95995035648,"available":95542206464,"internally_reserved":0,"allocated":207650816,"data_stored":206949209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":245092854},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,1,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.0579999999999998,"15min":2.0720000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":1.095},{"interface":"front","average":{"1min":1.5209999999999999,"5min":1.361,"15min":1.335},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3170000000000002,"5min":4.266,"15min":4.266},"last":0.48999999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.6269999999999998,"15min":2.6480000000000001},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":1.026},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2229999999999999,"15min":2.262},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.9500000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":9.5090000000000003,"5min":3.391,"15min":2.371},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":159.94999999999999,"5min":159.94999999999999,"15min":159.94999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":9.3230000000000004,"5min":3.7090000000000001,"15min":2.7730000000000001},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":159.851,"5min":159.851,"15min":159.851},"last":2.391}]},{"osd":3,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":7.569,"15min":8.0109999999999992},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":54.920000000000002,"5min":133.07499999999999,"15min":133.07499999999999},"last":1.117},{"interface":"front","average":{"1min":5.3339999999999996,"5min":7.3819999999999997,"15min":7.7240000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":54.848999999999997,"5min":133.03800000000001,"15min":133.03800000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.96199999999999997,"15min":0.92600000000000005},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.4329999999999998,"5min":5.173,"15min":5.173},"last":2.4359999999999999},{"interface":"front","average":{"1min":1.3180000000000001,"5min":2.1099999999999999,"15min":2.242},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.238,"5min":7.476,"15min":7.476},"last":1.0740000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.752,"15min":1.8420000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.2149999999999999,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.173,"5min":1.3100000000000001,"15min":1.333},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":4.2910000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":2.4620000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.625,"15min":1.6160000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.2430000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":2.4870000000000001},{"interface":"front","average":{"1min":2.9630000000000001,"5min":2.9049999999999998,"15min":2.895},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.1920000000000002,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":3.2869999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":335085568,"data_stored":335085568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":119803904,"data_stored":119799832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":203689984,"data_stored":203685912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":198316032,"data_stored":198311960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:45:29.488 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph status --format=json 2023-06-10T21:45:29.668 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:29.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:29.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:29.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:29.818 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:29.946 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:29.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:30.031 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:30.051 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:30.093 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:30.154 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:45:30.154 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:45:30.154 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:45:30.154 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:45:30.154 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:45:30.159 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:30.171 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:30.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:30.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:30.312 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:30.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:30 smithi064 ceph-mon[186706]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 78 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 868 op/s; 16344/3810 objects degraded (428.976%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:30.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:30 smithi064 ceph-mon[180297]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 78 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 868 op/s; 16344/3810 objects degraded (428.976%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:30.426 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:30.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:30.530 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:30.565 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:30.570 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:30.594 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:30.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:30.637 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:30.658 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:30.676 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:30.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:30 smithi152 ceph-mon[156819]: pgmap v1540: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 78 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 868 op/s; 16344/3810 objects degraded (428.976%); 2.5 MiB/s, 65 objects/s recovering 2023-06-10T21:45:30.694 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:30.715 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:30.716 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:45:30.717 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:45:31.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2945295628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:31.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 16344/3810 objects degraded (428.976%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:31.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[180297]: osdmap e417: 8 total, 8 up, 4 in 2023-06-10T21:45:31.603 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[180297]: failed to encode map e417 with expected crc 2023-06-10T21:45:31.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2945295628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:31.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 16344/3810 objects degraded (428.976%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:31.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[186706]: osdmap e417: 8 total, 8 up, 4 in 2023-06-10T21:45:31.604 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:31 smithi064 ceph-mon[186706]: failed to encode map e417 with expected crc 2023-06-10T21:45:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2945295628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:31 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 16344/3810 objects degraded (428.976%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:31 smithi152 ceph-mon[156819]: osdmap e417: 8 total, 8 up, 4 in 2023-06-10T21:45:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:31 smithi152 ceph-mon[156819]: failed to encode map e417 with expected crc 2023-06-10T21:45:31.847 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:45:32.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[186706]: failed to encode map e417 with expected crc 2023-06-10T21:45:32.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[186706]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 959 op/s; 15421/72 objects degraded (21418.056%); 741 KiB/s, 158 objects/s recovering 2023-06-10T21:45:32.314 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[186706]: osdmap e418: 8 total, 8 up, 4 in 2023-06-10T21:45:32.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[180297]: failed to encode map e417 with expected crc 2023-06-10T21:45:32.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[180297]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 959 op/s; 15421/72 objects degraded (21418.056%); 741 KiB/s, 158 objects/s recovering 2023-06-10T21:45:32.315 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:32 smithi064 ceph-mon[180297]: osdmap e418: 8 total, 8 up, 4 in 2023-06-10T21:45:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:32 smithi152 ceph-mon[156819]: failed to encode map e417 with expected crc 2023-06-10T21:45:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:32 smithi152 ceph-mon[156819]: pgmap v1542: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 959 op/s; 15421/72 objects degraded (21418.056%); 741 KiB/s, 158 objects/s recovering 2023-06-10T21:45:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:32 smithi152 ceph-mon[156819]: osdmap e418: 8 total, 8 up, 4 in 2023-06-10T21:45:32.981 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:32.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:32.998 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:33.087 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:33.113 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:33.134 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:32] "GET /metrics HTTP/1.1" 200 224091 "" "Prometheus/2.33.4" 2023-06-10T21:45:33.162 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:33.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:33.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:33.293 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:33.336 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:33.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:33.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[186706]: failed to encode map e417 with expected crc 2023-06-10T21:45:33.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[186706]: failed to encode map e418 with expected crc 2023-06-10T21:45:33.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:33.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[180297]: failed to encode map e417 with expected crc 2023-06-10T21:45:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[180297]: failed to encode map e418 with expected crc 2023-06-10T21:45:33.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:33 smithi152 ceph-mon[156819]: failed to encode map e417 with expected crc 2023-06-10T21:45:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:33 smithi152 ceph-mon[156819]: failed to encode map e418 with expected crc 2023-06-10T21:45:33.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:33.444 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:33.470 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:33.514 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:33.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:33.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:33.621 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:33.651 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:33.734 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:33.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:33.840 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:33.880 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:34.131 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:34.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:34.235 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:34.261 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:34.303 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:34.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:34.320 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:34 smithi064 ceph-mon[180297]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 15421/72 objects degraded (21418.056%); 0 B/s, 115 objects/s recovering 2023-06-10T21:45:34.321 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:34 smithi064 ceph-mon[186706]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 15421/72 objects degraded (21418.056%); 0 B/s, 115 objects/s recovering 2023-06-10T21:45:34.412 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:34.430 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:34.438 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:34.482 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:34.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:34.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:34.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:34.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:34.665 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:34.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:34.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:34.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:34 smithi152 ceph-mon[156819]: pgmap v1544: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 656 op/s; 15421/72 objects degraded (21418.056%); 0 B/s, 115 objects/s recovering 2023-06-10T21:45:34.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:34.802 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:34.846 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:34.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:34.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:34.978 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:35.019 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:35.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:35.126 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:35.154 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:35.304 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:35.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:35.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:35.333 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_8' removed 2023-06-10T21:45:35.334 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:45:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:45:35.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:45:35.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:45:35.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:45:35.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:35.461 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:35.537 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:35.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:35.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:35.672 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:45:35.673 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:45:35.674 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:35.716 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:35.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:35.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:45:35.823 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:35.849 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:35.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:35.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:35.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:35.996 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:36.025 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:36.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:36.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.324 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: osdmap e419: 8 total, 8 up, 4 in 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[186706]: Health check update: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:36.325 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: osdmap e419: 8 total, 8 up, 4 in 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:36.326 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:36 smithi064 ceph-mon[180297]: Health check update: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:36.376 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:36.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:36.481 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:36.511 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:36.512 INFO:teuthology.orchestra.run.smithi064.stdout:{"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2209,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":419,"num_osds":8,"num_up_osds":8,"osd_up_since":1686433380,"num_in_osds":4,"osd_in_since":1686433189,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":73}],"num_pgs":73,"num_pools":4,"num_objects":24,"data_bytes":595854,"bytes_used":3088048128,"bytes_avail":380892094464,"bytes_total":383980142592},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi064.wuuunt","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.64:8443/","prometheus":"http://172.21.15.64:9283/"}},"servicemap":{"epoch":30,"modified":"2023-06-10T21:44:56.203656+0000","services":{"osd":{"daemons":{"summary":"","1":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"6912257d-c37f-4420-ab14-fb7b8b3dc2a8":{"message":"Upgrade to 18.0.0-3603-g5937c68a (12m)\n [============................] (remaining: 15m)","progress":0.44999998807907104,"add_to_ceph_s":true},"9ab7ee0e-4ecd-48a4-8ef6-12d592a043b5":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 25s)","progress":0.8764045238494873,"add_to_ceph_s":true}}} 2023-06-10T21:45:36.512 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:36.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:36.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:36.670 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: failed to encode map e419 with expected crc 2023-06-10T21:45:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1039453029' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:45:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2942754261' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:45:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: osdmap e419: 8 total, 8 up, 4 in 2023-06-10T21:45:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:36 smithi152 ceph-mon[156819]: Health check update: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-10T21:45:36.698 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:36.743 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:36.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:36.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:36.851 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:36.878 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:36.922 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:36.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:37.029 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:37.055 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:37.096 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:37.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:37.201 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:37.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:37.322 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:37 smithi064 ceph-mon[180297]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized) 2023-06-10T21:45:37.323 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/941021080' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:37.563 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-06-10T21:45:37.563 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:45:37.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:37 smithi064 ceph-mon[186706]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized) 2023-06-10T21:45:37.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/941021080' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:37.622 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-10T21:45:37.624 INFO:tasks.radosbench:Beginning radosbench... 2023-06-10T21:45:37.624 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-06-10T21:45:37.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool create unique_pool_9 16 2023-06-10T21:45:37.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:37.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:37 smithi152 ceph-mon[156819]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15421/72 objects degraded (21418.056%), 9 pgs degraded, 9 pgs undersized) 2023-06-10T21:45:37.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/941021080' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-10T21:45:37.738 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:37.772 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:37.834 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:37.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:37.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:37.954 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:37.974 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:38.040 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:38.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:38.075 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:45:38.076 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:45:38.146 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:38.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:38.357 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:45:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:38 smithi064 ceph-mon[180297]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:38.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:38 smithi064 ceph-mon[186706]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:38.679 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:38.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:38.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:38 smithi152 ceph-mon[156819]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:38.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:38.816 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:45:38.817 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:38.864 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:38.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:38.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:38.918 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:38.919 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:38.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:39.006 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:39.203 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:39.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:39.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:39.309 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:39.336 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:39.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:40.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:40.051 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:40.646 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:40.651 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:40.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:40 smithi064 ceph-mon[186706]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:40.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:40 smithi064 ceph-mon[180297]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:40.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:40 smithi152 ceph-mon[156819]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:40.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:40.783 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:40.827 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:40.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:40.935 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:40.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:41.114 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:41.115 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:41.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:41.138 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:41.164 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:41.184 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:41.202 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:41.221 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:41.222 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:41.242 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:41.253 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:41.265 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:41.266 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.267 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:45:41.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:41.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:41.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:41.482 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2385300363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:41.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2385300363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:41.492 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2385300363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:42.578 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:42 smithi064 ceph-mon[180297]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:42.578 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:42 smithi064 ceph-mon[186706]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:42.650 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:42.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:42.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:42 smithi152 ceph-mon[156819]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:42.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:42.788 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:42.838 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:42.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:42.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:42.906 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:42] "GET /metrics HTTP/1.1" 200 224009 "" "Prometheus/2.33.4" 2023-06-10T21:45:42.942 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:42.976 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:43.063 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:43.095 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:43.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:43.199 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:43.232 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:43.338 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:43.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:43.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:43.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-10T21:45:43.480 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3863484470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:43.483 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:45:43.487 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_9' created 2023-06-10T21:45:43.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.751 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:43.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-10T21:45:43.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3863484470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.752 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:43 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:43.776 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:43.778 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:45:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-10T21:45:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3863484470' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:43 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:45:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:43 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:45:44.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:44.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:44.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:44.394 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:44.426 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:44.477 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:44.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:44.501 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:45:44.586 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: osdmap e420: 8 total, 8 up, 4 in 2023-06-10T21:45:44.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: from='client.47129 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:44.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:44.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: pgmap v1551: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:44.612 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[186706]: osdmap e421: 8 total, 8 up, 4 in 2023-06-10T21:45:44.612 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: osdmap e420: 8 total, 8 up, 4 in 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: from='client.47129 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:44.613 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: pgmap v1551: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:44.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:44 smithi064 ceph-mon[180297]: osdmap e421: 8 total, 8 up, 4 in 2023-06-10T21:45:44.614 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1048204699' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: osdmap e420: 8 total, 8 up, 4 in 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: failed to encode map e420 with expected crc 2023-06-10T21:45:44.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: from='client.47129 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:45:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: pgmap v1551: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:44.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:44 smithi152 ceph-mon[156819]: osdmap e421: 8 total, 8 up, 4 in 2023-06-10T21:45:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[186706]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[186706]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[186706]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[186706]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[186706]: osdmap e422: 8 total, 8 up, 4 in 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[180297]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[180297]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[180297]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[180297]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:45 smithi064 ceph-mon[180297]: osdmap e422: 8 total, 8 up, 4 in 2023-06-10T21:45:45.780 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:45.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:45.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:45.841 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-06-10T21:45:45.883 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1550,"stamp":"2023-06-10T21:45:43.462595+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":2377742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":230,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3022128,"kb_used_data":18128,"kb_used_omap":27,"kb_used_meta":3003940,"kb_avail":371958480,"statfs":{"total":383980142592,"available":380885483520,"internally_reserved":0,"allocated":18563072,"data_stored":14312853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3076035162},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":46,"apply_latency_ms":46,"commit_latency_ns":46000000,"apply_latency_ns":46000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-15421,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-220000,"num_write_kb":-7039937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-11410,"num_bytes_recovered":-670367744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.239623"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":458,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743105+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:45:32.743105+0000","last_peered":"2023-06-10T21:45:32.743105+0000","last_clean":"2023-06-10T21:45:32.743105+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:45:32.743105+0000","last_undegraded":"2023-06-10T21:45:32.743105+0000","last_fullsized":"2023-06-10T21:45:32.743105+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":88,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-06-10T21:45:35.743132+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:45:35.743132+0000","last_peered":"2023-06-10T21:45:35.743132+0000","last_clean":"2023-06-10T21:45:35.743132+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:45:35.743132+0000","last_undegraded":"2023-06-10T21:45:35.743132+0000","last_fullsized":"2023-06-10T21:45:35.743132+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":350,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253380+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:45:32.253380+0000","last_peered":"2023-06-10T21:45:32.253380+0000","last_clean":"2023-06-10T21:45:32.253380+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:45:32.253380+0000","last_undegraded":"2023-06-10T21:45:32.253380+0000","last_fullsized":"2023-06-10T21:45:32.253380+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":304,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268482+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:45:32.268482+0000","last_peered":"2023-06-10T21:45:32.268482+0000","last_clean":"2023-06-10T21:45:32.268482+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:45:32.268482+0000","last_undegraded":"2023-06-10T21:45:32.268482+0000","last_fullsized":"2023-06-10T21:45:32.268482+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.252984+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:45:32.252984+0000","last_peered":"2023-06-10T21:45:32.252984+0000","last_clean":"2023-06-10T21:45:32.252984+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:45:32.252984+0000","last_undegraded":"2023-06-10T21:45:32.252984+0000","last_fullsized":"2023-06-10T21:45:32.252984+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253582+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:45:32.253582+0000","last_peered":"2023-06-10T21:45:32.253582+0000","last_clean":"2023-06-10T21:45:32.253582+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:45:32.253582+0000","last_undegraded":"2023-06-10T21:45:32.253582+0000","last_fullsized":"2023-06-10T21:45:32.253582+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":284,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253445+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:45:32.253445+0000","last_peered":"2023-06-10T21:45:32.253445+0000","last_clean":"2023-06-10T21:45:32.253445+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:45:32.253445+0000","last_undegraded":"2023-06-10T21:45:32.253445+0000","last_fullsized":"2023-06-10T21:45:32.253445+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253056+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:45:32.253056+0000","last_peered":"2023-06-10T21:45:32.253056+0000","last_clean":"2023-06-10T21:45:32.253056+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:45:32.253056+0000","last_undegraded":"2023-06-10T21:45:32.253056+0000","last_fullsized":"2023-06-10T21:45:32.253056+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253955+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:45:32.253955+0000","last_peered":"2023-06-10T21:45:32.253955+0000","last_clean":"2023-06-10T21:45:32.253955+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:45:32.253955+0000","last_undegraded":"2023-06-10T21:45:32.253955+0000","last_fullsized":"2023-06-10T21:45:32.253955+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253385+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:45:32.253385+0000","last_peered":"2023-06-10T21:45:32.253385+0000","last_clean":"2023-06-10T21:45:32.253385+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:45:32.253385+0000","last_undegraded":"2023-06-10T21:45:32.253385+0000","last_fullsized":"2023-06-10T21:45:32.253385+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":176,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268723+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:45:32.268723+0000","last_peered":"2023-06-10T21:45:32.268723+0000","last_clean":"2023-06-10T21:45:32.268723+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:45:32.268723+0000","last_undegraded":"2023-06-10T21:45:32.268723+0000","last_fullsized":"2023-06-10T21:45:32.268723+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":419,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743100+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:45:32.743100+0000","last_peered":"2023-06-10T21:45:32.743100+0000","last_clean":"2023-06-10T21:45:32.743100+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:45:32.743100+0000","last_undegraded":"2023-06-10T21:45:32.743100+0000","last_fullsized":"2023-06-10T21:45:32.743100+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":342,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268215+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:45:32.268215+0000","last_peered":"2023-06-10T21:45:32.268215+0000","last_clean":"2023-06-10T21:45:32.268215+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:45:32.268215+0000","last_undegraded":"2023-06-10T21:45:32.268215+0000","last_fullsized":"2023-06-10T21:45:32.268215+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":133,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253464+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:45:32.253464+0000","last_peered":"2023-06-10T21:45:32.253464+0000","last_clean":"2023-06-10T21:45:32.253464+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:45:32.253464+0000","last_undegraded":"2023-06-10T21:45:32.253464+0000","last_fullsized":"2023-06-10T21:45:32.253464+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":478,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268468+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:45:32.268468+0000","last_peered":"2023-06-10T21:45:32.268468+0000","last_clean":"2023-06-10T21:45:32.268468+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:45:32.268468+0000","last_undegraded":"2023-06-10T21:45:32.268468+0000","last_fullsized":"2023-06-10T21:45:32.268468+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":367,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291948+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:45:32.291948+0000","last_peered":"2023-06-10T21:45:32.291948+0000","last_clean":"2023-06-10T21:45:32.291948+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:45:32.291948+0000","last_undegraded":"2023-06-10T21:45:32.291948+0000","last_fullsized":"2023-06-10T21:45:32.291948+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":289,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268442+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:45:32.268442+0000","last_peered":"2023-06-10T21:45:32.268442+0000","last_clean":"2023-06-10T21:45:32.268442+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:45:32.268442+0000","last_undegraded":"2023-06-10T21:45:32.268442+0000","last_fullsized":"2023-06-10T21:45:32.268442+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":101,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253718+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:45:32.253718+0000","last_peered":"2023-06-10T21:45:32.253718+0000","last_clean":"2023-06-10T21:45:32.253718+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:45:32.253718+0000","last_undegraded":"2023-06-10T21:45:32.253718+0000","last_fullsized":"2023-06-10T21:45:32.253718+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":431,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268104+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:45:32.268104+0000","last_peered":"2023-06-10T21:45:32.268104+0000","last_clean":"2023-06-10T21:45:32.268104+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:45:32.268104+0000","last_undegraded":"2023-06-10T21:45:32.268104+0000","last_fullsized":"2023-06-10T21:45:32.268104+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292245+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:45:32.292245+0000","last_peered":"2023-06-10T21:45:32.292245+0000","last_clean":"2023-06-10T21:45:32.292245+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:45:32.292245+0000","last_undegraded":"2023-06-10T21:45:32.292245+0000","last_fullsized":"2023-06-10T21:45:32.292245+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":443,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253277+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:45:32.253277+0000","last_peered":"2023-06-10T21:45:32.253277+0000","last_clean":"2023-06-10T21:45:32.253277+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:45:32.253277+0000","last_undegraded":"2023-06-10T21:45:32.253277+0000","last_fullsized":"2023-06-10T21:45:32.253277+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292374+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:45:32.292374+0000","last_peered":"2023-06-10T21:45:32.292374+0000","last_clean":"2023-06-10T21:45:32.292374+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:45:32.292374+0000","last_undegraded":"2023-06-10T21:45:32.292374+0000","last_fullsized":"2023-06-10T21:45:32.292374+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":211,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.269094+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:45:32.269094+0000","last_peered":"2023-06-10T21:45:32.269094+0000","last_clean":"2023-06-10T21:45:32.269094+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:45:32.269094+0000","last_undegraded":"2023-06-10T21:45:32.269094+0000","last_fullsized":"2023-06-10T21:45:32.269094+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253125+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:45:32.253125+0000","last_peered":"2023-06-10T21:45:32.253125+0000","last_clean":"2023-06-10T21:45:32.253125+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:45:32.253125+0000","last_undegraded":"2023-06-10T21:45:32.253125+0000","last_fullsized":"2023-06-10T21:45:32.253125+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":211,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268659+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:45:32.268659+0000","last_peered":"2023-06-10T21:45:32.268659+0000","last_clean":"2023-06-10T21:45:32.268659+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:45:32.268659+0000","last_undegraded":"2023-06-10T21:45:32.268659+0000","last_fullsized":"2023-06-10T21:45:32.268659+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291878+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:45:32.291878+0000","last_peered":"2023-06-10T21:45:32.291878+0000","last_clean":"2023-06-10T21:45:32.291878+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:45:32.291878+0000","last_undegraded":"2023-06-10T21:45:32.291878+0000","last_fullsized":"2023-06-10T21:45:32.291878+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":498,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268734+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:45:32.268734+0000","last_peered":"2023-06-10T21:45:32.268734+0000","last_clean":"2023-06-10T21:45:32.268734+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:45:32.268734+0000","last_undegraded":"2023-06-10T21:45:32.268734+0000","last_fullsized":"2023-06-10T21:45:32.268734+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268309+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:45:32.268309+0000","last_peered":"2023-06-10T21:45:32.268309+0000","last_clean":"2023-06-10T21:45:32.268309+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:45:32.268309+0000","last_undegraded":"2023-06-10T21:45:32.268309+0000","last_fullsized":"2023-06-10T21:45:32.268309+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":102,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253612+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:45:32.253612+0000","last_peered":"2023-06-10T21:45:32.253612+0000","last_clean":"2023-06-10T21:45:32.253612+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:45:32.253612+0000","last_undegraded":"2023-06-10T21:45:32.253612+0000","last_fullsized":"2023-06-10T21:45:32.253612+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253420+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:45:32.253420+0000","last_peered":"2023-06-10T21:45:32.253420+0000","last_clean":"2023-06-10T21:45:32.253420+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:45:32.253420+0000","last_undegraded":"2023-06-10T21:45:32.253420+0000","last_fullsized":"2023-06-10T21:45:32.253420+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":77,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253389+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:45:32.253389+0000","last_peered":"2023-06-10T21:45:32.253389+0000","last_clean":"2023-06-10T21:45:32.253389+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:45:32.253389+0000","last_undegraded":"2023-06-10T21:45:32.253389+0000","last_fullsized":"2023-06-10T21:45:32.253389+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291867+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:45:32.291867+0000","last_peered":"2023-06-10T21:45:32.291867+0000","last_clean":"2023-06-10T21:45:32.291867+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:45:32.291867+0000","last_undegraded":"2023-06-10T21:45:32.291867+0000","last_fullsized":"2023-06-10T21:45:32.291867+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":240,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291773+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:45:32.291773+0000","last_peered":"2023-06-10T21:45:32.291773+0000","last_clean":"2023-06-10T21:45:32.291773+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:45:32.291773+0000","last_undegraded":"2023-06-10T21:45:32.291773+0000","last_fullsized":"2023-06-10T21:45:32.291773+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":457,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743687+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:45:32.743687+0000","last_peered":"2023-06-10T21:45:32.743687+0000","last_clean":"2023-06-10T21:45:32.743687+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:45:32.743687+0000","last_undegraded":"2023-06-10T21:45:32.743687+0000","last_fullsized":"2023-06-10T21:45:32.743687+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":435,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-06-10T21:45:35.745349+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:45:35.745349+0000","last_peered":"2023-06-10T21:45:35.745349+0000","last_clean":"2023-06-10T21:45:35.745349+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:45:35.745349+0000","last_undegraded":"2023-06-10T21:45:35.745349+0000","last_fullsized":"2023-06-10T21:45:35.745349+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292229+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:45:32.292229+0000","last_peered":"2023-06-10T21:45:32.292229+0000","last_clean":"2023-06-10T21:45:32.292229+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:45:32.292229+0000","last_undegraded":"2023-06-10T21:45:32.292229+0000","last_fullsized":"2023-06-10T21:45:32.292229+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253260+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:45:32.253260+0000","last_peered":"2023-06-10T21:45:32.253260+0000","last_clean":"2023-06-10T21:45:32.253260+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:45:32.253260+0000","last_undegraded":"2023-06-10T21:45:32.253260+0000","last_fullsized":"2023-06-10T21:45:32.253260+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":367,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292501+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:45:32.292501+0000","last_peered":"2023-06-10T21:45:32.292501+0000","last_clean":"2023-06-10T21:45:32.292501+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:45:32.292501+0000","last_undegraded":"2023-06-10T21:45:32.292501+0000","last_fullsized":"2023-06-10T21:45:32.292501+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":202,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268266+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:45:32.268266+0000","last_peered":"2023-06-10T21:45:32.268266+0000","last_clean":"2023-06-10T21:45:32.268266+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:45:32.268266+0000","last_undegraded":"2023-06-10T21:45:32.268266+0000","last_fullsized":"2023-06-10T21:45:32.268266+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":83,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253309+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:45:32.253309+0000","last_peered":"2023-06-10T21:45:32.253309+0000","last_clean":"2023-06-10T21:45:32.253309+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:45:32.253309+0000","last_undegraded":"2023-06-10T21:45:32.253309+0000","last_fullsized":"2023-06-10T21:45:32.253309+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253296+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:45:32.253296+0000","last_peered":"2023-06-10T21:45:32.253296+0000","last_clean":"2023-06-10T21:45:32.253296+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:45:32.253296+0000","last_undegraded":"2023-06-10T21:45:32.253296+0000","last_fullsized":"2023-06-10T21:45:32.253296+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253206+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:45:32.253206+0000","last_peered":"2023-06-10T21:45:32.253206+0000","last_clean":"2023-06-10T21:45:32.253206+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:45:32.253206+0000","last_undegraded":"2023-06-10T21:45:32.253206+0000","last_fullsized":"2023-06-10T21:45:32.253206+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":380,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253127+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:45:32.253127+0000","last_peered":"2023-06-10T21:45:32.253127+0000","last_clean":"2023-06-10T21:45:32.253127+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:45:32.253127+0000","last_undegraded":"2023-06-10T21:45:32.253127+0000","last_fullsized":"2023-06-10T21:45:32.253127+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":284,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743678+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:45:32.743678+0000","last_peered":"2023-06-10T21:45:32.743678+0000","last_clean":"2023-06-10T21:45:32.743678+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:45:32.743678+0000","last_undegraded":"2023-06-10T21:45:32.743678+0000","last_fullsized":"2023-06-10T21:45:32.743678+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":304,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743212+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:45:32.743212+0000","last_peered":"2023-06-10T21:45:32.743212+0000","last_clean":"2023-06-10T21:45:32.743212+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:45:32.743212+0000","last_undegraded":"2023-06-10T21:45:32.743212+0000","last_fullsized":"2023-06-10T21:45:32.743212+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":309,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-06-10T21:45:35.741563+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:45:35.741563+0000","last_peered":"2023-06-10T21:45:35.741563+0000","last_clean":"2023-06-10T21:45:35.741563+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:45:35.741563+0000","last_undegraded":"2023-06-10T21:45:35.741563+0000","last_fullsized":"2023-06-10T21:45:35.741563+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253570+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:45:32.253570+0000","last_peered":"2023-06-10T21:45:32.253570+0000","last_clean":"2023-06-10T21:45:32.253570+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:45:32.253570+0000","last_undegraded":"2023-06-10T21:45:32.253570+0000","last_fullsized":"2023-06-10T21:45:32.253570+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":93,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253509+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:45:32.253509+0000","last_peered":"2023-06-10T21:45:32.253509+0000","last_clean":"2023-06-10T21:45:32.253509+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:45:32.253509+0000","last_undegraded":"2023-06-10T21:45:32.253509+0000","last_fullsized":"2023-06-10T21:45:32.253509+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":619,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.267910+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:45:32.267910+0000","last_peered":"2023-06-10T21:45:32.267910+0000","last_clean":"2023-06-10T21:45:32.267910+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:45:32.267910+0000","last_undegraded":"2023-06-10T21:45:32.267910+0000","last_fullsized":"2023-06-10T21:45:32.267910+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253307+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:45:32.253307+0000","last_peered":"2023-06-10T21:45:32.253307+0000","last_clean":"2023-06-10T21:45:32.253307+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:45:32.253307+0000","last_undegraded":"2023-06-10T21:45:32.253307+0000","last_fullsized":"2023-06-10T21:45:32.253307+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":302,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268826+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:45:32.268826+0000","last_peered":"2023-06-10T21:45:32.268826+0000","last_clean":"2023-06-10T21:45:32.268826+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:45:32.268826+0000","last_undegraded":"2023-06-10T21:45:32.268826+0000","last_fullsized":"2023-06-10T21:45:32.268826+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":471,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.267999+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:45:32.267999+0000","last_peered":"2023-06-10T21:45:32.267999+0000","last_clean":"2023-06-10T21:45:32.267999+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:45:32.267999+0000","last_undegraded":"2023-06-10T21:45:32.267999+0000","last_fullsized":"2023-06-10T21:45:32.267999+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253184+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:45:32.253184+0000","last_peered":"2023-06-10T21:45:32.253184+0000","last_clean":"2023-06-10T21:45:32.253184+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:45:32.253184+0000","last_undegraded":"2023-06-10T21:45:32.253184+0000","last_fullsized":"2023-06-10T21:45:32.253184+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":170,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.744058+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:45:32.744058+0000","last_peered":"2023-06-10T21:45:32.744058+0000","last_clean":"2023-06-10T21:45:32.744058+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:45:32.744058+0000","last_undegraded":"2023-06-10T21:45:32.744058+0000","last_fullsized":"2023-06-10T21:45:32.744058+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":450,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268804+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:45:32.268804+0000","last_peered":"2023-06-10T21:45:32.268804+0000","last_clean":"2023-06-10T21:45:32.268804+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:45:32.268804+0000","last_undegraded":"2023-06-10T21:45:32.268804+0000","last_fullsized":"2023-06-10T21:45:32.268804+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":299,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.743762+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:45:32.743762+0000","last_peered":"2023-06-10T21:45:32.743762+0000","last_clean":"2023-06-10T21:45:32.743762+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:45:32.743762+0000","last_undegraded":"2023-06-10T21:45:32.743762+0000","last_fullsized":"2023-06-10T21:45:32.743762+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":223,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253210+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:45:32.253210+0000","last_peered":"2023-06-10T21:45:32.253210+0000","last_clean":"2023-06-10T21:45:32.253210+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:45:32.253210+0000","last_undegraded":"2023-06-10T21:45:32.253210+0000","last_fullsized":"2023-06-10T21:45:32.253210+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":429,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292411+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:45:32.292411+0000","last_peered":"2023-06-10T21:45:32.292411+0000","last_clean":"2023-06-10T21:45:32.292411+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:45:32.292411+0000","last_undegraded":"2023-06-10T21:45:32.292411+0000","last_fullsized":"2023-06-10T21:45:32.292411+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":240,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292402+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:45:32.292402+0000","last_peered":"2023-06-10T21:45:32.292402+0000","last_clean":"2023-06-10T21:45:32.292402+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:45:32.292402+0000","last_undegraded":"2023-06-10T21:45:32.292402+0000","last_fullsized":"2023-06-10T21:45:32.292402+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253243+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:45:32.253243+0000","last_peered":"2023-06-10T21:45:32.253243+0000","last_clean":"2023-06-10T21:45:32.253243+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:45:32.253243+0000","last_undegraded":"2023-06-10T21:45:32.253243+0000","last_fullsized":"2023-06-10T21:45:32.253243+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-06-10T21:45:43.457627+0000","last_change":"2023-06-10T21:45:43.457627+0000","last_active":"2023-06-10T21:45:43.457627+0000","last_peered":"2023-06-10T21:45:43.457627+0000","last_clean":"2023-06-10T21:45:43.457627+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-06-10T21:45:43.457627+0000","last_undegraded":"2023-06-10T21:45:43.457627+0000","last_fullsized":"2023-06-10T21:45:43.457627+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":167,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.269141+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:45:32.269141+0000","last_peered":"2023-06-10T21:45:32.269141+0000","last_clean":"2023-06-10T21:45:32.269141+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:45:32.269141+0000","last_undegraded":"2023-06-10T21:45:32.269141+0000","last_fullsized":"2023-06-10T21:45:32.269141+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":281,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268291+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:45:32.268291+0000","last_peered":"2023-06-10T21:45:32.268291+0000","last_clean":"2023-06-10T21:45:32.268291+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:45:32.268291+0000","last_undegraded":"2023-06-10T21:45:32.268291+0000","last_fullsized":"2023-06-10T21:45:32.268291+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253157+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:45:32.253157+0000","last_peered":"2023-06-10T21:45:32.253157+0000","last_clean":"2023-06-10T21:45:32.253157+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:45:32.253157+0000","last_undegraded":"2023-06-10T21:45:32.253157+0000","last_fullsized":"2023-06-10T21:45:32.253157+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":118,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.268597+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:45:32.268597+0000","last_peered":"2023-06-10T21:45:32.268597+0000","last_clean":"2023-06-10T21:45:32.268597+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:45:32.268597+0000","last_undegraded":"2023-06-10T21:45:32.268597+0000","last_fullsized":"2023-06-10T21:45:32.268597+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":316,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-06-10T21:45:35.751732+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:45:35.751732+0000","last_peered":"2023-06-10T21:45:35.751732+0000","last_clean":"2023-06-10T21:45:35.751732+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:45:35.751732+0000","last_undegraded":"2023-06-10T21:45:35.751732+0000","last_fullsized":"2023-06-10T21:45:35.751732+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":81,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253463+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:45:32.253463+0000","last_peered":"2023-06-10T21:45:32.253463+0000","last_clean":"2023-06-10T21:45:32.253463+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:45:32.253463+0000","last_undegraded":"2023-06-10T21:45:32.253463+0000","last_fullsized":"2023-06-10T21:45:32.253463+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":323,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291853+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:45:32.291853+0000","last_peered":"2023-06-10T21:45:32.291853+0000","last_clean":"2023-06-10T21:45:32.291853+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:45:32.291853+0000","last_undegraded":"2023-06-10T21:45:32.291853+0000","last_fullsized":"2023-06-10T21:45:32.291853+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":104,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253223+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:45:32.253223+0000","last_peered":"2023-06-10T21:45:32.253223+0000","last_clean":"2023-06-10T21:45:32.253223+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:45:32.253223+0000","last_undegraded":"2023-06-10T21:45:32.253223+0000","last_fullsized":"2023-06-10T21:45:32.253223+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":394,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.291789+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:45:32.291789+0000","last_peered":"2023-06-10T21:45:32.291789+0000","last_clean":"2023-06-10T21:45:32.291789+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:45:32.291789+0000","last_undegraded":"2023-06-10T21:45:32.291789+0000","last_fullsized":"2023-06-10T21:45:32.291789+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":73,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.292337+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:45:32.292337+0000","last_peered":"2023-06-10T21:45:32.292337+0000","last_clean":"2023-06-10T21:45:32.292337+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:45:32.292337+0000","last_undegraded":"2023-06-10T21:45:32.292337+0000","last_fullsized":"2023-06-10T21:45:32.292337+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253253+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:45:32.253253+0000","last_peered":"2023-06-10T21:45:32.253253+0000","last_clean":"2023-06-10T21:45:32.253253+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:45:32.253253+0000","last_undegraded":"2023-06-10T21:45:32.253253+0000","last_fullsized":"2023-06-10T21:45:32.253253+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":340,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253750+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:45:32.253750+0000","last_peered":"2023-06-10T21:45:32.253750+0000","last_clean":"2023-06-10T21:45:32.253750+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:45:32.253750+0000","last_undegraded":"2023-06-10T21:45:32.253750+0000","last_fullsized":"2023-06-10T21:45:32.253750+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-06-10T21:45:32.253354+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:45:32.253354+0000","last_peered":"2023-06-10T21:45:32.253354+0000","last_clean":"2023-06-10T21:45:32.253354+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:45:32.253354+0000","last_undegraded":"2023-06-10T21:45:32.253354+0000","last_fullsized":"2023-06-10T21:45:32.253354+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430783,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":988168,"kb_used_data":4288,"kb_used_omap":0,"kb_used_meta":983872,"kb_avail":92756984,"statfs":{"total":95995035648,"available":94983151616,"internally_reserved":0,"allocated":4390912,"data_stored":3680544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1007484928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":27,"apply_latency_ms":27,"commit_latency_ns":27000000,"apply_latency_ns":27000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.272,"15min":2.5510000000000002},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":8.5129999999999999,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.5820000000000001,"15min":2.8540000000000001},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":20.568999999999999,"5min":22.462,"15min":185.58000000000001},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.6850000000000001,"15min":2.6640000000000001},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":4.9859999999999998,"5min":38.078000000000003,"15min":87.417000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.4970000000000001,"5min":2.0179999999999998,"15min":2.9790000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":4.8739999999999997,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:44:43 2023","interfaces":[{"interface":"back","average":{"1min":3.9569999999999999,"5min":3.1299999999999999,"15min":4.9729999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":36.627000000000002,"5min":75.219999999999999,"15min":172.33600000000001},"last":0.627},{"interface":"front","average":{"1min":4.4880000000000004,"5min":3.1379999999999999,"15min":4.9610000000000003},"min":{"1min":0.44600000000000001,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":36.533999999999999,"5min":75.108999999999995,"15min":172.185},"last":0.61099999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":3.9430000000000001,"15min":4.7370000000000001},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":5.0199999999999996,"5min":89.504999999999995,"15min":147.00700000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.655,"5min":3.7120000000000002,"15min":4.5149999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":4.931,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.73499999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.1080000000000001,"15min":0.66600000000000004},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":4.8479999999999999,"5min":13.429,"15min":13.429},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.645,"5min":1.266,"15min":0.82399999999999995},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":12.279999999999999,"5min":12.279999999999999,"15min":22.77},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:45:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":10.191000000000001,"15min":11.515000000000001},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.984,"5min":131.619,"15min":131.619},"last":0.76400000000000001},{"interface":"front","average":{"1min":2.036,"5min":10.006,"15min":11.334},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.9370000000000003,"5min":134.928,"15min":134.928},"last":0.72499999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.0189999999999999,"15min":0.74399999999999999},"min":{"1min":0.371,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":10.890000000000001},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.2210000000000001,"15min":0.76600000000000001},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.70099999999999996}]}]},{"osd":6,"up_from":0,"seq":180388627011,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566454,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075842,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1030160,"kb_used_data":5000,"kb_used_omap":7,"kb_used_meta":1025144,"kb_avail":92714992,"statfs":{"total":95995035648,"available":94940151808,"internally_reserved":0,"allocated":5120000,"data_stored":3765386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1049747537},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":2.375,"15min":4.2670000000000003},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":6.6429999999999998,"5min":83.733000000000004,"15min":171.33699999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.9690000000000001,"15min":4.0640000000000001},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9300000000000002,"5min":82.322999999999993,"15min":171.56200000000001},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":3.653,"15min":5.3120000000000003},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.217},"max":{"1min":5.2370000000000001,"5min":85.709999999999994,"15min":171.286},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.542,"5min":3.548,"15min":3.891},"min":{"1min":0.27800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.1379999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:45:25 2023","interfaces":[{"interface":"back","average":{"1min":5.1600000000000001,"5min":8.7620000000000005,"15min":6.665},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":25.725000000000001,"5min":145.07599999999999,"15min":171.78299999999999},"last":0.439},{"interface":"front","average":{"1min":4.1310000000000002,"5min":8.3109999999999999,"15min":6.4779999999999998},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":25.626999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":1.266,"15min":2.3130000000000002},"min":{"1min":0.33500000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.9629999999999992,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.903,"5min":1.7669999999999999,"15min":3.2370000000000001},"min":{"1min":0.36899999999999999,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.2539999999999996,"5min":14.858000000000001,"15min":171.18100000000001},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.9989999999999997,"15min":6.6539999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.0389999999999997,"5min":75.896000000000001,"15min":75.896000000000001},"last":0.502},{"interface":"front","average":{"1min":1.976,"5min":5.468,"15min":6.0499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":7.3460000000000001,"5min":76.337000000000003,"15min":76.337000000000003},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9670000000000001,"5min":1.476,"15min":2.4300000000000002},"min":{"1min":0.41999999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.0139999999999993,"5min":9.0139999999999993,"15min":171.441},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.948,"5min":1.931,"15min":3.3029999999999999},"min":{"1min":0.51900000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":4.9199999999999999,"5min":14.893000000000001,"15min":171.24600000000001},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:44:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9079999999999999,"5min":1.9430000000000001,"15min":4.0979999999999999},"min":{"1min":0.503,"5min":0.377,"15min":0.34200000000000003},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":219.131},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.9279999999999999,"5min":1.585,"15min":4.0250000000000004},"min":{"1min":0.48099999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":12.15,"5min":12.15,"15min":219.19},"last":0.51500000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824202,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747096,"kb_used_data":5136,"kb_used_omap":7,"kb_used_meta":741944,"kb_avail":92998056,"statfs":{"total":95995035648,"available":95230009344,"internally_reserved":0,"allocated":5259264,"data_stored":3776598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":759750675},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.1499999999999999,"15min":2.9460000000000002},"min":{"1min":0.254,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":3.4369999999999998,"5min":30.744,"15min":105.09},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.1160000000000001,"15min":3.153},"min":{"1min":0.26400000000000001,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":6.3479999999999999,"5min":30.805,"15min":99.852999999999994},"last":0.502}]},{"osd":1,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.45,"15min":3.21},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":6.5030000000000001,"5min":7.6509999999999998,"15min":94.766000000000005},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.028,"15min":3.0939999999999999},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":2.7519999999999998,"5min":8.9649999999999999,"15min":94.691000000000003},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":9.6219999999999999,"5min":3.6019999999999999,"15min":4.0990000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":126.913,"5min":126.913,"15min":231.07900000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":9.8379999999999992,"5min":3.8399999999999999,"15min":5.2599999999999998},"min":{"1min":0.34499999999999997,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":128.52199999999999,"5min":128.52199999999999,"15min":231.58600000000001},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.714,"15min":1.5209999999999999},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.5620000000000003,"5min":9.5150000000000006,"15min":13.417},"last":0.433},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.6639999999999999,"15min":2.2160000000000002},"min":{"1min":0.36399999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":8.8379999999999992,"5min":8.8379999999999992,"15min":19.728000000000002},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:45:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.097,"15min":1.0920000000000001},"min":{"1min":0.37,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.6269999999999998,"5min":6.532,"15min":6.532},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.6160000000000001,"15min":2.6739999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.1130000000000004,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":1.8480000000000001,"15min":2.3039999999999998},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":9.0250000000000004,"5min":9.0250000000000004,"15min":19.805},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.232,"15min":1.5449999999999999},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.0750000000000002,"5min":6.0750000000000002,"15min":17.327000000000002},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.1360000000000001,"15min":2.6179999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":8.9920000000000009,"5min":8.9920000000000009,"15min":79.984999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.335,"15min":1.839},"min":{"1min":0.45700000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6.1289999999999996,"5min":6.1289999999999996,"15min":80.272999999999996},"last":0.56999999999999995}]}]},{"osd":4,"up_from":0,"seq":128849019467,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245473,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256704,"kb_used_data":3704,"kb_used_omap":11,"kb_used_meta":252980,"kb_avail":93488448,"statfs":{"total":95995035648,"available":95732170752,"internally_reserved":0,"allocated":3792896,"data_stored":3090325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":259052022},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.0579999999999998,"15min":2.0720000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.5209999999999999,"5min":1.361,"15min":1.335},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3170000000000002,"5min":4.266,"15min":4.266},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.6269999999999998,"15min":2.6480000000000001},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2229999999999999,"15min":2.262},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.9500000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.67900000000000005}]},{"osd":2,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":9.5090000000000003,"5min":3.391,"15min":2.371},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":159.94999999999999,"5min":159.94999999999999,"15min":159.94999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":9.3230000000000004,"5min":3.7090000000000001,"15min":2.7730000000000001},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":159.851,"5min":159.851,"15min":159.851},"last":0.47599999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":7.569,"15min":8.0109999999999992},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":54.920000000000002,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.377},{"interface":"front","average":{"1min":5.3339999999999996,"5min":7.3819999999999997,"15min":7.7240000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":54.848999999999997,"5min":133.03800000000001,"15min":133.03800000000001},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.96199999999999997,"15min":0.92600000000000005},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.4329999999999998,"5min":5.173,"15min":5.173},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.3180000000000001,"5min":2.1099999999999999,"15min":2.242},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.238,"5min":7.476,"15min":7.476},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.752,"15min":1.8420000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.2149999999999999,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.173,"5min":1.3100000000000001,"15min":1.333},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":4.2910000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.625,"15min":1.6160000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.2430000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.9630000000000001,"5min":2.9049999999999998,"15min":2.895},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.1920000000000002,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":0.63800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:45:45.890 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:45.918 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:45 smithi152 ceph-mon[156819]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:45 smithi152 ceph-mon[156819]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:45 smithi152 ceph-mon[156819]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:45 smithi152 ceph-mon[156819]: failed to encode map e421 with expected crc 2023-06-10T21:45:45.919 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:45 smithi152 ceph-mon[156819]: osdmap e422: 8 total, 8 up, 4 in 2023-06-10T21:45:45.921 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:45.971 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:45.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:45.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:45.996 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:46.077 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:46.114 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:46.296 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:45:46.297 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:45:46.297 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:45:46.297 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:45:46.322 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:46.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:46.430 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:46.458 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:46.503 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:46.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:46.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:46.638 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[186706]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[186706]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:45:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[186706]: pgmap v1554: 89 pgs: 3 creating+peering, 6 unknown, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[180297]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[180297]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:45:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:46 smithi064 ceph-mon[180297]: pgmap v1554: 89 pgs: 3 creating+peering, 6 unknown, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:46.684 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:46.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:46.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:45:46.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:46 smithi152 ceph-mon[156819]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:46 smithi152 ceph-mon[156819]: failed to encode map e422 with expected crc 2023-06-10T21:45:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:46 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:45:46.822 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:46 smithi152 ceph-mon[156819]: pgmap v1554: 89 pgs: 3 creating+peering, 6 unknown, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:46.824 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:46.869 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:46.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:46.974 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:47.003 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:47.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:45:47.507 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:47.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:47.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:47.641 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:47.685 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:47.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:47.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:47.812 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:47.819 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:47.867 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:47.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:47.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:47.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:48.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:48.146 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:45:48.167 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:48.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:48.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:48.176 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:45:48.197 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:45:48.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:45:48.248 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:45:48.272 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:45:48.273 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:45:48.293 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:45:48.300 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:48.309 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:48.318 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:45:48.320 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.320 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.320 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.320 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.321 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:45:48.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:45:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:48.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:45:48.450 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:48.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1436970372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:48.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:48.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[186706]: pgmap v1555: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1436970372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:48.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:48 smithi064 ceph-mon[180297]: pgmap v1555: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:48.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1436970372' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:45:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:45:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:48 smithi152 ceph-mon[156819]: pgmap v1555: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:48.890 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:45:49.242 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:49.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:49.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:45:49.374 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:49.572 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:45:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:49.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:45:49.676 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:49.708 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:49.964 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:49.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:49.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:49.970 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:50.068 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:50.109 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:50.150 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:50.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:50.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:50.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:50.290 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:50.953 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:50.970 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:50.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:50.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:51.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:51.106 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:51.151 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:51.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:45:51.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:51.279 INFO:teuthology.orchestra.run.smithi064.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-06-10T21:45:51.296 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:51.339 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:45:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:51 smithi152 ceph-mon[156819]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3713761132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:51.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:51.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[180297]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:51.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3713761132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:51.471 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:51.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[186706]: pgmap v1556: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:51.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3713761132' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:51.471 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:45:52.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:52.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:52.359 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:52.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:45:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[186706]: osdmap e423: 8 total, 8 up, 4 in 2023-06-10T21:45:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[186706]: failed to encode map e423 with expected crc 2023-06-10T21:45:52.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[186706]: pgmap v1558: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:52.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:45:52.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[180297]: osdmap e423: 8 total, 8 up, 4 in 2023-06-10T21:45:52.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[180297]: failed to encode map e423 with expected crc 2023-06-10T21:45:52.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:52 smithi064 ceph-mon[180297]: pgmap v1558: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-10T21:45:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:52 smithi152 ceph-mon[156819]: osdmap e423: 8 total, 8 up, 4 in 2023-06-10T21:45:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:52 smithi152 ceph-mon[156819]: failed to encode map e423 with expected crc 2023-06-10T21:45:52.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:52 smithi152 ceph-mon[156819]: pgmap v1558: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:52.989 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:52.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:53.076 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:45:53.077 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:45:53.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:45:53.098 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:45:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:45:52] "GET /metrics HTTP/1.1" 200 223934 "" "Prometheus/2.33.4" 2023-06-10T21:45:53.124 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:53.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:53 smithi064 ceph-mon[180297]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:53 smithi064 ceph-mon[180297]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:53 smithi064 ceph-mon[186706]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:53 smithi064 ceph-mon[186706]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:53 smithi152 ceph-mon[156819]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:53 smithi152 ceph-mon[156819]: failed to encode map e423 with expected crc 2023-06-10T21:45:53.875 DEBUG:teuthology.orchestra.run.smithi064:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-06-10T21:45:53.877 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-10T21:45:53.877 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-10T21:45:54.168 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:hints = 1 2023-06-10T21:45:54.168 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-10T21:45:54.168 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object prefix: benchmark_data_smithi064_559916 2023-06-10T21:45:54.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:45:54.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 0 0 0 0 0 0 - 0 2023-06-10T21:45:54.431 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:45:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:54.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:45:54.511 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:45:54.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:54.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[186706]: failed to encode map e423 with expected crc 2023-06-10T21:45:54.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:54.561 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[186706]: pgmap v1559: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:54.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[180297]: failed to encode map e423 with expected crc 2023-06-10T21:45:54.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:54.562 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:54 smithi064 ceph-mon[180297]: pgmap v1559: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:54.610 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:54.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:54 smithi152 ceph-mon[156819]: failed to encode map e423 with expected crc 2023-06-10T21:45:54.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:54 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:45:54.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:54 smithi152 ceph-mon[156819]: pgmap v1559: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2023-06-10T21:45:54.817 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:54.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:54.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:54.980 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:55.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:45:55.118 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:55.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 1 15 3025 3010 187.995 188.125 0.00360088 0.0052972 2023-06-10T21:45:55.181 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:45:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:55.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:45:55.287 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:55.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:56.022 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:56.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:56.032 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:45:56.129 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:56.159 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:56.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 2 16 6274 6258 195.486 203 0.00233586 0.00497973 2023-06-10T21:45:56.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4246919725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:56.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4246919725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:56.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4246919725' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:45:57.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 3 16 9876 9860 205.357 225.125 0.0029381 0.00486208 2023-06-10T21:45:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: osdmap e424: 8 total, 8 up, 4 in 2023-06-10T21:45:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[186706]: pgmap v1561: 89 pgs: 89 active+clean; 105 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 13 MiB/s wr, 208 op/s 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: osdmap e424: 8 total, 8 up, 4 in 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:57 smithi064 ceph-mon[180297]: pgmap v1561: 89 pgs: 89 active+clean; 105 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 13 MiB/s wr, 208 op/s 2023-06-10T21:45:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:45:57.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:45:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: osdmap e424: 8 total, 8 up, 4 in 2023-06-10T21:45:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: failed to encode map e424 with expected crc 2023-06-10T21:45:57.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:57 smithi152 ceph-mon[156819]: pgmap v1561: 89 pgs: 89 active+clean; 105 MiB data, 3.1 GiB used, 354 GiB / 358 GiB avail; 13 MiB/s wr, 208 op/s 2023-06-10T21:45:58.169 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 4 16 13067 13051 203.873 199.438 0.00966666 0.00489939 2023-06-10T21:45:58.192 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:58.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:58.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:58.296 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:45:58.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:45:58 smithi064 ceph-mon[180297]: pgmap v1562: 89 pgs: 89 active+clean; 254 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 32 MiB/s wr, 507 op/s 2023-06-10T21:45:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:45:58 smithi064 ceph-mon[186706]: pgmap v1562: 89 pgs: 89 active+clean; 254 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 32 MiB/s wr, 507 op/s 2023-06-10T21:45:58.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:45:58 smithi152 ceph-mon[156819]: pgmap v1562: 89 pgs: 89 active+clean; 254 MiB data, 3.4 GiB used, 354 GiB / 358 GiB avail; 32 MiB/s wr, 507 op/s 2023-06-10T21:45:58.996 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:45:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:59.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:45:59.102 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:45:59.160 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:59.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 5 16 15583 15567 194.25 157.25 0.0096298 0.00514125 2023-06-10T21:45:59.180 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:45:59.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:45:59.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:59.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:45:59.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:45:59.620 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:59.688 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:45:59.693 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:45:59.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:59.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:45:59.703 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:45:59.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:45:59.837 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:45:59.902 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:45:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:45:59.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:00.009 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:00.041 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:00.080 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:00.082 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:00.119 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:00.158 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:00.177 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 6 16 17604 17588 182.922 126.312 0.0044325 0.00545978 2023-06-10T21:46:00.191 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:00.194 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/509735449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:00.226 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:00.262 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:00.300 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:00.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.303 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.304 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.304 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.304 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:46:00.340 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/509735449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:00.340 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/509735449' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:01.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:46:01.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 7 16 20123 20107 179.283 157.438 0.00586516 0.00557122 2023-06-10T21:46:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[180297]: from='client.37149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[180297]: pgmap v1563: 89 pgs: 89 active+clean; 666 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-10T21:46:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:01.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[186706]: from='client.37149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[186706]: pgmap v1563: 89 pgs: 89 active+clean; 666 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-10T21:46:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:01.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:01.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:01 smithi152 ceph-mon[156819]: from='client.37149 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:01 smithi152 ceph-mon[156819]: pgmap v1563: 89 pgs: 89 active+clean; 666 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-10T21:46:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:01.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:02.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 8 16 22728 22712 177.225 162.812 0.00763323 0.00563743 2023-06-10T21:46:02.754 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:02.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:02.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:02.860 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:02.915 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:03.002 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:03.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:03.110 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:03.128 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:02] "GET /metrics HTTP/1.1" 200 223934 "" "Prometheus/2.33.4" 2023-06-10T21:46:03.168 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:03.178 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 9 16 25187 25171 174.611 153.688 0.0118076 0.00572132 2023-06-10T21:46:03.243 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:03.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:03.354 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:03.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[186706]: pgmap v1564: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:03.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:03.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[180297]: pgmap v1564: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:03.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:03.418 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:03.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:03 smithi152 ceph-mon[156819]: pgmap v1564: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:03.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:04.062 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1562,"stamp":"2023-06-10T21:45:58.228892+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":266672014,"num_objects":4084,"num_object_clones":0,"num_object_copies":12252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4084,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":4321,"num_write_kb":262286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":2377742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4255,"ondisk_log_size":4255,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3555916,"kb_used_data":548204,"kb_used_omap":27,"kb_used_meta":3007588,"kb_avail":371424692,"statfs":{"total":383980142592,"available":380338884608,"internally_reserved":0,"allocated":561360896,"data_stored":557084626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3079770714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4060,"num_write_kb":259840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002223"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":463,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272401+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:45:51.272401+0000","last_peered":"2023-06-10T21:45:51.272401+0000","last_clean":"2023-06-10T21:45:51.272401+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:45:51.272401+0000","last_undegraded":"2023-06-10T21:45:51.272401+0000","last_fullsized":"2023-06-10T21:45:51.272401+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514343+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:45:45.514343+0000","last_peered":"2023-06-10T21:45:45.514343+0000","last_clean":"2023-06-10T21:45:45.514343+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:45:45.514343+0000","last_undegraded":"2023-06-10T21:45:45.514343+0000","last_fullsized":"2023-06-10T21:45:45.514343+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":355,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:52.459091+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:45:52.459091+0000","last_peered":"2023-06-10T21:45:52.459091+0000","last_clean":"2023-06-10T21:45:52.459091+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:45:52.459091+0000","last_undegraded":"2023-06-10T21:45:52.459091+0000","last_fullsized":"2023-06-10T21:45:52.459091+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":308,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513593+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:45:45.513593+0000","last_peered":"2023-06-10T21:45:45.513593+0000","last_clean":"2023-06-10T21:45:45.513593+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:45:45.513593+0000","last_undegraded":"2023-06-10T21:45:45.513593+0000","last_fullsized":"2023-06-10T21:45:45.513593+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513043+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:45:45.513043+0000","last_peered":"2023-06-10T21:45:45.513043+0000","last_clean":"2023-06-10T21:45:45.513043+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:45:45.513043+0000","last_undegraded":"2023-06-10T21:45:45.513043+0000","last_fullsized":"2023-06-10T21:45:45.513043+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":105,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470994+0000","last_change":"2023-06-10T21:43:01.231656+0000","last_active":"2023-06-10T21:45:45.470994+0000","last_peered":"2023-06-10T21:45:45.470994+0000","last_clean":"2023-06-10T21:45:45.470994+0000","last_became_active":"2023-06-10T21:43:01.227746+0000","last_became_peered":"2023-06-10T21:43:01.227746+0000","last_unstale":"2023-06-10T21:45:45.470994+0000","last_undegraded":"2023-06-10T21:45:45.470994+0000","last_fullsized":"2023-06-10T21:45:45.470994+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470888+0000","last_change":"2023-06-10T21:43:01.211767+0000","last_active":"2023-06-10T21:45:45.470888+0000","last_peered":"2023-06-10T21:45:45.470888+0000","last_clean":"2023-06-10T21:45:45.470888+0000","last_became_active":"2023-06-10T21:43:01.211290+0000","last_became_peered":"2023-06-10T21:43:01.211290+0000","last_unstale":"2023-06-10T21:45:45.470888+0000","last_undegraded":"2023-06-10T21:45:45.470888+0000","last_fullsized":"2023-06-10T21:45:45.470888+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513252+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:45:45.513252+0000","last_peered":"2023-06-10T21:45:45.513252+0000","last_clean":"2023-06-10T21:45:45.513252+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:45:45.513252+0000","last_undegraded":"2023-06-10T21:45:45.513252+0000","last_fullsized":"2023-06-10T21:45:45.513252+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":96,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470887+0000","last_change":"2023-06-10T21:43:01.233218+0000","last_active":"2023-06-10T21:45:45.470887+0000","last_peered":"2023-06-10T21:45:45.470887+0000","last_clean":"2023-06-10T21:45:45.470887+0000","last_became_active":"2023-06-10T21:43:01.232828+0000","last_became_peered":"2023-06-10T21:43:01.232828+0000","last_unstale":"2023-06-10T21:45:45.470887+0000","last_undegraded":"2023-06-10T21:45:45.470887+0000","last_fullsized":"2023-06-10T21:45:45.470887+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":174,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513839+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:45:45.513839+0000","last_peered":"2023-06-10T21:45:45.513839+0000","last_clean":"2023-06-10T21:45:45.513839+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:45:45.513839+0000","last_undegraded":"2023-06-10T21:45:45.513839+0000","last_fullsized":"2023-06-10T21:45:45.513839+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":180,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513351+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:45:45.513351+0000","last_peered":"2023-06-10T21:45:45.513351+0000","last_clean":"2023-06-10T21:45:45.513351+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:45:45.513351+0000","last_undegraded":"2023-06-10T21:45:45.513351+0000","last_fullsized":"2023-06-10T21:45:45.513351+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":424,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272492+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:45:51.272492+0000","last_peered":"2023-06-10T21:45:51.272492+0000","last_clean":"2023-06-10T21:45:51.272492+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:45:51.272492+0000","last_undegraded":"2023-06-10T21:45:51.272492+0000","last_fullsized":"2023-06-10T21:45:51.272492+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":346,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513388+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:45:45.513388+0000","last_peered":"2023-06-10T21:45:45.513388+0000","last_clean":"2023-06-10T21:45:45.513388+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:45:45.513388+0000","last_undegraded":"2023-06-10T21:45:45.513388+0000","last_fullsized":"2023-06-10T21:45:45.513388+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":137,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470364+0000","last_change":"2023-06-10T21:43:01.225948+0000","last_active":"2023-06-10T21:45:45.470364+0000","last_peered":"2023-06-10T21:45:45.470364+0000","last_clean":"2023-06-10T21:45:45.470364+0000","last_became_active":"2023-06-10T21:43:01.225449+0000","last_became_peered":"2023-06-10T21:43:01.225449+0000","last_unstale":"2023-06-10T21:45:45.470364+0000","last_undegraded":"2023-06-10T21:45:45.470364+0000","last_fullsized":"2023-06-10T21:45:45.470364+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":482,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513072+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:45:45.513072+0000","last_peered":"2023-06-10T21:45:45.513072+0000","last_clean":"2023-06-10T21:45:45.513072+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:45:45.513072+0000","last_undegraded":"2023-06-10T21:45:45.513072+0000","last_fullsized":"2023-06-10T21:45:45.513072+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":371,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.515390+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:45:45.515390+0000","last_peered":"2023-06-10T21:45:45.515390+0000","last_clean":"2023-06-10T21:45:45.515390+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:45:45.515390+0000","last_undegraded":"2023-06-10T21:45:45.515390+0000","last_fullsized":"2023-06-10T21:45:45.515390+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":293,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513035+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:45:45.513035+0000","last_peered":"2023-06-10T21:45:45.513035+0000","last_clean":"2023-06-10T21:45:45.513035+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:45:45.513035+0000","last_undegraded":"2023-06-10T21:45:45.513035+0000","last_fullsized":"2023-06-10T21:45:45.513035+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470780+0000","last_change":"2023-06-10T21:43:01.227110+0000","last_active":"2023-06-10T21:45:45.470780+0000","last_peered":"2023-06-10T21:45:45.470780+0000","last_clean":"2023-06-10T21:45:45.470780+0000","last_became_active":"2023-06-10T21:43:01.225562+0000","last_became_peered":"2023-06-10T21:43:01.225562+0000","last_unstale":"2023-06-10T21:45:45.470780+0000","last_undegraded":"2023-06-10T21:45:45.470780+0000","last_fullsized":"2023-06-10T21:45:45.470780+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":435,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.512999+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:45:45.512999+0000","last_peered":"2023-06-10T21:45:45.512999+0000","last_clean":"2023-06-10T21:45:45.512999+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:45:45.512999+0000","last_undegraded":"2023-06-10T21:45:45.512999+0000","last_fullsized":"2023-06-10T21:45:45.512999+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514286+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:45:45.514286+0000","last_peered":"2023-06-10T21:45:45.514286+0000","last_clean":"2023-06-10T21:45:45.514286+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:45:45.514286+0000","last_undegraded":"2023-06-10T21:45:45.514286+0000","last_fullsized":"2023-06-10T21:45:45.514286+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":447,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.469902+0000","last_change":"2023-06-10T21:43:01.211622+0000","last_active":"2023-06-10T21:45:45.469902+0000","last_peered":"2023-06-10T21:45:45.469902+0000","last_clean":"2023-06-10T21:45:45.469902+0000","last_became_active":"2023-06-10T21:43:01.211298+0000","last_became_peered":"2023-06-10T21:43:01.211298+0000","last_unstale":"2023-06-10T21:45:45.469902+0000","last_undegraded":"2023-06-10T21:45:45.469902+0000","last_fullsized":"2023-06-10T21:45:45.469902+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514518+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:45:45.514518+0000","last_peered":"2023-06-10T21:45:45.514518+0000","last_clean":"2023-06-10T21:45:45.514518+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:45:45.514518+0000","last_undegraded":"2023-06-10T21:45:45.514518+0000","last_fullsized":"2023-06-10T21:45:45.514518+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"423'336","reported_seq":347,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:55.981949+0000","last_change":"2023-06-10T21:45:44.480492+0000","last_active":"2023-06-10T21:45:55.981949+0000","last_peered":"2023-06-10T21:45:55.981949+0000","last_clean":"2023-06-10T21:45:55.981949+0000","last_became_active":"2023-06-10T21:45:44.480237+0000","last_became_peered":"2023-06-10T21:45:44.480237+0000","last_unstale":"2023-06-10T21:45:55.981949+0000","last_undegraded":"2023-06-10T21:45:55.981949+0000","last_fullsized":"2023-06-10T21:45:55.981949+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":336,"log_dups_size":0,"ondisk_log_size":336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:15:36.524573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22020096,"num_objects":336,"num_object_clones":0,"num_object_copies":1008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":336,"num_write_kb":21504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":215,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513743+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:45:45.513743+0000","last_peered":"2023-06-10T21:45:45.513743+0000","last_clean":"2023-06-10T21:45:45.513743+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:45:45.513743+0000","last_undegraded":"2023-06-10T21:45:45.513743+0000","last_fullsized":"2023-06-10T21:45:45.513743+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.469774+0000","last_change":"2023-06-10T21:43:01.236036+0000","last_active":"2023-06-10T21:45:45.469774+0000","last_peered":"2023-06-10T21:45:45.469774+0000","last_clean":"2023-06-10T21:45:45.469774+0000","last_became_active":"2023-06-10T21:43:01.229981+0000","last_became_peered":"2023-06-10T21:43:01.229981+0000","last_unstale":"2023-06-10T21:45:45.469774+0000","last_undegraded":"2023-06-10T21:45:45.469774+0000","last_fullsized":"2023-06-10T21:45:45.469774+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"423'350","reported_seq":360,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:55.975964+0000","last_change":"2023-06-10T21:45:44.477018+0000","last_active":"2023-06-10T21:45:55.975964+0000","last_peered":"2023-06-10T21:45:55.975964+0000","last_clean":"2023-06-10T21:45:55.975964+0000","last_became_active":"2023-06-10T21:45:44.476687+0000","last_became_peered":"2023-06-10T21:45:44.476687+0000","last_unstale":"2023-06-10T21:45:55.975964+0000","last_undegraded":"2023-06-10T21:45:55.975964+0000","last_fullsized":"2023-06-10T21:45:55.975964+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":350,"log_dups_size":0,"ondisk_log_size":350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:10:48.652326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22937600,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":350,"num_write_kb":22400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":215,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513735+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:45:45.513735+0000","last_peered":"2023-06-10T21:45:45.513735+0000","last_clean":"2023-06-10T21:45:45.513735+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:45:45.513735+0000","last_undegraded":"2023-06-10T21:45:45.513735+0000","last_fullsized":"2023-06-10T21:45:45.513735+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513935+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:45:45.513935+0000","last_peered":"2023-06-10T21:45:45.513935+0000","last_clean":"2023-06-10T21:45:45.513935+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:45:45.513935+0000","last_undegraded":"2023-06-10T21:45:45.513935+0000","last_fullsized":"2023-06-10T21:45:45.513935+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"424'404","reported_seq":417,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.241443+0000","last_change":"2023-06-10T21:45:44.477367+0000","last_active":"2023-06-10T21:45:56.241443+0000","last_peered":"2023-06-10T21:45:56.241443+0000","last_clean":"2023-06-10T21:45:56.241443+0000","last_became_active":"2023-06-10T21:45:44.477107+0000","last_became_peered":"2023-06-10T21:45:44.477107+0000","last_unstale":"2023-06-10T21:45:56.241443+0000","last_undegraded":"2023-06-10T21:45:56.241443+0000","last_fullsized":"2023-06-10T21:45:56.241443+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":404,"log_dups_size":0,"ondisk_log_size":404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:35:06.980899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26476544,"num_objects":404,"num_object_clones":0,"num_object_copies":1212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":404,"num_write_kb":25856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":502,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513448+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:45:45.513448+0000","last_peered":"2023-06-10T21:45:45.513448+0000","last_clean":"2023-06-10T21:45:45.513448+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:45:45.513448+0000","last_undegraded":"2023-06-10T21:45:45.513448+0000","last_fullsized":"2023-06-10T21:45:45.513448+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":186,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.512901+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:45:45.512901+0000","last_peered":"2023-06-10T21:45:45.512901+0000","last_clean":"2023-06-10T21:45:45.512901+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:45:45.512901+0000","last_undegraded":"2023-06-10T21:45:45.512901+0000","last_fullsized":"2023-06-10T21:45:45.512901+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"423'358","reported_seq":370,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:55.981081+0000","last_change":"2023-06-10T21:45:44.476967+0000","last_active":"2023-06-10T21:45:55.981081+0000","last_peered":"2023-06-10T21:45:55.981081+0000","last_clean":"2023-06-10T21:45:55.981081+0000","last_became_active":"2023-06-10T21:45:44.476670+0000","last_became_peered":"2023-06-10T21:45:44.476670+0000","last_unstale":"2023-06-10T21:45:55.981081+0000","last_undegraded":"2023-06-10T21:45:55.981081+0000","last_fullsized":"2023-06-10T21:45:55.981081+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":358,"log_dups_size":0,"ondisk_log_size":358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:11:49.819095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23461888,"num_objects":358,"num_object_clones":0,"num_object_copies":1074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":358,"num_write_kb":22912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":106,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470457+0000","last_change":"2023-06-10T21:43:01.222066+0000","last_active":"2023-06-10T21:45:45.470457+0000","last_peered":"2023-06-10T21:45:45.470457+0000","last_clean":"2023-06-10T21:45:45.470457+0000","last_became_active":"2023-06-10T21:43:01.212327+0000","last_became_peered":"2023-06-10T21:43:01.212327+0000","last_unstale":"2023-06-10T21:45:45.470457+0000","last_undegraded":"2023-06-10T21:45:45.470457+0000","last_fullsized":"2023-06-10T21:45:45.470457+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514076+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:45:45.514076+0000","last_peered":"2023-06-10T21:45:45.514076+0000","last_clean":"2023-06-10T21:45:45.514076+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:45:45.514076+0000","last_undegraded":"2023-06-10T21:45:45.514076+0000","last_fullsized":"2023-06-10T21:45:45.514076+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.6","version":"424'402","reported_seq":415,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.251844+0000","last_change":"2023-06-10T21:45:44.478058+0000","last_active":"2023-06-10T21:45:56.251844+0000","last_peered":"2023-06-10T21:45:56.251844+0000","last_clean":"2023-06-10T21:45:56.251844+0000","last_became_active":"2023-06-10T21:45:44.477758+0000","last_became_peered":"2023-06-10T21:45:44.477758+0000","last_unstale":"2023-06-10T21:45:56.251844+0000","last_undegraded":"2023-06-10T21:45:56.251844+0000","last_fullsized":"2023-06-10T21:45:56.251844+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":402,"log_dups_size":0,"ondisk_log_size":402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:39:43.723763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26345472,"num_objects":402,"num_object_clones":0,"num_object_copies":1206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":402,"num_write_kb":25728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":81,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513973+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:45:45.513973+0000","last_peered":"2023-06-10T21:45:45.513973+0000","last_clean":"2023-06-10T21:45:45.513973+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:45:45.513973+0000","last_undegraded":"2023-06-10T21:45:45.513973+0000","last_fullsized":"2023-06-10T21:45:45.513973+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.515328+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:45:45.515328+0000","last_peered":"2023-06-10T21:45:45.515328+0000","last_clean":"2023-06-10T21:45:45.515328+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:45:45.515328+0000","last_undegraded":"2023-06-10T21:45:45.515328+0000","last_fullsized":"2023-06-10T21:45:45.515328+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"423'121","reported_seq":132,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.778661+0000","last_change":"2023-06-10T21:45:44.477136+0000","last_active":"2023-06-10T21:45:54.778661+0000","last_peered":"2023-06-10T21:45:54.778661+0000","last_clean":"2023-06-10T21:45:54.778661+0000","last_became_active":"2023-06-10T21:45:44.476746+0000","last_became_peered":"2023-06-10T21:45:44.476746+0000","last_unstale":"2023-06-10T21:45:54.778661+0000","last_undegraded":"2023-06-10T21:45:54.778661+0000","last_fullsized":"2023-06-10T21:45:54.778661+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":121,"num_write_kb":7744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514249+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:45:45.514249+0000","last_peered":"2023-06-10T21:45:45.514249+0000","last_clean":"2023-06-10T21:45:45.514249+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:45:45.514249+0000","last_undegraded":"2023-06-10T21:45:45.514249+0000","last_fullsized":"2023-06-10T21:45:45.514249+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":462,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272832+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:45:51.272832+0000","last_peered":"2023-06-10T21:45:51.272832+0000","last_clean":"2023-06-10T21:45:51.272832+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:45:51.272832+0000","last_undegraded":"2023-06-10T21:45:51.272832+0000","last_fullsized":"2023-06-10T21:45:51.272832+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"423'33","reported_seq":44,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.272594+0000","last_change":"2023-06-10T21:45:44.480834+0000","last_active":"2023-06-10T21:45:54.272594+0000","last_peered":"2023-06-10T21:45:54.272594+0000","last_clean":"2023-06-10T21:45:54.272594+0000","last_became_active":"2023-06-10T21:45:44.480535+0000","last_became_peered":"2023-06-10T21:45:44.480535+0000","last_unstale":"2023-06-10T21:45:54.272594+0000","last_undegraded":"2023-06-10T21:45:54.272594+0000","last_fullsized":"2023-06-10T21:45:54.272594+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":33,"log_dups_size":0,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2162688,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":33,"num_write_kb":2112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":440,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272512+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:45:51.272512+0000","last_peered":"2023-06-10T21:45:51.272512+0000","last_clean":"2023-06-10T21:45:51.272512+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:45:51.272512+0000","last_undegraded":"2023-06-10T21:45:51.272512+0000","last_fullsized":"2023-06-10T21:45:51.272512+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514268+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:45:45.514268+0000","last_peered":"2023-06-10T21:45:45.514268+0000","last_clean":"2023-06-10T21:45:45.514268+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:45:45.514268+0000","last_undegraded":"2023-06-10T21:45:45.514268+0000","last_fullsized":"2023-06-10T21:45:45.514268+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"423'35","reported_seq":46,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.272154+0000","last_change":"2023-06-10T21:45:44.482366+0000","last_active":"2023-06-10T21:45:54.272154+0000","last_peered":"2023-06-10T21:45:54.272154+0000","last_clean":"2023-06-10T21:45:54.272154+0000","last_became_active":"2023-06-10T21:45:44.482065+0000","last_became_peered":"2023-06-10T21:45:44.482065+0000","last_unstale":"2023-06-10T21:45:54.272154+0000","last_undegraded":"2023-06-10T21:45:54.272154+0000","last_fullsized":"2023-06-10T21:45:54.272154+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":35,"log_dups_size":0,"ondisk_log_size":35,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293760,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":35,"num_write_kb":2240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513500+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:45:45.513500+0000","last_peered":"2023-06-10T21:45:45.513500+0000","last_clean":"2023-06-10T21:45:45.513500+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:45:45.513500+0000","last_undegraded":"2023-06-10T21:45:45.513500+0000","last_fullsized":"2023-06-10T21:45:45.513500+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":371,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514640+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:45:45.514640+0000","last_peered":"2023-06-10T21:45:45.514640+0000","last_clean":"2023-06-10T21:45:45.514640+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:45:45.514640+0000","last_undegraded":"2023-06-10T21:45:45.514640+0000","last_fullsized":"2023-06-10T21:45:45.514640+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"424'390","reported_seq":403,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.245078+0000","last_change":"2023-06-10T21:45:44.477475+0000","last_active":"2023-06-10T21:45:56.245078+0000","last_peered":"2023-06-10T21:45:56.245078+0000","last_clean":"2023-06-10T21:45:56.245078+0000","last_became_active":"2023-06-10T21:45:44.477218+0000","last_became_peered":"2023-06-10T21:45:44.477218+0000","last_unstale":"2023-06-10T21:45:56.245078+0000","last_undegraded":"2023-06-10T21:45:56.245078+0000","last_fullsized":"2023-06-10T21:45:56.245078+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":390,"log_dups_size":0,"ondisk_log_size":390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:15:26.255634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25559040,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":390,"num_write_kb":24960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":206,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513431+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:45:45.513431+0000","last_peered":"2023-06-10T21:45:45.513431+0000","last_clean":"2023-06-10T21:45:45.513431+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:45:45.513431+0000","last_undegraded":"2023-06-10T21:45:45.513431+0000","last_fullsized":"2023-06-10T21:45:45.513431+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470588+0000","last_change":"2023-06-10T21:43:01.217304+0000","last_active":"2023-06-10T21:45:45.470588+0000","last_peered":"2023-06-10T21:45:45.470588+0000","last_clean":"2023-06-10T21:45:45.470588+0000","last_became_active":"2023-06-10T21:43:01.216925+0000","last_became_peered":"2023-06-10T21:43:01.216925+0000","last_unstale":"2023-06-10T21:45:45.470588+0000","last_undegraded":"2023-06-10T21:45:45.470588+0000","last_fullsized":"2023-06-10T21:45:45.470588+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470109+0000","last_change":"2023-06-10T21:43:01.229656+0000","last_active":"2023-06-10T21:45:45.470109+0000","last_peered":"2023-06-10T21:45:45.470109+0000","last_clean":"2023-06-10T21:45:45.470109+0000","last_became_active":"2023-06-10T21:43:01.229293+0000","last_became_peered":"2023-06-10T21:43:01.229293+0000","last_unstale":"2023-06-10T21:45:45.470109+0000","last_undegraded":"2023-06-10T21:45:45.470109+0000","last_fullsized":"2023-06-10T21:45:45.470109+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.e","version":"424'401","reported_seq":414,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.256607+0000","last_change":"2023-06-10T21:45:44.480812+0000","last_active":"2023-06-10T21:45:56.256607+0000","last_peered":"2023-06-10T21:45:56.256607+0000","last_clean":"2023-06-10T21:45:56.256607+0000","last_became_active":"2023-06-10T21:45:44.479574+0000","last_became_peered":"2023-06-10T21:45:44.479574+0000","last_unstale":"2023-06-10T21:45:56.256607+0000","last_undegraded":"2023-06-10T21:45:56.256607+0000","last_fullsized":"2023-06-10T21:45:56.256607+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":401,"log_dups_size":0,"ondisk_log_size":401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:01:32.420364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26279936,"num_objects":401,"num_object_clones":0,"num_object_copies":1203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":401,"num_write_kb":25664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":106,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470044+0000","last_change":"2023-06-10T21:43:01.229461+0000","last_active":"2023-06-10T21:45:45.470044+0000","last_peered":"2023-06-10T21:45:45.470044+0000","last_clean":"2023-06-10T21:45:45.470044+0000","last_became_active":"2023-06-10T21:43:01.228851+0000","last_became_peered":"2023-06-10T21:43:01.228851+0000","last_unstale":"2023-06-10T21:45:45.470044+0000","last_undegraded":"2023-06-10T21:45:45.470044+0000","last_fullsized":"2023-06-10T21:45:45.470044+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":384,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470106+0000","last_change":"2023-06-10T21:43:01.216428+0000","last_active":"2023-06-10T21:45:45.470106+0000","last_peered":"2023-06-10T21:45:45.470106+0000","last_clean":"2023-06-10T21:45:45.470106+0000","last_became_active":"2023-06-10T21:43:01.211445+0000","last_became_peered":"2023-06-10T21:43:01.211445+0000","last_unstale":"2023-06-10T21:45:45.470106+0000","last_undegraded":"2023-06-10T21:45:45.470106+0000","last_fullsized":"2023-06-10T21:45:45.470106+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":289,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.273152+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:45:51.273152+0000","last_peered":"2023-06-10T21:45:51.273152+0000","last_clean":"2023-06-10T21:45:51.273152+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:45:51.273152+0000","last_undegraded":"2023-06-10T21:45:51.273152+0000","last_fullsized":"2023-06-10T21:45:51.273152+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"423'131","reported_seq":142,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.785831+0000","last_change":"2023-06-10T21:45:44.477165+0000","last_active":"2023-06-10T21:45:54.785831+0000","last_peered":"2023-06-10T21:45:54.785831+0000","last_clean":"2023-06-10T21:45:54.785831+0000","last_became_active":"2023-06-10T21:45:44.476825+0000","last_became_peered":"2023-06-10T21:45:44.476825+0000","last_unstale":"2023-06-10T21:45:54.785831+0000","last_undegraded":"2023-06-10T21:45:54.785831+0000","last_fullsized":"2023-06-10T21:45:54.785831+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":131,"log_dups_size":0,"ondisk_log_size":131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8585216,"num_objects":131,"num_object_clones":0,"num_object_copies":393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":131,"num_write_kb":8384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":309,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272424+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:45:51.272424+0000","last_peered":"2023-06-10T21:45:51.272424+0000","last_clean":"2023-06-10T21:45:51.272424+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:45:51.272424+0000","last_undegraded":"2023-06-10T21:45:51.272424+0000","last_fullsized":"2023-06-10T21:45:51.272424+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":314,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.272202+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:45:51.272202+0000","last_peered":"2023-06-10T21:45:51.272202+0000","last_clean":"2023-06-10T21:45:51.272202+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:45:51.272202+0000","last_undegraded":"2023-06-10T21:45:51.272202+0000","last_fullsized":"2023-06-10T21:45:51.272202+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":105,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470915+0000","last_change":"2023-06-10T21:43:01.231390+0000","last_active":"2023-06-10T21:45:45.470915+0000","last_peered":"2023-06-10T21:45:45.470915+0000","last_clean":"2023-06-10T21:45:45.470915+0000","last_became_active":"2023-06-10T21:43:01.225737+0000","last_became_peered":"2023-06-10T21:43:01.225737+0000","last_unstale":"2023-06-10T21:45:45.470915+0000","last_undegraded":"2023-06-10T21:45:45.470915+0000","last_fullsized":"2023-06-10T21:45:45.470915+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.f","version":"423'25","reported_seq":36,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.272660+0000","last_change":"2023-06-10T21:45:44.480803+0000","last_active":"2023-06-10T21:45:54.272660+0000","last_peered":"2023-06-10T21:45:54.272660+0000","last_clean":"2023-06-10T21:45:54.272660+0000","last_became_active":"2023-06-10T21:45:44.480472+0000","last_became_peered":"2023-06-10T21:45:44.480472+0000","last_unstale":"2023-06-10T21:45:54.272660+0000","last_undegraded":"2023-06-10T21:45:54.272660+0000","last_fullsized":"2023-06-10T21:45:54.272660+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":25,"log_dups_size":0,"ondisk_log_size":25,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":25,"num_write_kb":1600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":97,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470296+0000","last_change":"2023-06-10T21:43:01.229767+0000","last_active":"2023-06-10T21:45:45.470296+0000","last_peered":"2023-06-10T21:45:45.470296+0000","last_clean":"2023-06-10T21:45:45.470296+0000","last_became_active":"2023-06-10T21:43:01.229472+0000","last_became_peered":"2023-06-10T21:43:01.229472+0000","last_unstale":"2023-06-10T21:45:45.470296+0000","last_undegraded":"2023-06-10T21:45:45.470296+0000","last_fullsized":"2023-06-10T21:45:45.470296+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":623,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.512614+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:45:45.512614+0000","last_peered":"2023-06-10T21:45:45.512614+0000","last_clean":"2023-06-10T21:45:45.512614+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:45:45.512614+0000","last_undegraded":"2023-06-10T21:45:45.512614+0000","last_fullsized":"2023-06-10T21:45:45.512614+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513944+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:45:45.513944+0000","last_peered":"2023-06-10T21:45:45.513944+0000","last_clean":"2023-06-10T21:45:45.513944+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:45:45.513944+0000","last_undegraded":"2023-06-10T21:45:45.513944+0000","last_fullsized":"2023-06-10T21:45:45.513944+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":306,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513557+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:45:45.513557+0000","last_peered":"2023-06-10T21:45:45.513557+0000","last_clean":"2023-06-10T21:45:45.513557+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:45:45.513557+0000","last_undegraded":"2023-06-10T21:45:45.513557+0000","last_fullsized":"2023-06-10T21:45:45.513557+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"424'393","reported_seq":406,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.250777+0000","last_change":"2023-06-10T21:45:44.478367+0000","last_active":"2023-06-10T21:45:56.250777+0000","last_peered":"2023-06-10T21:45:56.250777+0000","last_clean":"2023-06-10T21:45:56.250777+0000","last_became_active":"2023-06-10T21:45:44.476855+0000","last_became_peered":"2023-06-10T21:45:44.476855+0000","last_unstale":"2023-06-10T21:45:56.250777+0000","last_undegraded":"2023-06-10T21:45:56.250777+0000","last_fullsized":"2023-06-10T21:45:56.250777+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":393,"log_dups_size":0,"ondisk_log_size":393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:43.311166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25755648,"num_objects":393,"num_object_clones":0,"num_object_copies":1179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":393,"num_write_kb":25152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":475,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.512618+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:45:45.512618+0000","last_peered":"2023-06-10T21:45:45.512618+0000","last_clean":"2023-06-10T21:45:45.512618+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:45:45.512618+0000","last_undegraded":"2023-06-10T21:45:45.512618+0000","last_fullsized":"2023-06-10T21:45:45.512618+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513751+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:45:45.513751+0000","last_peered":"2023-06-10T21:45:45.513751+0000","last_clean":"2023-06-10T21:45:45.513751+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:45:45.513751+0000","last_undegraded":"2023-06-10T21:45:45.513751+0000","last_fullsized":"2023-06-10T21:45:45.513751+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":175,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.273150+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:45:51.273150+0000","last_peered":"2023-06-10T21:45:51.273150+0000","last_clean":"2023-06-10T21:45:51.273150+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:45:51.273150+0000","last_undegraded":"2023-06-10T21:45:51.273150+0000","last_fullsized":"2023-06-10T21:45:51.273150+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"423'129","reported_seq":140,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.784102+0000","last_change":"2023-06-10T21:45:44.474931+0000","last_active":"2023-06-10T21:45:54.784102+0000","last_peered":"2023-06-10T21:45:54.784102+0000","last_clean":"2023-06-10T21:45:54.784102+0000","last_became_active":"2023-06-10T21:45:44.474576+0000","last_became_peered":"2023-06-10T21:45:44.474576+0000","last_unstale":"2023-06-10T21:45:54.784102+0000","last_undegraded":"2023-06-10T21:45:54.784102+0000","last_fullsized":"2023-06-10T21:45:54.784102+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":129,"log_dups_size":0,"ondisk_log_size":129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8454144,"num_objects":129,"num_object_clones":0,"num_object_copies":387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":129,"num_write_kb":8256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":454,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513576+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:45:45.513576+0000","last_peered":"2023-06-10T21:45:45.513576+0000","last_clean":"2023-06-10T21:45:45.513576+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:45:45.513576+0000","last_undegraded":"2023-06-10T21:45:45.513576+0000","last_fullsized":"2023-06-10T21:45:45.513576+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":304,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.273221+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:45:51.273221+0000","last_peered":"2023-06-10T21:45:51.273221+0000","last_clean":"2023-06-10T21:45:51.273221+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:45:51.273221+0000","last_undegraded":"2023-06-10T21:45:51.273221+0000","last_fullsized":"2023-06-10T21:45:51.273221+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":227,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470131+0000","last_change":"2023-06-10T21:43:01.225747+0000","last_active":"2023-06-10T21:45:45.470131+0000","last_peered":"2023-06-10T21:45:45.470131+0000","last_clean":"2023-06-10T21:45:45.470131+0000","last_became_active":"2023-06-10T21:43:01.224045+0000","last_became_peered":"2023-06-10T21:43:01.224045+0000","last_unstale":"2023-06-10T21:45:45.470131+0000","last_undegraded":"2023-06-10T21:45:45.470131+0000","last_fullsized":"2023-06-10T21:45:45.470131+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"423'150","reported_seq":161,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:54.770842+0000","last_change":"2023-06-10T21:45:44.480921+0000","last_active":"2023-06-10T21:45:54.770842+0000","last_peered":"2023-06-10T21:45:54.770842+0000","last_clean":"2023-06-10T21:45:54.770842+0000","last_became_active":"2023-06-10T21:45:44.480408+0000","last_became_peered":"2023-06-10T21:45:44.480408+0000","last_unstale":"2023-06-10T21:45:54.770842+0000","last_undegraded":"2023-06-10T21:45:54.770842+0000","last_fullsized":"2023-06-10T21:45:54.770842+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":150,"log_dups_size":0,"ondisk_log_size":150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":150,"num_write_kb":9600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":433,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514563+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:45:45.514563+0000","last_peered":"2023-06-10T21:45:45.514563+0000","last_clean":"2023-06-10T21:45:45.514563+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:45:45.514563+0000","last_undegraded":"2023-06-10T21:45:45.514563+0000","last_fullsized":"2023-06-10T21:45:45.514563+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":244,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514414+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:45:45.514414+0000","last_peered":"2023-06-10T21:45:45.514414+0000","last_clean":"2023-06-10T21:45:45.514414+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:45:45.514414+0000","last_undegraded":"2023-06-10T21:45:45.514414+0000","last_fullsized":"2023-06-10T21:45:45.514414+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513447+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:45:45.513447+0000","last_peered":"2023-06-10T21:45:45.513447+0000","last_clean":"2023-06-10T21:45:45.513447+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:45:45.513447+0000","last_undegraded":"2023-06-10T21:45:45.513447+0000","last_fullsized":"2023-06-10T21:45:45.513447+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"424'402","reported_seq":415,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-06-10T21:45:56.251691+0000","last_change":"2023-06-10T21:45:44.477091+0000","last_active":"2023-06-10T21:45:56.251691+0000","last_peered":"2023-06-10T21:45:56.251691+0000","last_clean":"2023-06-10T21:45:56.251691+0000","last_became_active":"2023-06-10T21:45:44.476764+0000","last_became_peered":"2023-06-10T21:45:44.476764+0000","last_unstale":"2023-06-10T21:45:56.251691+0000","last_undegraded":"2023-06-10T21:45:56.251691+0000","last_fullsized":"2023-06-10T21:45:56.251691+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":402,"log_dups_size":0,"ondisk_log_size":402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:40:54.289558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26345472,"num_objects":402,"num_object_clones":0,"num_object_copies":1206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":402,"num_write_kb":25728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":171,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513776+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:45:45.513776+0000","last_peered":"2023-06-10T21:45:45.513776+0000","last_clean":"2023-06-10T21:45:45.513776+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:45:45.513776+0000","last_undegraded":"2023-06-10T21:45:45.513776+0000","last_fullsized":"2023-06-10T21:45:45.513776+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":285,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513430+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:45:45.513430+0000","last_peered":"2023-06-10T21:45:45.513430+0000","last_clean":"2023-06-10T21:45:45.513430+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:45:45.513430+0000","last_undegraded":"2023-06-10T21:45:45.513430+0000","last_fullsized":"2023-06-10T21:45:45.513430+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.469706+0000","last_change":"2023-06-10T21:43:01.229745+0000","last_active":"2023-06-10T21:45:45.469706+0000","last_peered":"2023-06-10T21:45:45.469706+0000","last_clean":"2023-06-10T21:45:45.469706+0000","last_became_active":"2023-06-10T21:43:01.229445+0000","last_became_peered":"2023-06-10T21:43:01.229445+0000","last_unstale":"2023-06-10T21:45:45.469706+0000","last_undegraded":"2023-06-10T21:45:45.469706+0000","last_fullsized":"2023-06-10T21:45:45.469706+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":122,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513191+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:45:45.513191+0000","last_peered":"2023-06-10T21:45:45.513191+0000","last_clean":"2023-06-10T21:45:45.513191+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:45:45.513191+0000","last_undegraded":"2023-06-10T21:45:45.513191+0000","last_fullsized":"2023-06-10T21:45:45.513191+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":321,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-06-10T21:45:51.273286+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:45:51.273286+0000","last_peered":"2023-06-10T21:45:51.273286+0000","last_clean":"2023-06-10T21:45:51.273286+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:45:51.273286+0000","last_undegraded":"2023-06-10T21:45:51.273286+0000","last_fullsized":"2023-06-10T21:45:51.273286+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":85,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470459+0000","last_change":"2023-06-10T21:43:01.227816+0000","last_active":"2023-06-10T21:45:45.470459+0000","last_peered":"2023-06-10T21:45:45.470459+0000","last_clean":"2023-06-10T21:45:45.470459+0000","last_became_active":"2023-06-10T21:43:01.227005+0000","last_became_peered":"2023-06-10T21:43:01.227005+0000","last_unstale":"2023-06-10T21:45:45.470459+0000","last_undegraded":"2023-06-10T21:45:45.470459+0000","last_fullsized":"2023-06-10T21:45:45.470459+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":327,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514151+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:45:45.514151+0000","last_peered":"2023-06-10T21:45:45.514151+0000","last_clean":"2023-06-10T21:45:45.514151+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:45:45.514151+0000","last_undegraded":"2023-06-10T21:45:45.514151+0000","last_fullsized":"2023-06-10T21:45:45.514151+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":108,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470576+0000","last_change":"2023-06-10T21:43:01.227292+0000","last_active":"2023-06-10T21:45:45.470576+0000","last_peered":"2023-06-10T21:45:45.470576+0000","last_clean":"2023-06-10T21:45:45.470576+0000","last_became_active":"2023-06-10T21:43:01.225901+0000","last_became_peered":"2023-06-10T21:43:01.225901+0000","last_unstale":"2023-06-10T21:45:45.470576+0000","last_undegraded":"2023-06-10T21:45:45.470576+0000","last_fullsized":"2023-06-10T21:45:45.470576+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":398,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514283+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:45:45.514283+0000","last_peered":"2023-06-10T21:45:45.514283+0000","last_clean":"2023-06-10T21:45:45.514283+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:45:45.514283+0000","last_undegraded":"2023-06-10T21:45:45.514283+0000","last_fullsized":"2023-06-10T21:45:45.514283+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":77,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.514385+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:45:45.514385+0000","last_peered":"2023-06-10T21:45:45.514385+0000","last_clean":"2023-06-10T21:45:45.514385+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:45:45.514385+0000","last_undegraded":"2023-06-10T21:45:45.514385+0000","last_fullsized":"2023-06-10T21:45:45.514385+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513790+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:45:45.513790+0000","last_peered":"2023-06-10T21:45:45.513790+0000","last_clean":"2023-06-10T21:45:45.513790+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:45:45.513790+0000","last_undegraded":"2023-06-10T21:45:45.513790+0000","last_fullsized":"2023-06-10T21:45:45.513790+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":344,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.470986+0000","last_change":"2023-06-10T21:43:01.211590+0000","last_active":"2023-06-10T21:45:45.470986+0000","last_peered":"2023-06-10T21:45:45.470986+0000","last_clean":"2023-06-10T21:45:45.470986+0000","last_became_active":"2023-06-10T21:43:01.210849+0000","last_became_peered":"2023-06-10T21:43:01.210849+0000","last_unstale":"2023-06-10T21:45:45.470986+0000","last_undegraded":"2023-06-10T21:45:45.470986+0000","last_fullsized":"2023-06-10T21:45:45.470986+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":92,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-06-10T21:45:45.513532+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:45:45.513532+0000","last_peered":"2023-06-10T21:45:45.513532+0000","last_clean":"2023-06-10T21:45:45.513532+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:45:45.513532+0000","last_undegraded":"2023-06-10T21:45:45.513532+0000","last_fullsized":"2023-06-10T21:45:45.513532+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4060,"num_write_kb":259840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":703463424,"data_stored":703463424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4060,"ondisk_log_size":4060,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430786,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997736,"kb_used_data":13664,"kb_used_omap":0,"kb_used_meta":984064,"kb_avail":92747416,"statfs":{"total":95995035648,"available":94973353984,"internally_reserved":0,"allocated":13991936,"data_stored":13279677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1007681536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.272,"15min":2.5510000000000002},"min":{"1min":0.29099999999999998,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":8.5129999999999999,"5min":22.277000000000001,"15min":184.83699999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.2949999999999999,"5min":1.5820000000000001,"15min":2.8540000000000001},"min":{"1min":0.28799999999999998,"5min":0.28199999999999997,"15min":0.23899999999999999},"max":{"1min":20.568999999999999,"5min":22.462,"15min":185.58000000000001},"last":0.49299999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.6850000000000001,"15min":2.6640000000000001},"min":{"1min":0.29699999999999999,"5min":0.26700000000000002,"15min":0.25600000000000001},"max":{"1min":4.9859999999999998,"5min":38.078000000000003,"15min":87.417000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.4970000000000001,"5min":2.0179999999999998,"15min":2.9790000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":4.8739999999999997,"5min":38.341000000000001,"15min":87.438000000000002},"last":0.60899999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:45:44 2023","interfaces":[{"interface":"back","average":{"1min":5.9900000000000002,"5min":3.8359999999999999,"15min":5.1550000000000002},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":90.212999999999994,"5min":90.212999999999994,"15min":172.33600000000001},"last":0.625},{"interface":"front","average":{"1min":5.9009999999999998,"5min":3.9590000000000001,"15min":5.2210000000000001},"min":{"1min":0.41699999999999998,"5min":0.33900000000000002,"15min":0.30499999999999999},"max":{"1min":90.361999999999995,"5min":90.361999999999995,"15min":172.185},"last":0.69099999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":3.9430000000000001,"15min":4.7370000000000001},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.27000000000000002},"max":{"1min":5.0199999999999996,"5min":89.504999999999995,"15min":147.00700000000001},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.655,"5min":3.7120000000000002,"15min":4.5149999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":4.931,"5min":90.215999999999994,"15min":146.93899999999999},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.1080000000000001,"15min":0.66600000000000004},"min":{"1min":0.315,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":4.8479999999999999,"5min":13.429,"15min":13.429},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.645,"5min":1.266,"15min":0.82399999999999995},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":12.279999999999999,"5min":12.279999999999999,"15min":22.77},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Jun 10 21:45:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2440000000000002,"5min":10.191000000000001,"15min":11.515000000000001},"min":{"1min":0.34799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.984,"5min":131.619,"15min":131.619},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.036,"5min":10.006,"15min":11.334},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.9370000000000003,"5min":134.928,"15min":134.928},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:45:11 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.0189999999999999,"15min":0.74399999999999999},"min":{"1min":0.371,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.9660000000000002,"5min":7.9660000000000002,"15min":10.890000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.2210000000000001,"15min":0.76600000000000001},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":8.1020000000000003,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.73699999999999999}]}]},{"osd":6,"up_from":0,"seq":180388627014,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566457,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020433,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075845,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1294784,"kb_used_data":261368,"kb_used_omap":7,"kb_used_meta":1033336,"kb_avail":92450368,"statfs":{"total":95995035648,"available":94669176832,"internally_reserved":0,"allocated":267640832,"data_stored":266276585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1058136145},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":2.375,"15min":4.2670000000000003},"min":{"1min":0.23400000000000001,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":6.6429999999999998,"5min":83.733000000000004,"15min":171.33699999999999},"last":1.581},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.9690000000000001,"15min":4.0640000000000001},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9300000000000002,"5min":82.322999999999993,"15min":171.56200000000001},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.4089999999999998,"15min":5.0590000000000002},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":11.398999999999999,"5min":85.709999999999994,"15min":171.286},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.478,"5min":2.71,"15min":3.7530000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.9119999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":1.7430000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:45:25 2023","interfaces":[{"interface":"back","average":{"1min":5.1600000000000001,"5min":8.7620000000000005,"15min":6.665},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":25.725000000000001,"5min":145.07599999999999,"15min":171.78299999999999},"last":1.8819999999999999},{"interface":"front","average":{"1min":4.1310000000000002,"5min":8.3109999999999999,"15min":6.4779999999999998},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":25.626999999999999,"5min":144.99000000000001,"15min":171.65100000000001},"last":0.625}]},{"osd":4,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.3140000000000001,"15min":2.1859999999999999},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.28100000000000003},"max":{"1min":6.3259999999999996,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.504},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.423,"15min":2.9969999999999999},"min":{"1min":0.41699999999999998,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.8150000000000004,"5min":5.8150000000000004,"15min":171.18100000000001},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:45:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":5.9989999999999997,"15min":6.6539999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":6.0389999999999997,"5min":75.896000000000001,"15min":75.896000000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":1.976,"5min":5.468,"15min":6.0499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":7.3460000000000001,"5min":76.337000000000003,"15min":76.337000000000003},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.603,"15min":2.3290000000000002},"min":{"1min":0.44400000000000001,"5min":0.41999999999999998,"15min":0.307},"max":{"1min":6.3879999999999999,"5min":9.0139999999999993,"15min":171.441},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.55,"15min":3.0819999999999999},"min":{"1min":0.46500000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":3.6320000000000001,"5min":4.9199999999999999,"15min":171.24600000000001},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":8.1129999999999995,"5min":3.2330000000000001,"15min":4.4729999999999999},"min":{"1min":0.46600000000000003,"5min":0.46500000000000002,"15min":0.34200000000000003},"max":{"1min":124.071,"5min":124.071,"15min":219.131},"last":1.7050000000000001},{"interface":"front","average":{"1min":8.0570000000000004,"5min":3.008,"15min":4.3579999999999997},"min":{"1min":0.504,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":125.203,"5min":125.203,"15min":219.19},"last":0.59399999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824205,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1006532,"kb_used_data":269436,"kb_used_omap":7,"kb_used_meta":737080,"kb_avail":92738620,"statfs":{"total":95995035648,"available":94964346880,"internally_reserved":0,"allocated":275902464,"data_stored":274407162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":754769939},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.1499999999999999,"15min":2.9460000000000002},"min":{"1min":0.254,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":3.4369999999999998,"5min":30.744,"15min":105.09},"last":2.3969999999999998},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.1160000000000001,"15min":3.153},"min":{"1min":0.26400000000000001,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":6.3479999999999999,"5min":30.805,"15min":99.852999999999994},"last":0.501}]},{"osd":1,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.45,"15min":3.21},"min":{"1min":0.247,"5min":0.23599999999999999,"15min":0.20499999999999999},"max":{"1min":6.5030000000000001,"5min":7.6509999999999998,"15min":94.766000000000005},"last":2.1179999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.028,"15min":3.0939999999999999},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":2.7519999999999998,"5min":8.9649999999999999,"15min":94.691000000000003},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":9.6219999999999999,"5min":3.6019999999999999,"15min":4.0990000000000002},"min":{"1min":0.28599999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":126.913,"5min":126.913,"15min":231.07900000000001},"last":1.204},{"interface":"front","average":{"1min":9.8379999999999992,"5min":3.8399999999999999,"15min":5.2599999999999998},"min":{"1min":0.34499999999999997,"5min":0.26800000000000002,"15min":0.22800000000000001},"max":{"1min":128.52199999999999,"5min":128.52199999999999,"15min":231.58600000000001},"last":0.95099999999999996}]},{"osd":4,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":1.897,"5min":1.714,"15min":1.5209999999999999},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":5.5620000000000003,"5min":9.5150000000000006,"15min":13.417},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.0790000000000002,"5min":1.6639999999999999,"15min":2.2160000000000002},"min":{"1min":0.36399999999999999,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":8.8379999999999992,"5min":8.8379999999999992,"15min":19.728000000000002},"last":1.0800000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:45:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.097,"15min":1.0920000000000001},"min":{"1min":0.37,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.6269999999999998,"5min":6.532,"15min":6.532},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.2679999999999998,"5min":2.6160000000000001,"15min":2.6739999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.1130000000000004,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":1.8480000000000001,"15min":2.3039999999999998},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":9.0250000000000004,"5min":9.0250000000000004,"15min":19.805},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.232,"15min":1.5449999999999999},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.0750000000000002,"5min":6.0750000000000002,"15min":17.327000000000002},"last":0.90400000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:45:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":2.1360000000000001,"15min":2.6179999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":8.9920000000000009,"5min":8.9920000000000009,"15min":79.984999999999999},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.2909999999999999,"5min":1.335,"15min":1.839},"min":{"1min":0.45700000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6.1289999999999996,"5min":6.1289999999999996,"15min":80.272999999999996},"last":0.72799999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019470,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245476,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":256864,"kb_used_data":3736,"kb_used_omap":11,"kb_used_meta":253108,"kb_avail":93488288,"statfs":{"total":95995035648,"available":95732006912,"internally_reserved":0,"allocated":3825664,"data_stored":3121202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":259183094},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":2.0579999999999998,"15min":2.0720000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.2590000000000003,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.5209999999999999,"5min":1.361,"15min":1.335},"min":{"1min":0.28999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.3170000000000002,"5min":4.266,"15min":4.266},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.6269999999999998,"15min":2.6480000000000001},"min":{"1min":0.30099999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.441},{"interface":"front","average":{"1min":1.9910000000000001,"5min":2.2229999999999999,"15min":2.262},"min":{"1min":0.33800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.9500000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":9.5090000000000003,"5min":3.391,"15min":2.371},"min":{"1min":0.27700000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":159.94999999999999,"5min":159.94999999999999,"15min":159.94999999999999},"last":4.335},{"interface":"front","average":{"1min":9.3230000000000004,"5min":3.7090000000000001,"15min":2.7730000000000001},"min":{"1min":0.32600000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":159.851,"5min":159.851,"15min":159.851},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":7.569,"15min":8.0109999999999992},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":54.920000000000002,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":5.3339999999999996,"5min":7.3819999999999997,"15min":7.7240000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":54.848999999999997,"5min":133.03800000000001,"15min":133.03800000000001},"last":1.1779999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":0.96199999999999997,"15min":0.92600000000000005},"min":{"1min":0.312,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.4329999999999998,"5min":5.173,"15min":5.173},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":2.1099999999999999,"15min":2.242},"min":{"1min":0.33500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.238,"5min":7.476,"15min":7.476},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.752,"15min":1.8420000000000001},"min":{"1min":0.33900000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.2149999999999999,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.173,"5min":1.3100000000000001,"15min":1.333},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":4.2910000000000004,"5min":5.4509999999999996,"15min":5.4509999999999996},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Jun 10 21:45:04 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.625,"15min":1.6160000000000001},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.2430000000000003,"5min":6.5670000000000002,"15min":6.5670000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.9630000000000001,"5min":2.9049999999999998,"15min":2.895},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.1920000000000002,"5min":9.4160000000000004,"15min":9.4160000000000004},"last":0.71399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":263258112,"data_stored":263258112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":288161792,"data_stored":288161792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":131203072,"data_stored":131203072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20840448,"data_stored":20840448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:46:04.068 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:46:04.069 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:46:04.069 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-10T21:46:04.069 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-06-10T21:46:04.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 5 0 2023-06-10T21:46:04.112 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:04.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:04.188 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 10 16 27259 27243 170.001 129.5 0.00521725 0.00587657 2023-06-10T21:46:04.218 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:04.309 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:04.370 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:46:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:46:04.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:46:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:46:04.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:04.448 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:04.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:04.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:04.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:04.606 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:46:04.606 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:46:04.606 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:04.614 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:04.694 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:04.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:04.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:04.805 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:04.838 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:46:04.838 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104494 12.9 12:9ca311c5:::benchmark_data_smithi064_442287_object16444:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:46:04.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.64226738100001, 2023-06-10T21:46:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6361784579999998, 2023-06-10T21:46:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104494 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:46:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177997+0000", 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.177995+0000", 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178006+0000", 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1514e-05 2023-06-10T21:46:04.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178024+0000", 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7651000000000001e-05 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.178035+0000", 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0857e-05 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.188934+0000", 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010898873 2023-06-10T21:46:04.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189075+0000", 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014062699999999999 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,3", 2023-06-10T21:46:04.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.189180+0000", 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105855 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.196756+0000", 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075760269999999999 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207957+0000", 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011200676999999999 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245880+0000", 2023-06-10T21:46:04.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.037923327999999999 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.245895+0000", 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4575e-05 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200093+0000", 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95419813399999998 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200108+0000", 2023-06-10T21:46:04.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5024e-05 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754247+0000", 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5541385060000001 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.754441+0000", 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019410900000000001 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814067+0000", 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059626237999999998 2023-06-10T21:46:04.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814176+0000", 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010860400000000001 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104503 12.a 12:53f91a42:::benchmark_data_smithi064_442287_object16453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.61436693899998, 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.527545849, 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.855 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104503 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205898+0000", 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205896+0000", 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205904+0000", 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9699999999999999e-06 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205916+0000", 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2194999999999999e-05 2023-06-10T21:46:04.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.205924+0000", 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5380000000000002e-06 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218750+0000", 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012825721999999999 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.218889+0000", 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138617 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3", 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.219004+0000", 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011551 2023-06-10T21:46:04.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.224321+0000", 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053164609999999998 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246026+0000", 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021705486 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.246036+0000", 2023-06-10T21:46:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0294e-05 2023-06-10T21:46:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202839+0000", 2023-06-10T21:46:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95680261799999999 2023-06-10T21:46:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.202852+0000", 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2459999999999999e-05 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636477+0000", 2023-06-10T21:46:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.433625159 2023-06-10T21:46:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:46:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.636649+0000", 2023-06-10T21:46:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017257199999999999 2023-06-10T21:46:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733342+0000", 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.096692527 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.733443+0000", 2023-06-10T21:46:04.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101641 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104492 12.8 12:145682b0:::benchmark_data_smithi064_442287_object16442:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e330)", 2023-06-10T21:46:04.871 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.58957150100002, 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1766732329999998, 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.872 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104492 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.873 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230693+0000", 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230691+0000", 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.230699+0000", 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0790000000000001e-06 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.237985+0000", 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0072853450000000004 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.238000+0000", 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5784999999999999e-05 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258465+0000", 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020465041999999999 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.258473+0000", 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9489999999999996e-06 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257243+0000", 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99876928499999995 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257255+0000", 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2431e-05 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.405789+0000", 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.148533523 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407366+0000", 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001577572 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104510 12.9 12:91eab998:::benchmark_data_smithi064_442287_object16460:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.55549100000002, 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5536998, 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.882 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104510 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:46:04.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264774+0000", 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264772+0000", 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264780+0000", 2023-06-10T21:46:04.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6229999999999997e-06 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264792+0000", 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1987999999999999e-05 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264800+0000", 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8189999999999998e-06 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264846+0000", 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5782000000000003e-05 2023-06-10T21:46:04.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.264860+0000", 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4267000000000001e-05 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200139+0000", 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93527848099999999 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.200148+0000", 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7260000000000008e-06 2023-06-10T21:46:04.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758081+0000", 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5579332859999999 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758234+0000", 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015339999999999999 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1", 2023-06-10T21:46:04.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.758918+0000", 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00068356899999999997 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.784273+0000", 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025354811000000001 2023-06-10T21:46:04.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.785804+0000", 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00153134 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814218+0000", 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028414053000000002 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.814269+0000", 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.075e-05 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.818473+0000", 2023-06-10T21:46:04.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042044639999999998 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.896 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104512 12.8 12:1b4cbe21:::benchmark_data_smithi064_442287_object16462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.54892192, 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1733947009999999, 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.897 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104512 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:46:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271343+0000", 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271341+0000", 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271350+0000", 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8789999999999998e-06 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271394+0000", 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.384e-05 2023-06-10T21:46:04.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271404+0000", 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6309999999999998e-06 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271472+0000", 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7986999999999998e-05 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.271488+0000", 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5911000000000001e-05 2023-06-10T21:46:04.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257279+0000", 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98579087099999996 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257287+0000", 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9410000000000007e-06 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.407591+0000", 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15030481 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408064+0000", 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047220399999999999 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.408146+0000", 2023-06-10T21:46:04.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2446000000000003e-05 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420391+0000", 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012245285999999999 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423446+0000", 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030550500000000001 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429481+0000", 2023-06-10T21:46:04.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060345260000000001 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444541+0000", 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015059771 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444604+0000", 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3081000000000001e-05 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444737+0000", 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000133468 2023-06-10T21:46:04.911 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104514 12.8 12:1ddd6575:::benchmark_data_smithi064_442287_object16464:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:46:04.912 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 463.541358355, 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1660305000000002, 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104514 2023-06-10T21:46:04.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278906+0000", 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278905+0000", 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:46:04.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278910+0000", 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2499999999999997e-06 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.916 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278921+0000", 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981999999999999e-05 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278927+0000", 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5470000000000004e-06 2023-06-10T21:46:04.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.278994+0000", 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6595999999999997e-05 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.279006+0000", 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2479e-05 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257307+0000", 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97830109899999995 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.257315+0000", 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3119999999999994e-06 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410688+0000", 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.15337278 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410832+0000", 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014341000000000001 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.410967+0000", 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135816 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.420402+0000", 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094344049999999999 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.423489+0000", 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030871060000000001 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.429518+0000", 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060294930000000004 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444782+0000", 2023-06-10T21:46:04.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015263465 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444832+0000", 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9935999999999998e-05 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.444937+0000", 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010481599999999999 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117672 12.10 12:08d95f39:::benchmark_data_smithi064_442287_object29622:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 420.60907022700002, 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9927129360000002, 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.927 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117672 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:46:04.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.929 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211194+0000", 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211197+0000", 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8619999999999999e-06 2023-06-10T21:46:04.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211203+0000", 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0959999999999997e-06 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211208+0000", 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.412e-06 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560435+0000", 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922703799999999 2023-06-10T21:46:04.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560446+0000", 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0944999999999999e-05 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075800+0000", 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515354077 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075806+0000", 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0939999999999996e-06 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203794+0000", 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1279870999999999 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.203907+0000", 2023-06-10T21:46:04.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113702 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117682 12.18 12:18fc4302:::benchmark_data_smithi064_442287_object29632:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:46:04.936 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 420.60901522699999, 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.083346562, 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.937 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117682 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211249+0000", 2023-06-10T21:46:04.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211252+0000", 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7800000000000001e-06 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211258+0000", 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.404e-06 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211263+0000", 2023-06-10T21:46:04.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1460000000000004e-06 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560489+0000", 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34922595299999998 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560495+0000", 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2299999999999996e-06 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075893+0000", 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.515397806 2023-06-10T21:46:04.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075905+0000", 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1467e-05 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.944 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183546+0000", 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1076415719999999 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183688+0000", 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000141898 2023-06-10T21:46:04.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,7", 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.183979+0000", 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029059499999999999 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.186643+0000", 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026644519999999999 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.231646+0000", 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.045002563000000002 2023-06-10T21:46:04.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294378+0000", 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.062732531999999994 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294444+0000", 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5165000000000004e-05 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294496+0000", 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2339999999999997e-05 2023-06-10T21:46:04.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294596+0000", 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9959999999999998e-05 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117683 12.10 12:0c4ca780:::benchmark_data_smithi064_442287_object29633:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 420.60899633700001, 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.12118121, 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.951 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117683 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.952 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211268+0000", 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211270+0000", 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2350000000000002e-06 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211276+0000", 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.587e-06 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211281+0000", 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2000000000000002e-06 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560465+0000", 2023-06-10T21:46:04.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34918397899999998 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.560471+0000", 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7729999999999996e-06 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075818+0000", 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5153477930000001 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075823+0000", 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8350000000000003e-06 2023-06-10T21:46:04.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.294814+0000", 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.218991033 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.959 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.295436+0000", 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00062166299999999995 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313438+0000", 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018001952000000002 2023-06-10T21:46:04.960 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313553+0000", 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115364 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:46:04.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.313880+0000", 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032708500000000002 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.322782+0000", 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089019919999999992 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331101+0000", 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083185140000000008 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331137+0000", 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6034000000000001e-05 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331263+0000", 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012637400000000001 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.331298+0000", 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5088e-05 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.332449+0000", 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001151032 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117693 12.19 12:9f069f6e:::benchmark_data_smithi064_442287_object29643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:46:04.966 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 420.15257652100001, 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5931760800000001, 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.967 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117693 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.968 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667688+0000", 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667686+0000", 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667699+0000", 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2989e-05 2023-06-10T21:46:04.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667723+0000", 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3802e-05 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667736+0000", 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.375e-05 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667797+0000", 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0807000000000002e-05 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.667811+0000", 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3402e-05 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075633+0000", 2023-06-10T21:46:04.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4078227939999999 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.075643+0000", 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5529999999999998e-06 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205257+0000", 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1296138060000001 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.205964+0000", 2023-06-10T21:46:04.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00070749699999999997 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253657+0000", 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.047692997000000001 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.976 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253748+0000", 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0437000000000002e-05 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5", 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.253982+0000", 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023403700000000001 2023-06-10T21:46:04.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.258419+0000", 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044375309999999998 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259238+0000", 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00081873500000000001 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259276+0000", 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8238999999999998e-05 2023-06-10T21:46:04.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259376+0000", 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9351000000000005e-05 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:04.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.259436+0000", 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0044000000000003e-05 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.260864+0000", 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014284499999999999 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.981 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136391 12.a 12:50cbddcd:::benchmark_data_smithi064_442287_object48341:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.64279947099999, 2023-06-10T21:46:04.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.351712574, 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136391 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.983 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177465+0000", 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:04.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177464+0000", 2023-06-10T21:46:04.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:46:04.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:04.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177470+0000", 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8900000000000004e-06 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177481+0000", 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1082e-05 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177488+0000", 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.605e-06 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.988 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.177545+0000", 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6372999999999997e-05 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.182867+0000", 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053217430000000003 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210800+0000", 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027933428 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.210903+0000", 2023-06-10T21:46:04.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103509 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,7", 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.211048+0000", 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144902 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.215466+0000", 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044176579999999997 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221482+0000", 2023-06-10T21:46:04.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060159760000000001 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257726+0000", 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036244365000000001 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.257739+0000", 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2807e-05 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.221998+0000", 2023-06-10T21:46:04.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96425910500000001 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222010+0000", 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1863e-05 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454898+0000", 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2328882999999999 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.454985+0000", 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6615999999999999e-05 2023-06-10T21:46:04.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529107+0000", 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.074122399000000005 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:04.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529178+0000", 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0131999999999999e-05 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:04.998 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136405 12.a 12:54ed30e6:::benchmark_data_smithi064_442287_object48355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.60367792900001, 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3845864269999999, 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:04.999 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136405 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216587+0000", 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.001 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216586+0000", 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216589+0000", 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5150000000000001e-06 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.002 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216594+0000", 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1039999999999998e-06 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216598+0000", 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4460000000000003e-06 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.221515+0000", 2023-06-10T21:46:05.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049165859999999997 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222958+0000", 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014432430000000001 2023-06-10T21:46:05.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261169+0000", 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.038210604000000002 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261175+0000", 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2589999999999997e-06 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222103+0000", 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092808100000005 2023-06-10T21:46:05.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222109+0000", 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3199999999999996e-06 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529563+0000", 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307454071 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529785+0000", 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022143400000000001 2023-06-10T21:46:05.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549764+0000", 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01997898 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.010 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.549868+0000", 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103929 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.550267+0000", 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000398811 2023-06-10T21:46:05.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.568050+0000", 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017783812999999999 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575392+0000", 2023-06-10T21:46:05.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007341106 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.575441+0000", 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9833000000000002e-05 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583056+0000", 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076141569999999999 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598427+0000", 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015371246 2023-06-10T21:46:05.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.598489+0000", 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2738999999999997e-05 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.601173+0000", 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002683506 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136406 12.a 12:5c33d118:::benchmark_data_smithi064_442287_object48356:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.60365703500003, 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4148715210000002, 2023-06-10T21:46:05.018 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136406 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.019 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216607+0000", 2023-06-10T21:46:05.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216609+0000", 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1780000000000002e-06 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216614+0000", 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.549e-06 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.216618+0000", 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3000000000000003e-06 2023-06-10T21:46:05.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.222999+0000", 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063813000000000003 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.024 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224213+0000", 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001213312 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261137+0000", 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036923926000000003 2023-06-10T21:46:05.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261146+0000", 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2450000000000004e-06 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222072+0000", 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96092633900000002 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222078+0000", 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0990000000000004e-06 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529195+0000", 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307116508 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529519+0000", 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032444499999999997 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583294+0000", 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.053774523999999997 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583424+0000", 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130443 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583605+0000", 2023-06-10T21:46:05.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018088100000000001 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.588407+0000", 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048019949999999999 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593807+0000", 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053999369999999996 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.593843+0000", 2023-06-10T21:46:05.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5809999999999998e-05 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.597698+0000", 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0038552539999999998 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629308+0000", 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.031610232000000002 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.629379+0000", 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0947999999999999e-05 2023-06-10T21:46:05.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.631479+0000", 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020995969999999999 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136411 12.a 12:52d92197:::benchmark_data_smithi064_442287_object48361:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 375.59598583500002, 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.391707663, 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.036 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136411 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.037 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224279+0000", 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224277+0000", 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224285+0000", 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9610000000000005e-06 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224297+0000", 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.153e-05 2023-06-10T21:46:05.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224306+0000", 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6539999999999994e-06 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224396+0000", 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9741999999999996e-05 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.224406+0000", 2023-06-10T21:46:05.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6120000000000005e-06 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261193+0000", 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.036787411999999999 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.261198+0000", 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2730000000000002e-06 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222130+0000", 2023-06-10T21:46:05.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96093205800000003 2023-06-10T21:46:05.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.222135+0000", 2023-06-10T21:46:05.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6940000000000003e-06 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.529810+0000", 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.307674896 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.530375+0000", 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00056451500000000003 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565747+0000", 2023-06-10T21:46:05.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.035372210000000001 2023-06-10T21:46:05.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.048 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.049 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.565873+0000", 2023-06-10T21:46:05.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125976 2023-06-10T21:46:05.049 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.049 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.049 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.566075+0000", 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000202573 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582894+0000", 2023-06-10T21:46:05.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016818204999999999 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.582953+0000", 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9228000000000003e-05 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583006+0000", 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3650000000000003e-05 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.583198+0000", 2023-06-10T21:46:05.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019192600000000001 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613246+0000", 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.030047554000000001 2023-06-10T21:46:05.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.613310+0000", 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3886999999999993e-05 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.615986+0000", 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026765270000000002 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156289 12.19 12:9ee93671:::benchmark_data_smithi064_442287_object68239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 335.62478935500002, 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2463835950000002, 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:05.056 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156289 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:46:05.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195475+0000", 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195473+0000", 2023-06-10T21:46:05.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195481+0000", 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8739999999999995e-06 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195499+0000", 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7822e-05 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.195507+0000", 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4540000000000007e-06 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.197882+0000", 2023-06-10T21:46:05.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023747109999999998 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.199076+0000", 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011941499999999999 2023-06-10T21:46:05.063 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212744+0000", 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013667643 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.064 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.212868+0000", 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124313 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,1,5,7", 2023-06-10T21:46:05.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.213164+0000", 2023-06-10T21:46:05.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029563400000000002 2023-06-10T21:46:05.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231891+0000", 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018727414000000001 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258125+0000", 2023-06-10T21:46:05.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.026233570000000001 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.258140+0000", 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5158e-05 2023-06-10T21:46:05.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211376+0000", 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95323585399999999 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211388+0000", 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2655e-05 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382314+0000", 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1709252590000001 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:46:05.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.382471+0000", 2023-06-10T21:46:05.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000157326 2023-06-10T21:46:05.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441738+0000", 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.059266970000000002 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441859+0000", 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120934 2023-06-10T21:46:05.073 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156297 12.19 12:9878c605:::benchmark_data_smithi064_442287_object68247:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:46:05.074 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 335.60951949700001, 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2780029549999998, 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.075 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156297 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210745+0000", 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210743+0000", 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:46:05.078 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210751+0000", 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8199999999999997e-06 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210765+0000", 2023-06-10T21:46:05.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3745e-05 2023-06-10T21:46:05.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210773+0000", 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1030000000000002e-06 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.230620+0000", 2023-06-10T21:46:05.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019846534999999998 2023-06-10T21:46:05.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.231381+0000", 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076190000000000003 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260069+0000", 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.028687363 2023-06-10T21:46:05.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260080+0000", 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0786e-05 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211426+0000", 2023-06-10T21:46:05.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134606499999996 2023-06-10T21:46:05.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211436+0000", 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0145e-05 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.441910+0000", 2023-06-10T21:46:05.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2304743680000001 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.442245+0000", 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000334514 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454189+0000", 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011943949000000001 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454304+0000", 2023-06-10T21:46:05.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000115359 2023-06-10T21:46:05.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:46:05.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.454674+0000", 2023-06-10T21:46:05.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000370176 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459572+0000", 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048976749999999998 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459582+0000", 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0105999999999999e-05 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464585+0000", 2023-06-10T21:46:05.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050026599999999999 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487733+0000", 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.023147893999999999 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.487794+0000", 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1424e-05 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.093 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488748+0000", 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00095399799999999996 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156298 12.19 12:98e24496:::benchmark_data_smithi064_442287_object68248:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:46:05.094 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 335.60948099299998, 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2788083060000002, 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:46:05.095 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156298 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:46:05.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210784+0000", 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210783+0000", 2023-06-10T21:46:05.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210786+0000", 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5849999999999998e-06 2023-06-10T21:46:05.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210792+0000", 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9750000000000004e-06 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.210797+0000", 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9509999999999997e-06 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260110+0000", 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.049313312999999998 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:29.260119+0000", 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9889999999999998e-06 2023-06-10T21:46:05.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211462+0000", 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95134341099999997 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.211471+0000", 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7490000000000004e-06 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.446320+0000", 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.234848945 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.447323+0000", 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001003122 2023-06-10T21:46:05.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457450+0000", 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010126634000000001 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457569+0000", 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119261 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.457915+0000", 2023-06-10T21:46:05.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00034610199999999999 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.459652+0000", 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001736502 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.464637+0000", 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049851749999999997 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.467552+0000", 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002914954 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488815+0000", 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021263659000000001 2023-06-10T21:46:05.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488870+0000", 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4682000000000003e-05 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.489592+0000", 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00072164799999999997 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106688 13.4 13:21a54017:::benchmark_data_smithi064_501823_object106687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.29774349799999, 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1085864490000001, 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:46:05.113 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106688 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522521+0000", 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522548+0000", 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7114000000000002e-05 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522555+0000", 2023-06-10T21:46:05.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5259999999999996e-06 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522561+0000", 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7930000000000003e-06 2023-06-10T21:46:05.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.528701+0000", 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061403400000000002 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.528877+0000", 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017563999999999999 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3", 2023-06-10T21:46:05.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.529052+0000", 2023-06-10T21:46:05.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000174902 2023-06-10T21:46:05.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.540055+0000", 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011003051999999999 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545371+0000", 2023-06-10T21:46:05.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053163510000000004 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545384+0000", 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2266000000000001e-05 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.195964+0000", 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058062100000003 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.195974+0000", 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9119999999999993e-06 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615197+0000", 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4192233459999999 2023-06-10T21:46:05.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615857+0000", 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00065989 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.630989+0000", 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015131350999999999 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631107+0000", 2023-06-10T21:46:05.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118786 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106692 13.4 13:277bf932:::benchmark_data_smithi064_501823_object106691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:46:05.128 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.29759948700001, 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1091667009999999, 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:46:05.129 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106692 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:46:05.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522665+0000", 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522689+0000", 2023-06-10T21:46:05.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4600000000000002e-05 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522697+0000", 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7389999999999999e-06 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.522703+0000", 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6459999999999998e-06 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545434+0000", 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022731411 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545443+0000", 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3910000000000004e-06 2023-06-10T21:46:05.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196033+0000", 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058933500000005 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196043+0000", 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0361e-05 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.615912+0000", 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4198693060000001 2023-06-10T21:46:05.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.616007+0000", 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5063999999999996e-05 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:46:05.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.616124+0000", 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000117063 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.628240+0000", 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012115310000000001 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.629520+0000", 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001279896 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631158+0000", 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016387319999999999 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631224+0000", 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5912000000000002e-05 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631832+0000", 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00060740999999999996 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106701 13.4 13:2a3b2934:::benchmark_data_smithi064_501823_object106700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 184.28280544, 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0991637939999999, 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:46:05.143 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106701 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537459+0000", 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:46:05.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537458+0000", 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:46:05.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537484+0000", 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5945e-05 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537492+0000", 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9789999999999993e-06 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.537499+0000", 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2550000000000003e-06 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.148 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545468+0000", 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079695500000000006 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.545477+0000", 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6009999999999995e-06 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196065+0000", 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.65058839099999999 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.196073+0000", 2023-06-10T21:46:05.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7109999999999995e-06 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618573+0000", 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.422500458 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618724+0000", 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015112 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.618864+0000", 2023-06-10T21:46:05.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000139529 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.631882+0000", 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013018237 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.632220+0000", 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033821300000000002 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.635418+0000", 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031970430000000001 2023-06-10T21:46:05.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.635476+0000", 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8950000000000003e-05 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:46:05.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.636623+0000", 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001146389 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:46:05.156 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:46:05.157 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:05.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 11 16 29209 29193 165.62 121.875 0.00399372 0.00603288 2023-06-10T21:46:05.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:05 smithi064 ceph-mon[180297]: pgmap v1565: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:05 smithi064 ceph-mon[186706]: pgmap v1565: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:05.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:05 smithi152 ceph-mon[156819]: pgmap v1565: 89 pgs: 89 active+clean; 1.1 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:06.092 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:46:06.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 12 16 30608 30592 159.112 87.4375 0.00722972 0.0062723 2023-06-10T21:46:06.727 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:06.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:06.831 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:07.134 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:07.153 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:46:07.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 13 16 31990 31974 153.523 86.375 0.00879364 0.00650834 2023-06-10T21:46:07.278 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:07.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:07.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:07.379 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:07.381 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:07 smithi064 ceph-mon[186706]: pgmap v1566: 89 pgs: 89 active+clean; 1.6 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-06-10T21:46:07.383 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:07 smithi064 ceph-mon[180297]: pgmap v1566: 89 pgs: 89 active+clean; 1.6 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-06-10T21:46:07.388 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:07.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:07 smithi152 ceph-mon[156819]: pgmap v1566: 89 pgs: 89 active+clean; 1.6 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-06-10T21:46:07.480 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:07.589 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:07.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:07.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:07.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:07.754 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:07.832 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:07.937 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:07.948 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:07.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:08.055 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:08.110 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:08.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 14 16 33667 33651 150.047 104.812 0.00978815 0.00665983 2023-06-10T21:46:08.413 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:08.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:08.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:08.580 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:09.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 15 16 35109 35093 146.056 90.125 0.00827301 0.00684286 2023-06-10T21:46:09.283 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:09.285 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:46:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:09.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:09.392 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:09.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:09.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[180297]: pgmap v1567: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-06-10T21:46:09.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1093521074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[186706]: pgmap v1567: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-06-10T21:46:09.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1093521074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:09 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.605 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:09.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:09.639 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:09 smithi152 ceph-mon[156819]: pgmap v1567: 89 pgs: 89 active+clean; 1.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-06-10T21:46:09.640 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1093521074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.640 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:09 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:09.710 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:09.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:09.924 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:46:09.924 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:46:09.924 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:46:09.924 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:46:10.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 16 16 36549 36533 142.555 90 0.0103269 0.0070093 2023-06-10T21:46:10.271 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: osdmap e425: 8 total, 8 up, 4 in 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.272 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:10 smithi152 ceph-mon[156819]: pgmap v1569: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:46:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: osdmap e425: 8 total, 8 up, 4 in 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[186706]: pgmap v1569: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: osdmap e425: 8 total, 8 up, 4 in 2023-06-10T21:46:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: failed to encode map e425 with expected crc 2023-06-10T21:46:10.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:10 smithi064 ceph-mon[180297]: pgmap v1569: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-06-10T21:46:10.928 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:10.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:11.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:46:11.035 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:11.099 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:11.186 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 17 16 37796 37780 138.757 77.9375 0.0135624 0.0072028 2023-06-10T21:46:11.528 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:11.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:11.877 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:11.956 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:12.146 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:12.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:12.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:12.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 18 16 39325 39309 136.359 95.5625 0.0102917 0.00732783 2023-06-10T21:46:12.251 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:12.332 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:12.475 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:12.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:12.522 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:12.541 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:12.559 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:12.578 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:12.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:12.596 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:12.617 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:12.639 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:12.672 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:12.705 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:12.707 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:12.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:12.720 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.721 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.722 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:46:12.812 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:12.903 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:13.050 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:13.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:13.158 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:12] "GET /metrics HTTP/1.1" 200 224070 "" "Prometheus/2.33.4" 2023-06-10T21:46:13.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:13.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 19 16 40672 40656 133.615 84.1875 0.0119554 0.0074791 2023-06-10T21:46:13.220 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:13.271 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 0 (802) 2023-06-10T21:46:13.367 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:13.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:13.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:13.471 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:13.515 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:13.553 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:13.554 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:13.579 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:46:13.581 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:13 smithi152 ceph-mon[156819]: pgmap v1570: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:13.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1313163493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.582 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1119776979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[186706]: pgmap v1570: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1313163493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.607 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1119776979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:13.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[180297]: pgmap v1570: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-06-10T21:46:13.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1313163493' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-10T21:46:13.608 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1119776979' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:13.923 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:13.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:13.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:14.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:14.056 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:14.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:14.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:14.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:46:14.184171+0000 min lat: 0.00126485 max lat: 0.327302 avg lat: 0.00756341 2023-06-10T21:46:14.186 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:46:14.186 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 20 16 40815 40799 127.386 8.9375 0.0027216 0.00756341 2023-06-10T21:46:14.204 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:14.231 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:14.413 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:14.415 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:14 smithi152 ceph-mon[156819]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:14 smithi152 ceph-mon[156819]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-10T21:46:14.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:14 smithi152 ceph-mon[156819]: osdmap e426: 8 total, 8 up, 4 in 2023-06-10T21:46:14.416 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:14 smithi152 ceph-mon[156819]: pgmap v1572: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:46:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:14.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:14.470 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:46:14.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:14.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[180297]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[180297]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-10T21:46:14.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[180297]: osdmap e426: 8 total, 8 up, 4 in 2023-06-10T21:46:14.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[180297]: pgmap v1572: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:46:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[186706]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[186706]: failed to encode map e426 with expected crc 2023-06-10T21:46:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-10T21:46:14.568 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[186706]: osdmap e426: 8 total, 8 up, 4 in 2023-06-10T21:46:14.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:14 smithi064 ceph-mon[186706]: pgmap v1572: 89 pgs: 89 active+clean; 2.3 GiB data, 9.6 GiB used, 348 GiB / 358 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-10T21:46:14.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:14.636 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:14.689 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:14.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:14.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:14.860 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:14.978 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:14.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:15.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:15.089 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:15.138 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:15.185 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 21 16 42234 42218 125.544 88.6875 0.00749633 0.00795964 2023-06-10T21:46:15.222 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:15.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:15.326 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:15.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[186706]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[186706]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[186706]: osdmap e427: 8 total, 8 up, 4 in 2023-06-10T21:46:15.379 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[186706]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[180297]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[180297]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[180297]: osdmap e427: 8 total, 8 up, 4 in 2023-06-10T21:46:15.380 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:15 smithi064 ceph-mon[180297]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.384 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:15.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:15 smithi152 ceph-mon[156819]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:15 smithi152 ceph-mon[156819]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:15 smithi152 ceph-mon[156819]: osdmap e427: 8 total, 8 up, 4 in 2023-06-10T21:46:15.397 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:15 smithi152 ceph-mon[156819]: failed to encode map e427 with expected crc 2023-06-10T21:46:15.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:15.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:15.584 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:15.614 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:15.766 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:15.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:15.873 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:15.922 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:16.149 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:16.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:16.186 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 22 16 43249 43233 122.722 63.4375 0.00453006 0.00814364 2023-06-10T21:46:16.257 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:16.332 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:16.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:46:16.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:16.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:16.599 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:16.646 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:16.776 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:16.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:16.884 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:16.931 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:17.013 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:17.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:17.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:17.069 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:17.069 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:17.113 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:17.117 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:17.165 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:17.186 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 23 16 44686 44670 121.292 89.8125 0.0124897 0.00823893 2023-06-10T21:46:17.289 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:17.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:17.304 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:46:17.397 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:17.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:17 smithi152 ceph-mon[156819]: pgmap v1574: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:17.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:17 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:46:17.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:17.462 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:17.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[186706]: pgmap v1574: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:17.569 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:46:17.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:17.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[180297]: pgmap v1574: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:17.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2023-06-10T21:46:17.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:18.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:18.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:46:18.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:18.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:18.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:18.187 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 24 16 45707 45691 118.891 63.8125 0.0088158 0.00840582 2023-06-10T21:46:18.242 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:18.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:18.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: failed to encode map e428 with expected crc 2023-06-10T21:46:18.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: osdmap e428: 8 total, 8 up, 4 in 2023-06-10T21:46:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:18.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:18.501 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:18.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:18.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: failed to encode map e428 with expected crc 2023-06-10T21:46:18.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:18.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: osdmap e428: 8 total, 8 up, 4 in 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: failed to encode map e428 with expected crc 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3575289960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: osdmap e428: 8 total, 8 up, 4 in 2023-06-10T21:46:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:18.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:19.187 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 25 16 46553 46537 116.252 52.875 0.00896828 0.00858027 2023-06-10T21:46:19.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:19 smithi064 ceph-mon[180297]: failed to encode map e428 with expected crc 2023-06-10T21:46:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:19 smithi064 ceph-mon[180297]: pgmap v1576: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 60 MiB/s wr, 958 op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:19 smithi064 ceph-mon[186706]: failed to encode map e428 with expected crc 2023-06-10T21:46:19.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:19 smithi064 ceph-mon[186706]: pgmap v1576: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 60 MiB/s wr, 958 op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:19.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:19 smithi152 ceph-mon[156819]: failed to encode map e428 with expected crc 2023-06-10T21:46:19.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:19 smithi152 ceph-mon[156819]: pgmap v1576: 89 pgs: 1 active, 4 activating, 84 active+clean; 2.6 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 60 MiB/s wr, 958 op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:19.479 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5] out_osds: [6, 4, 0, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:46:19.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:46:19.624 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:19.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:19.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:19.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:19.806 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:19.938 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:19.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:20.045 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:20.116 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:20.187 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 26 16 47606 47590 114.313 65.8125 0.00776251 0.00874127 2023-06-10T21:46:20.262 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:20.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:20.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:20.323 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:20 smithi064 ceph-mon[186706]: pgmap v1577: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:20.323 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:20 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-06-10T21:46:20.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:20 smithi064 ceph-mon[180297]: pgmap v1577: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:20.324 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:20 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-06-10T21:46:20.332 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:20.375 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:20.384 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:20.384 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:20.397 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:20.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:20 smithi152 ceph-mon[156819]: pgmap v1577: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:20.426 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:20 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive) 2023-06-10T21:46:20.433 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:20.441 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:20.475 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:20.518 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:20.549 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:20.555 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:20.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:20.586 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:20.618 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:20.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:20.687 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:20.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.688 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:46:20.701 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:20.785 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:20.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:20.892 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:20.950 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:21.188 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 27 16 48949 48933 113.189 83.9375 0.013819 0.00883046 2023-06-10T21:46:21.261 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:46:21.487 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:46:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3933242426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3933242426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3933242426' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:21.739 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:22.189 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 28 16 49867 49851 111.19 57.375 0.00734785 0.00898845 2023-06-10T21:46:22.430 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:22.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:22.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:22.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:22 smithi064 ceph-mon[180297]: pgmap v1578: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:22 smithi064 ceph-mon[186706]: pgmap v1578: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:22.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:22 smithi152 ceph-mon[156819]: pgmap v1578: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 89 MiB/s wr, 1.43k op/s; 0 B/s, 2 objects/s recovering 2023-06-10T21:46:22.726 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:23.075 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:23.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:23.160 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:22] "GET /metrics HTTP/1.1" 200 224090 "" "Prometheus/2.33.4" 2023-06-10T21:46:23.189 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:23.190 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 29 16 51159 51143 110.14 80.75 0.0132606 0.00907432 2023-06-10T21:46:23.246 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:23.261 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:23.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:23.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:23.525 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:23.600 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:23.607 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:46:23.607 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:46:23.607 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:46:23.607 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:46:23.608 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:46:24.189 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:24.190 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 30 16 52476 52460 109.212 82.3125 0.0125005 0.00915024 2023-06-10T21:46:24.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:24.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:24.242 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:46:24.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:24.332 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:46:24.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:46:24.349 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:24.440 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:24.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:46:24.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:24.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:24.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:24.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:24.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:46:24.713 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:24.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:24.825 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:24.851 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:24.905 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:24.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:25.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:46:25.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:25.070 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:25.178 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:25.190 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 31 16 53450 53434 107.654 60.875 0.0129583 0.00928319 2023-06-10T21:46:25.214 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:25.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:25.322 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:25.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:46:25.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: osdmap e429: 8 total, 8 up, 4 in 2023-06-10T21:46:25.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: pgmap v1580: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:46:25.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.329 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.330 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:46:25.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: osdmap e429: 8 total, 8 up, 4 in 2023-06-10T21:46:25.330 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: pgmap v1580: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:46:25.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.331 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:25 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.361 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:25.463 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:25.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:25.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/531330853' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:46:25.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: osdmap e429: 8 total, 8 up, 4 in 2023-06-10T21:46:25.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: pgmap v1580: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-10T21:46:25.476 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.477 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:25 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:25.566 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:25.594 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:25.776 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:25.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:25.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:25.920 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:26.006 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:26.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:26.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:26.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:46:26.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:26.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 32 16 54326 54310 105.995 54.75 0.0110123 0.00942997 2023-06-10T21:46:26.192 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:26.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[186706]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:26 smithi064 ceph-mon[180297]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:26 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:26 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:26 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:26 smithi152 ceph-mon[156819]: failed to encode map e429 with expected crc 2023-06-10T21:46:26.444 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:26.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:26.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:26.549 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:26.614 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:26.624 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:26.703 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:26.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:26.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:26.808 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:26.861 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:26.996 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:27.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:27.083 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:46:27.083 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:46:27.083 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:46:27.083 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [] 2023-06-10T21:46:27.083 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:46:27.105 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:27.156 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:27.182 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:27.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 33 16 55520 55504 105.044 74.625 0.0155904 0.0095147 2023-06-10T21:46:27.292 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:46:27.329 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:27.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:27.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:27.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:27.465 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:27.508 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:27.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:27.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:27.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:27.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[180297]: pgmap v1581: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 62 MiB/s wr, 986 op/s 2023-06-10T21:46:27.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4011845502' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[186706]: pgmap v1581: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 62 MiB/s wr, 986 op/s 2023-06-10T21:46:27.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4011845502' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:27 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.668 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:27.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:27 smithi152 ceph-mon[156819]: pgmap v1581: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 62 MiB/s wr, 986 op/s 2023-06-10T21:46:27.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4011845502' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:27 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:27.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:46:28.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 34 16 56552 56536 103.852 64.5 0.0136682 0.00962342 2023-06-10T21:46:28.277 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:28 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:46:28.278 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:28 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:46:28.278 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:28 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:28.278 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:28 smithi152 ceph-mon[156819]: osdmap e430: 8 total, 8 up, 4 in 2023-06-10T21:46:28.278 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:28 smithi152 ceph-mon[156819]: failed to encode map e430 with expected crc 2023-06-10T21:46:28.374 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:28.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:28.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:28.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:28.522 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:46:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:46:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[180297]: osdmap e430: 8 total, 8 up, 4 in 2023-06-10T21:46:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[180297]: failed to encode map e430 with expected crc 2023-06-10T21:46:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:46:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:46:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[186706]: osdmap e430: 8 total, 8 up, 4 in 2023-06-10T21:46:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:28 smithi064 ceph-mon[186706]: failed to encode map e430 with expected crc 2023-06-10T21:46:28.997 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:29.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:29.115 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:29.165 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:29.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 35 16 57815 57799 103.141 78.9375 0.0140954 0.00969017 2023-06-10T21:46:29.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[186706]: failed to encode map e430 with expected crc 2023-06-10T21:46:29.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[186706]: pgmap v1583: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:46:29.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:29.283 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:29.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:29.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:29.388 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:29.412 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:29 smithi152 ceph-mon[156819]: failed to encode map e430 with expected crc 2023-06-10T21:46:29.413 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:29 smithi152 ceph-mon[156819]: pgmap v1583: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:46:29.413 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:29 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:29.414 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:29.457 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:29.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:29.560 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:29.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[180297]: failed to encode map e430 with expected crc 2023-06-10T21:46:29.593 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[180297]: pgmap v1583: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-10T21:46:29.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:29 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:29.598 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:29.745 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:29.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:29.850 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:29.919 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:30.006 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:30.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:30.123 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:30.174 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:30.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 36 16 58878 58862 102.122 66.4375 0.00298359 0.00978034 2023-06-10T21:46:30.270 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:30.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:30.375 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:30.444 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:30.538 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:30.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:30.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:30 smithi152 ceph-mon[156819]: pgmap v1584: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-06-10T21:46:30.573 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:30.585 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:46:30.585 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:59249 14.7 14:eeedaca1:::benchmark_data_smithi064_559916_object59248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e430)", 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.578841+0000", 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0012477180000000001, 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012818420000000001, 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:46:30.586 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 59249 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.587 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578841+0000", 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.588 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578841+0000", 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578839+0000", 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6030999999999999e-05 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.589 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578895+0000", 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6203999999999999e-05 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578911+0000", 2023-06-10T21:46:30.590 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0840000000000003e-06 2023-06-10T21:46:30.591 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.591 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.591 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.591 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578918+0000", 2023-06-10T21:46:30.591 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5171000000000001e-05 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579014+0000", 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00017154599999999999 2023-06-10T21:46:30.592 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579185+0000", 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018373 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.593 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,5", 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579369+0000", 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00052824899999999995 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.594 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:59251 14.7 14.3f647647 (undecoded) ondisk+write+known_if_redirected e430)", 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.578931+0000", 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001157255, 2023-06-10T21:46:30.595 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001274812, 2023-06-10T21:46:30.596 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.596 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:46:30.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:46:30.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:46:30.596 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:46:30.597 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 59251 2023-06-10T21:46:30.597 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.597 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.597 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578931+0000", 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.598 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578931+0000", 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578930+0000", 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1476999999999997e-05 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.599 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578962+0000", 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9809999999999992e-06 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.600 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578971+0000", 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0608000000000001e-05 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.578981+0000", 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0376e-05 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.601 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:59255 14.f 14.4d59859f (undecoded) ondisk+write+known_if_redirected e430)", 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.579005+0000", 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001082857, 2023-06-10T21:46:30.602 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012560270000000001, 2023-06-10T21:46:30.603 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.603 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:46:30.603 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:46:30.603 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:46:30.604 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:46:30.604 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 59255 2023-06-10T21:46:30.604 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.604 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.605 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.605 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579005+0000", 2023-06-10T21:46:30.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.605 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579005+0000", 2023-06-10T21:46:30.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:30.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579005+0000", 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4390000000000001e-05 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579039+0000", 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5846999999999999e-05 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579055+0000", 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.695e-06 2023-06-10T21:46:30.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579063+0000", 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7290999999999997e-05 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.609 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:59259 14.f 14.2f274cf (undecoded) ondisk+write+known_if_redirected e430)", 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.579069+0000", 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0010193019999999999, 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001246313, 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.610 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "queued for pg", 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 59259 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579069+0000", 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579069+0000", 2023-06-10T21:46:30.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579068+0000", 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2223e-05 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579101+0000", 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0891e-05 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579111+0000", 2023-06-10T21:46:30.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2660000000000001e-06 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579117+0000", 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7661999999999999e-05 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.615 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:59252 14.3 e430/420 14:c6cdfe45:::benchmark_data_smithi064_559916_object59251:head v 430'3740, mlcod=430'3738)", 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.579481+0000", 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00060778000000000004, 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00089348800000000001, 2023-06-10T21:46:30.616 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579481+0000", 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579481+0000", 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579480+0000", 2023-06-10T21:46:30.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5010000000000003e-05 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579545+0000", 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0671e-05 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579555+0000", 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8390000000000003e-06 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579561+0000", 2023-06-10T21:46:30.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120495 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579682+0000", 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9766999999999999e-05 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:46:30.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579702+0000", 2023-06-10T21:46:30.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022101600000000001 2023-06-10T21:46:30.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.622 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.622 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:59250 14.1 e430/420 14:8275ddef:::benchmark_data_smithi064_559916_object59249:head v 430'3751, mlcod=430'3749)", 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:46:30.579576+0000", 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00051226099999999997, 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00085886800000000004, 2023-06-10T21:46:30.623 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:46:30.624 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:46:30.624 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:46:30.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:46:30.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579576+0000", 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579576+0000", 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:46:30.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579576+0000", 2023-06-10T21:46:30.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8642e-05 2023-06-10T21:46:30.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579614+0000", 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5510000000000002e-06 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:46:30.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579622+0000", 2023-06-10T21:46:30.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3850000000000004e-06 2023-06-10T21:46:30.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:46:30.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579626+0000", 2023-06-10T21:46:30.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7310000000000003e-05 2023-06-10T21:46:30.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:46:30.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579713+0000", 2023-06-10T21:46:30.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7070000000000001e-05 2023-06-10T21:46:30.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:46:30.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:46:30.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:46:30.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:46:30.579731+0000", 2023-06-10T21:46:30.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000154504 2023-06-10T21:46:30.631 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.632 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:46:30.632 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.632 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:46:30.632 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:46:30.632 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 6 2023-06-10T21:46:30.633 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:46:30.633 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:30.642 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:30.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:30.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:30 smithi064 ceph-mon[180297]: pgmap v1584: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-06-10T21:46:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:30 smithi064 ceph-mon[186706]: pgmap v1584: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 53 MiB/s wr, 852 op/s 2023-06-10T21:46:30.681 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:30.702 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:30.713 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:30.759 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:30.793 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:30.808 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:30.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:30.829 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.837 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.838 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:46:30.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:30.954 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:30.996 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:30.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:30.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:31.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:46:31.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:31.129 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:31.191 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:31.192 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 37 16 59719 59703 100.783 52.5625 0.00354199 0.00991021 2023-06-10T21:46:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:31.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:31.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/864470609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:31.296 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:31.358 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:31.539 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:31.540 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/864470609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:31.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:31.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:31.642 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:31.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/864470609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:31.717 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:31.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:46:31.861 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:31.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:31.973 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:32.023 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:32.115 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:32.172 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:32.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:32.195 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 38 16 60732 60716 99.7891 63.3125 0.0156991 0.0100171 2023-06-10T21:46:32.283 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:32.336 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:32.345 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:32.425 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:32.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:32.434 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:32 smithi152 ceph-mon[156819]: pgmap v1585: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-06-10T21:46:32.535 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:32.576 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:32 smithi064 ceph-mon[180297]: pgmap v1585: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-06-10T21:46:32.576 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:32 smithi064 ceph-mon[186706]: pgmap v1585: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 86 MiB/s wr, 1.37k op/s 2023-06-10T21:46:32.602 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:32.692 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:32.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:32.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:32.860 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:33.090 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:33.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:32] "GET /metrics HTTP/1.1" 200 224090 "" "Prometheus/2.33.4" 2023-06-10T21:46:33.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:33.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 39 16 61687 61671 98.7613 59.6875 0.00793974 0.0100634 2023-06-10T21:46:33.220 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:33.277 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:33.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:33.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:33.312 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:46:33.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:33.445 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/244702973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:33.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/244702973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:33.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/244702973' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:33 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:46:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:33.755 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:33.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:46:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:33.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:33.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:33.968 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:34.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:46:34.194581+0000 min lat: 0.00126485 max lat: 1.07351 avg lat: 0.0101993 2023-06-10T21:46:34.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:46:34.197 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 40 16 62778 62762 97.9973 68.1875 0.0156037 0.0101993 2023-06-10T21:46:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:46:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:34 smithi152 ceph-mon[156819]: osdmap e431: 8 total, 8 up, 4 in 2023-06-10T21:46:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:34 smithi152 ceph-mon[156819]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:34 smithi152 ceph-mon[156819]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.322 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:34 smithi152 ceph-mon[156819]: pgmap v1587: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-10T21:46:34.351 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:34.356 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:34.357 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:34.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:34.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:46:34.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:34.518 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:34.589 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[180297]: osdmap e431: 8 total, 8 up, 4 in 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[180297]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[180297]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[180297]: pgmap v1587: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:46:34.590 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[186706]: osdmap e431: 8 total, 8 up, 4 in 2023-06-10T21:46:34.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[186706]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[186706]: failed to encode map e431 with expected crc 2023-06-10T21:46:34.591 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:34 smithi064 ceph-mon[186706]: pgmap v1587: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-10T21:46:34.601 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:34.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:34.705 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:34.748 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:34.831 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:34.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:34.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:34.847 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:34.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:34.986 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:35.173 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:35.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:35.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 41 16 63825 63809 97.2035 65.4375 0.00717557 0.0102835 2023-06-10T21:46:35.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:35.360 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:46:35.361 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:35.410 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:46:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[186706]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[186706]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[186706]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2506087123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[180297]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[180297]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[180297]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2506087123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:35.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:35 smithi152 ceph-mon[156819]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:35 smithi152 ceph-mon[156819]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:35 smithi152 ceph-mon[156819]: failed to encode map e431 with expected crc 2023-06-10T21:46:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:35 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2506087123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:35.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:46:36.010 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:46:36.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 42 16 64838 64822 96.3967 63.3125 0.00330403 0.0103581 2023-06-10T21:46:36.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:36.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.565 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.566 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:46:36.566 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: osdmap e432: 8 total, 8 up, 4 in 2023-06-10T21:46:36.566 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:36 smithi152 ceph-mon[156819]: pgmap v1589: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:46:36.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:36.605 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:36.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:46:36.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: osdmap e432: 8 total, 8 up, 4 in 2023-06-10T21:46:36.606 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[186706]: pgmap v1589: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: failed to encode map e432 with expected crc 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: osdmap e432: 8 total, 8 up, 4 in 2023-06-10T21:46:36.607 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:36 smithi064 ceph-mon[180297]: pgmap v1589: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-10T21:46:36.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:36.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:36.709 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:36.737 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:37.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 43 16 65478 65462 95.0858 40 0.0124038 0.0105128 2023-06-10T21:46:37.325 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:37.353 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:37.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:37.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:37.538 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:37.642 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:37.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:37.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:37.810 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:37.888 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:37.889 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:37.893 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:37.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:37.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:37.975 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:38.002 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:38.010 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:38.050 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:38.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:38.089 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:38.111 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:38.112 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:38.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:38.132 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:38.157 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2318302434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:38.157 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2318302434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:38.165 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:38.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:38.193 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2318302434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:38.196 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 44 16 66562 66546 94.4647 67.75 0.010826 0.0105813 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.208 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.209 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:46:38.216 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:38.281 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:38.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:46:39.011 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:39 smithi064 ceph-mon[180297]: pgmap v1590: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-10T21:46:39.011 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:39 smithi064 ceph-mon[186706]: pgmap v1590: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-10T21:46:39.201 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 45 16 67503 67487 93.6639 58.8125 0.0223553 0.0106711 2023-06-10T21:46:39.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:46:39.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:39 smithi152 ceph-mon[156819]: pgmap v1590: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-10T21:46:39.603 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:39.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:39.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:39.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:39.915 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:39.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:40.033 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:40.104 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:40.201 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 46 16 68628 68612 93.1564 70.3125 0.0121357 0.0107285 2023-06-10T21:46:40.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:40.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:40.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:40.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:40.403 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:40 smithi064 ceph-mon[180297]: pgmap v1591: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 49 MiB/s wr, 781 op/s 2023-06-10T21:46:40.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:40 smithi064 ceph-mon[186706]: pgmap v1591: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 49 MiB/s wr, 781 op/s 2023-06-10T21:46:40.446 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:40.446 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:40.478 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:40.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:40.585 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:40.664 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:40 smithi152 ceph-mon[156819]: pgmap v1591: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 49 MiB/s wr, 781 op/s 2023-06-10T21:46:40.857 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:40.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:40.975 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:41.021 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:41.120 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:41.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:41.203 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 47 16 69705 69689 92.6019 67.3125 0.0116727 0.010794 2023-06-10T21:46:41.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:41.290 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:41.381 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:46:41.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:46:41.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:46:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:46:41.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:46:42.203 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 48 16 70890 70874 92.2158 74.0625 0.00966128 0.0108388 2023-06-10T21:46:42.366 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[180297]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[180297]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:46:42.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[180297]: osdmap e433: 8 total, 8 up, 4 in 2023-06-10T21:46:42.367 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[180297]: pgmap v1593: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:46:42.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[186706]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[186706]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:46:42.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[186706]: osdmap e433: 8 total, 8 up, 4 in 2023-06-10T21:46:42.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:42 smithi064 ceph-mon[186706]: pgmap v1593: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:46:42.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:42 smithi152 ceph-mon[156819]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:42 smithi152 ceph-mon[156819]: failed to encode map e433 with expected crc 2023-06-10T21:46:42.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:42 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3242344416' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:46:42.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:42 smithi152 ceph-mon[156819]: osdmap e433: 8 total, 8 up, 4 in 2023-06-10T21:46:42.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:42 smithi152 ceph-mon[156819]: pgmap v1593: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-10T21:46:42.992 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:42.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:42.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:43.098 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:43.147 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:42] "GET /metrics HTTP/1.1" 200 224089 "" "Prometheus/2.33.4" 2023-06-10T21:46:43.149 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:43.203 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 49 16 71977 71961 91.7205 67.9375 0.0180632 0.0108977 2023-06-10T21:46:43.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:43 smithi064 ceph-mon[186706]: failed to encode map e433 with expected crc 2023-06-10T21:46:43.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:43 smithi064 ceph-mon[180297]: failed to encode map e433 with expected crc 2023-06-10T21:46:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:43 smithi152 ceph-mon[156819]: failed to encode map e433 with expected crc 2023-06-10T21:46:43.927 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:43.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:43.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:44.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:44.049 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 5 1 2023-06-10T21:46:44.107 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:44.204 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 50 16 72842 72826 90.9677 54.0625 0.0107585 0.0109878 2023-06-10T21:46:44.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:44.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:44.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:44.527 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:44.563 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:44 smithi064 ceph-mon[186706]: pgmap v1594: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 969 op/s 2023-06-10T21:46:44.564 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:44 smithi064 ceph-mon[180297]: pgmap v1594: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 969 op/s 2023-06-10T21:46:44.574 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:44 smithi152 ceph-mon[156819]: pgmap v1594: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 969 op/s 2023-06-10T21:46:44.742 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:44.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:44.831 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:46:44.831 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:46:44.831 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:46:44.831 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:46:44.856 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:44.903 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:44.945 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:45.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 51 16 73786 73770 90.3374 59 0.0114676 0.0110647 2023-06-10T21:46:45.433 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:46:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2501631435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:45.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:45.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2501631435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2501631435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:46:46.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 52 16 74841 74825 89.8683 65.9375 0.0152957 0.0111229 2023-06-10T21:46:46.477 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:46.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:46.592 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:46.646 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[186706]: failed to encode map e434 with expected crc 2023-06-10T21:46:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:46.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[186706]: osdmap e434: 8 total, 8 up, 4 in 2023-06-10T21:46:46.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[186706]: pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 61 MiB/s wr, 981 op/s 2023-06-10T21:46:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:46.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[180297]: failed to encode map e434 with expected crc 2023-06-10T21:46:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[180297]: osdmap e434: 8 total, 8 up, 4 in 2023-06-10T21:46:46.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:46 smithi064 ceph-mon[180297]: pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 61 MiB/s wr, 981 op/s 2023-06-10T21:46:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:46 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:46 smithi152 ceph-mon[156819]: failed to encode map e434 with expected crc 2023-06-10T21:46:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:46 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:46:46.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:46 smithi152 ceph-mon[156819]: osdmap e434: 8 total, 8 up, 4 in 2023-06-10T21:46:46.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:46 smithi152 ceph-mon[156819]: pgmap v1596: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 61 MiB/s wr, 981 op/s 2023-06-10T21:46:46.696 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:46.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:46.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:46.802 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:46.857 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:46.954 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:46.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:47.006 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:46:47.062 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:47.088 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:47.139 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:47.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:47.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 53 16 75750 75734 89.245 56.8125 0.0195021 0.0112 2023-06-10T21:46:47.249 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:47.305 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:47.662 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:47.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:47.768 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:47.802 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:47.895 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:47.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:48.005 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:48.056 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:48.134 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:48.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:48.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 54 16 76881 76865 88.9014 70.6875 0.0139329 0.0112438 2023-06-10T21:46:48.241 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:48.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:48.365 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:48.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:48.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:48.476 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:48.507 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:48.911 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:48.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:49.020 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:49.066 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:49.156 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:49.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:49.182 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:49.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 55 16 77816 77800 88.3478 58.4375 0.00929609 0.0113134 2023-06-10T21:46:49.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:49.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:49.214 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[180297]: pgmap v1597: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:46:49.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:49.326 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:49.359 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:46:49.407 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:46:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:49.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:49 smithi152 ceph-mon[156819]: pgmap v1597: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:46:49.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:46:49.482 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:46:49.523 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:46:49.561 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:46:49.603 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:46:49.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:46:49.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:46:49.614 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:49 smithi064 ceph-mon[186706]: pgmap v1597: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-10T21:46:49.626 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:49.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:49.646 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.657 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.658 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:46:49.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:46:49.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:49.778 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:49.845 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:49.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:49.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:49.953 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:49.991 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:50.063 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:50.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:50.170 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:50.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 56 16 78796 78780 87.864 61.25 0.013547 0.0113769 2023-06-10T21:46:50.216 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:50.225 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1712633138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:50.238 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-06-10T21:46:50.293 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:50.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:50.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:46:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1712633138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3045465647' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:50.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:50 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:50.453 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:50.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1712633138' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:46:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3045465647' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:50.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3045465647' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:50.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:50 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:46:51.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 57 16 79656 79640 87.2658 53.75 0.0128033 0.0114548 2023-06-10T21:46:51.240 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: osdmap e435: 8 total, 8 up, 4 in 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.241 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[186706]: pgmap v1599: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 804 op/s 2023-06-10T21:46:51.524 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: osdmap e435: 8 total, 8 up, 4 in 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:51 smithi064 ceph-mon[180297]: pgmap v1599: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 804 op/s 2023-06-10T21:46:51.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:46:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:51.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:46:51.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: osdmap e435: 8 total, 8 up, 4 in 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: failed to encode map e435 with expected crc 2023-06-10T21:46:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:51 smithi152 ceph-mon[156819]: pgmap v1599: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 50 MiB/s wr, 804 op/s 2023-06-10T21:46:51.717 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:51.800 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:51.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:51.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:51.970 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:52.069 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:52.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:52.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:52.188 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:52.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 58 16 80852 80836 87.0501 74.75 0.012173 0.0114828 2023-06-10T21:46:52.250 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:52.302 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:46:52.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:46:52.339 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:52.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:52.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:52.391 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:52.456 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:52.498 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:52.574 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:52.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:52.679 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:52.740 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:52.815 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:52.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:52.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:52.924 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:46:52.968 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:52.985 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:46:52.985 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:46:53.051 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:46:53.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:53.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:46:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:46:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:46:52] "GET /metrics HTTP/1.1" 200 224094 "" "Prometheus/2.33.4" 2023-06-10T21:46:53.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:53.206 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 59 16 81845 81829 86.6267 62.0625 0.0196406 0.0115394 2023-06-10T21:46:53.208 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:53.290 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:53.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:53.399 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:53.465 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:53.474 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:46:53.529 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:53 smithi064 ceph-mon[180297]: pgmap v1600: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:46:53.529 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:53 smithi064 ceph-mon[186706]: pgmap v1600: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:46:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:53 smithi152 ceph-mon[156819]: pgmap v1600: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-10T21:46:54.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:46:54.205311+0000 min lat: 0.00126485 max lat: 1.07351 avg lat: 0.0116052 2023-06-10T21:46:54.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:46:54.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 60 16 82761 82745 86.1373 57.25 0.00561041 0.0116052 2023-06-10T21:46:54.762 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:54.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:54.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:54.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:54.874 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:46:54.959 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:55.124 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:55.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:55.146 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:55.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 61 16 83703 83687 85.6905 58.875 0.00765416 0.0116648 2023-06-10T21:46:55.243 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:55.320 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:55.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:55 smithi064 ceph-mon[186706]: pgmap v1601: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:55.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:55 smithi064 ceph-mon[180297]: pgmap v1601: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:55.599 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:55.629 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:55.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:55.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:55 smithi152 ceph-mon[156819]: pgmap v1601: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:55.735 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:55.772 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:55.844 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:55.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:55.952 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:56.009 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:56.091 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:56.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:56.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:56.356 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 62 16 84716 84700 85.3297 63.3125 0.0131142 0.0117152 2023-06-10T21:46:56.356 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:46:56.407 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:56.418 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:56.516 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:56.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:56.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:56.624 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:56.652 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:56.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:46:56.656 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:57.161 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:46:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:57.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:46:57.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 63 16 85737 85721 84.9881 63.8125 0.00904897 0.0117613 2023-06-10T21:46:57.269 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:57.295 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:57.339 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:57.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:57.447 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:46:57.518 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:57.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: pgmap v1602: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:57.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:57.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:57.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: osdmap e436: 8 total, 8 up, 4 in 2023-06-10T21:46:57.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[186706]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: pgmap v1602: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:57.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:57.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:57.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: osdmap e436: 8 total, 8 up, 4 in 2023-06-10T21:46:57.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:57 smithi064 ceph-mon[180297]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: pgmap v1602: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 61 MiB/s wr, 983 op/s 2023-06-10T21:46:57.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:46:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3553647289' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:46:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: osdmap e436: 8 total, 8 up, 4 in 2023-06-10T21:46:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:57 smithi152 ceph-mon[156819]: failed to encode map e436 with expected crc 2023-06-10T21:46:57.821 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:46:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:57.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:46:57.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:46:57.994 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:58.064 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:46:58.085 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:46:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:58.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:46:58.196 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:46:58.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 64 16 86647 86631 84.549 56.875 0.0151964 0.0118219 2023-06-10T21:46:58.240 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:58.341 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:46:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:58.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:46:58.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:46:58.495 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:58 smithi152 ceph-mon[156819]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:58 smithi152 ceph-mon[156819]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.496 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:58 smithi152 ceph-mon[156819]: pgmap v1604: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:46:58.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[186706]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[186706]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.503 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[186706]: pgmap v1604: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:46:58.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[180297]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[180297]: failed to encode map e436 with expected crc 2023-06-10T21:46:58.505 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:58 smithi064 ceph-mon[180297]: pgmap v1604: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-10T21:46:58.509 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:58.581 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:46:58.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:58.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:46:58.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:46:58.749 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:59.184 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:46:59.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 65 16 87658 87642 84.2204 63.1875 0.0175269 0.0118691 2023-06-10T21:46:59.207 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:46:59.300 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:46:59.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:46:59.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:46:59.407 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:46:59.464 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:46:59.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:46:59 smithi064 ceph-mon[180297]: from='client.47237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:59.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:46:59 smithi064 ceph-mon[186706]: from='client.47237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:46:59 smithi152 ceph-mon[156819]: from='client.47237 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:46:59.867 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:47:00.207 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 66 16 88562 88546 83.8005 56.5 0.0112166 0.0119277 2023-06-10T21:47:00.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:00 smithi064 ceph-mon[186706]: pgmap v1605: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 985 op/s 2023-06-10T21:47:00.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:00 smithi064 ceph-mon[180297]: pgmap v1605: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 985 op/s 2023-06-10T21:47:00.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:00 smithi152 ceph-mon[156819]: pgmap v1605: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 985 op/s 2023-06-10T21:47:00.828 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1604,"stamp":"2023-06-10T21:46:58.251789+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5525280654,"num_objects":84324,"num_object_clones":0,"num_object_copies":252972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84324,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":84561,"num_write_kb":5397646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":2377742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75595,"ondisk_log_size":75595,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":19297072,"kb_used_data":16028304,"kb_used_omap":27,"kb_used_meta":3268708,"kb_avail":355683536,"statfs":{"total":383980142592,"available":364219940864,"internally_reserved":0,"allocated":16412983296,"data_stored":16408643073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3347157594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":568459264,"num_objects":8674,"num_object_clones":0,"num_object_copies":26022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8674,"num_write_kb":555136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.039273"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":476,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.214972+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:46:50.214972+0000","last_peered":"2023-06-10T21:46:50.214972+0000","last_clean":"2023-06-10T21:46:50.214972+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:46:50.214972+0000","last_undegraded":"2023-06-10T21:46:50.214972+0000","last_fullsized":"2023-06-10T21:46:50.214972+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401091+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:46:45.401091+0000","last_peered":"2023-06-10T21:46:45.401091+0000","last_clean":"2023-06-10T21:46:45.401091+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:46:45.401091+0000","last_undegraded":"2023-06-10T21:46:45.401091+0000","last_fullsized":"2023-06-10T21:46:45.401091+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":367,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401409+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:46:45.401409+0000","last_peered":"2023-06-10T21:46:45.401409+0000","last_clean":"2023-06-10T21:46:45.401409+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:46:45.401409+0000","last_undegraded":"2023-06-10T21:46:45.401409+0000","last_fullsized":"2023-06-10T21:46:45.401409+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":320,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393785+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:46:45.393785+0000","last_peered":"2023-06-10T21:46:45.393785+0000","last_clean":"2023-06-10T21:46:45.393785+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:46:45.393785+0000","last_undegraded":"2023-06-10T21:46:45.393785+0000","last_fullsized":"2023-06-10T21:46:45.393785+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400080+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:46:45.400080+0000","last_peered":"2023-06-10T21:46:45.400080+0000","last_clean":"2023-06-10T21:46:45.400080+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:46:45.400080+0000","last_undegraded":"2023-06-10T21:46:45.400080+0000","last_fullsized":"2023-06-10T21:46:45.400080+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":84,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401618+0000","last_change":"2023-06-10T21:46:14.280677+0000","last_active":"2023-06-10T21:46:45.401618+0000","last_peered":"2023-06-10T21:46:45.401618+0000","last_clean":"2023-06-10T21:46:45.401618+0000","last_became_active":"2023-06-10T21:46:14.280384+0000","last_became_peered":"2023-06-10T21:46:14.280384+0000","last_unstale":"2023-06-10T21:46:45.401618+0000","last_undegraded":"2023-06-10T21:46:45.401618+0000","last_fullsized":"2023-06-10T21:46:45.401618+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:20:53.330571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":267,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401485+0000","last_change":"2023-06-10T21:46:14.280309+0000","last_active":"2023-06-10T21:46:45.401485+0000","last_peered":"2023-06-10T21:46:45.401485+0000","last_clean":"2023-06-10T21:46:45.401485+0000","last_became_active":"2023-06-10T21:46:14.280033+0000","last_became_peered":"2023-06-10T21:46:14.280033+0000","last_unstale":"2023-06-10T21:46:45.401485+0000","last_undegraded":"2023-06-10T21:46:45.401485+0000","last_fullsized":"2023-06-10T21:46:45.401485+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:44:39.412406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400169+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:46:45.400169+0000","last_peered":"2023-06-10T21:46:45.400169+0000","last_clean":"2023-06-10T21:46:45.400169+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:46:45.400169+0000","last_undegraded":"2023-06-10T21:46:45.400169+0000","last_fullsized":"2023-06-10T21:46:45.400169+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":95,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393872+0000","last_change":"2023-06-10T21:46:14.280460+0000","last_active":"2023-06-10T21:46:45.393872+0000","last_peered":"2023-06-10T21:46:45.393872+0000","last_clean":"2023-06-10T21:46:45.393872+0000","last_became_active":"2023-06-10T21:46:14.280171+0000","last_became_peered":"2023-06-10T21:46:14.280171+0000","last_unstale":"2023-06-10T21:46:45.393872+0000","last_undegraded":"2023-06-10T21:46:45.393872+0000","last_fullsized":"2023-06-10T21:46:45.393872+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":187,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221734+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:46:50.221734+0000","last_peered":"2023-06-10T21:46:50.221734+0000","last_clean":"2023-06-10T21:46:50.221734+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:46:50.221734+0000","last_undegraded":"2023-06-10T21:46:50.221734+0000","last_fullsized":"2023-06-10T21:46:50.221734+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":192,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395548+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:46:45.395548+0000","last_peered":"2023-06-10T21:46:45.395548+0000","last_clean":"2023-06-10T21:46:45.395548+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:46:45.395548+0000","last_undegraded":"2023-06-10T21:46:45.395548+0000","last_fullsized":"2023-06-10T21:46:45.395548+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":436,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215052+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:46:50.215052+0000","last_peered":"2023-06-10T21:46:50.215052+0000","last_clean":"2023-06-10T21:46:50.215052+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:46:50.215052+0000","last_undegraded":"2023-06-10T21:46:50.215052+0000","last_fullsized":"2023-06-10T21:46:50.215052+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":358,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.394828+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:46:45.394828+0000","last_peered":"2023-06-10T21:46:45.394828+0000","last_clean":"2023-06-10T21:46:45.394828+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:46:45.394828+0000","last_undegraded":"2023-06-10T21:46:45.394828+0000","last_fullsized":"2023-06-10T21:46:45.394828+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215156+0000","last_change":"2023-06-10T21:46:14.281135+0000","last_active":"2023-06-10T21:46:50.215156+0000","last_peered":"2023-06-10T21:46:50.215156+0000","last_clean":"2023-06-10T21:46:50.215156+0000","last_became_active":"2023-06-10T21:46:14.280752+0000","last_became_peered":"2023-06-10T21:46:14.280752+0000","last_unstale":"2023-06-10T21:46:50.215156+0000","last_undegraded":"2023-06-10T21:46:50.215156+0000","last_fullsized":"2023-06-10T21:46:50.215156+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:30:54.587316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":494,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395149+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:46:45.395149+0000","last_peered":"2023-06-10T21:46:45.395149+0000","last_clean":"2023-06-10T21:46:45.395149+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:46:45.395149+0000","last_undegraded":"2023-06-10T21:46:45.395149+0000","last_fullsized":"2023-06-10T21:46:45.395149+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":383,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400482+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:46:45.400482+0000","last_peered":"2023-06-10T21:46:45.400482+0000","last_clean":"2023-06-10T21:46:45.400482+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:46:45.400482+0000","last_undegraded":"2023-06-10T21:46:45.400482+0000","last_fullsized":"2023-06-10T21:46:45.400482+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":305,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393412+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:46:45.393412+0000","last_peered":"2023-06-10T21:46:45.393412+0000","last_clean":"2023-06-10T21:46:45.393412+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:46:45.393412+0000","last_undegraded":"2023-06-10T21:46:45.393412+0000","last_fullsized":"2023-06-10T21:46:45.393412+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":302,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395829+0000","last_change":"2023-06-10T21:46:14.281414+0000","last_active":"2023-06-10T21:46:45.395829+0000","last_peered":"2023-06-10T21:46:45.395829+0000","last_clean":"2023-06-10T21:46:45.395829+0000","last_became_active":"2023-06-10T21:46:14.280518+0000","last_became_peered":"2023-06-10T21:46:14.280518+0000","last_unstale":"2023-06-10T21:46:45.395829+0000","last_undegraded":"2023-06-10T21:46:45.395829+0000","last_fullsized":"2023-06-10T21:46:45.395829+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":447,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.396966+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:46:45.396966+0000","last_peered":"2023-06-10T21:46:45.396966+0000","last_clean":"2023-06-10T21:46:45.396966+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:46:45.396966+0000","last_undegraded":"2023-06-10T21:46:45.396966+0000","last_fullsized":"2023-06-10T21:46:45.396966+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402645+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:46:45.402645+0000","last_peered":"2023-06-10T21:46:45.402645+0000","last_clean":"2023-06-10T21:46:45.402645+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:46:45.402645+0000","last_undegraded":"2023-06-10T21:46:45.402645+0000","last_fullsized":"2023-06-10T21:46:45.402645+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":415,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.405314+0000","last_change":"2023-06-10T21:46:14.279613+0000","last_active":"2023-06-10T21:46:45.405314+0000","last_peered":"2023-06-10T21:46:45.405314+0000","last_clean":"2023-06-10T21:46:45.405314+0000","last_became_active":"2023-06-10T21:46:14.279252+0000","last_became_peered":"2023-06-10T21:46:14.279252+0000","last_unstale":"2023-06-10T21:46:45.405314+0000","last_undegraded":"2023-06-10T21:46:45.405314+0000","last_fullsized":"2023-06-10T21:46:45.405314+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:58:42.469928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401692+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:46:45.401692+0000","last_peered":"2023-06-10T21:46:45.401692+0000","last_clean":"2023-06-10T21:46:45.401692+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:46:45.401692+0000","last_undegraded":"2023-06-10T21:46:45.401692+0000","last_fullsized":"2023-06-10T21:46:45.401692+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"435'5254","reported_seq":5278,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.129911+0000","last_change":"2023-06-10T21:45:44.480492+0000","last_active":"2023-06-10T21:46:56.129911+0000","last_peered":"2023-06-10T21:46:56.129911+0000","last_clean":"2023-06-10T21:46:56.129911+0000","last_became_active":"2023-06-10T21:45:44.480237+0000","last_became_peered":"2023-06-10T21:45:44.480237+0000","last_unstale":"2023-06-10T21:46:56.129911+0000","last_undegraded":"2023-06-10T21:46:56.129911+0000","last_fullsized":"2023-06-10T21:46:56.129911+0000","mapping_epoch":420,"log_start":"424'1000","ondisk_log_start":"424'1000","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4254,"log_dups_size":0,"ondisk_log_size":4254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:15:36.524573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344326144,"num_objects":5254,"num_object_clones":0,"num_object_copies":15762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5254,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":227,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393736+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:46:45.393736+0000","last_peered":"2023-06-10T21:46:45.393736+0000","last_clean":"2023-06-10T21:46:45.393736+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:46:45.393736+0000","last_undegraded":"2023-06-10T21:46:45.393736+0000","last_fullsized":"2023-06-10T21:46:45.393736+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.392665+0000","last_change":"2023-06-10T21:46:14.279497+0000","last_active":"2023-06-10T21:46:45.392665+0000","last_peered":"2023-06-10T21:46:45.392665+0000","last_clean":"2023-06-10T21:46:45.392665+0000","last_became_active":"2023-06-10T21:46:14.279186+0000","last_became_peered":"2023-06-10T21:46:14.279186+0000","last_unstale":"2023-06-10T21:46:45.392665+0000","last_undegraded":"2023-06-10T21:46:45.392665+0000","last_fullsized":"2023-06-10T21:46:45.392665+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"435'5325","reported_seq":5347,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.106064+0000","last_change":"2023-06-10T21:45:44.477018+0000","last_active":"2023-06-10T21:46:56.106064+0000","last_peered":"2023-06-10T21:46:56.106064+0000","last_clean":"2023-06-10T21:46:56.106064+0000","last_became_active":"2023-06-10T21:45:44.476687+0000","last_became_peered":"2023-06-10T21:45:44.476687+0000","last_unstale":"2023-06-10T21:46:56.106064+0000","last_undegraded":"2023-06-10T21:46:56.106064+0000","last_fullsized":"2023-06-10T21:46:56.106064+0000","mapping_epoch":420,"log_start":"424'1000","ondisk_log_start":"424'1000","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4325,"log_dups_size":0,"ondisk_log_size":4325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:10:48.652326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348979200,"num_objects":5325,"num_object_clones":0,"num_object_copies":15975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5325,"num_write_kb":340800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":227,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393918+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:46:45.393918+0000","last_peered":"2023-06-10T21:46:45.393918+0000","last_clean":"2023-06-10T21:46:45.393918+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:46:45.393918+0000","last_undegraded":"2023-06-10T21:46:45.393918+0000","last_fullsized":"2023-06-10T21:46:45.393918+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":89,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401492+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:46:45.401492+0000","last_peered":"2023-06-10T21:46:45.401492+0000","last_clean":"2023-06-10T21:46:45.401492+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:46:45.401492+0000","last_undegraded":"2023-06-10T21:46:45.401492+0000","last_fullsized":"2023-06-10T21:46:45.401492+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"435'5317","reported_seq":5342,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.307439+0000","last_change":"2023-06-10T21:45:44.477367+0000","last_active":"2023-06-10T21:46:56.307439+0000","last_peered":"2023-06-10T21:46:56.307439+0000","last_clean":"2023-06-10T21:46:56.307439+0000","last_became_active":"2023-06-10T21:45:44.477107+0000","last_became_peered":"2023-06-10T21:45:44.477107+0000","last_unstale":"2023-06-10T21:46:56.307439+0000","last_undegraded":"2023-06-10T21:46:56.307439+0000","last_fullsized":"2023-06-10T21:46:56.307439+0000","mapping_epoch":420,"log_start":"423'100","ondisk_log_start":"423'100","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5217,"log_dups_size":0,"ondisk_log_size":5217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:35:06.980899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348454912,"num_objects":5317,"num_object_clones":0,"num_object_copies":15951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5317,"num_write_kb":340288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":514,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395622+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:46:45.395622+0000","last_peered":"2023-06-10T21:46:45.395622+0000","last_clean":"2023-06-10T21:46:45.395622+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:46:45.395622+0000","last_undegraded":"2023-06-10T21:46:45.395622+0000","last_fullsized":"2023-06-10T21:46:45.395622+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":198,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393313+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:46:45.393313+0000","last_peered":"2023-06-10T21:46:45.393313+0000","last_clean":"2023-06-10T21:46:45.393313+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:46:45.393313+0000","last_undegraded":"2023-06-10T21:46:45.393313+0000","last_fullsized":"2023-06-10T21:46:45.393313+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"435'5389","reported_seq":5413,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.066025+0000","last_change":"2023-06-10T21:45:44.476967+0000","last_active":"2023-06-10T21:46:56.066025+0000","last_peered":"2023-06-10T21:46:56.066025+0000","last_clean":"2023-06-10T21:46:56.066025+0000","last_became_active":"2023-06-10T21:45:44.476670+0000","last_became_peered":"2023-06-10T21:45:44.476670+0000","last_unstale":"2023-06-10T21:46:56.066025+0000","last_undegraded":"2023-06-10T21:46:56.066025+0000","last_fullsized":"2023-06-10T21:46:56.066025+0000","mapping_epoch":420,"log_start":"424'1100","ondisk_log_start":"424'1100","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4289,"log_dups_size":0,"ondisk_log_size":4289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:11:49.819095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353173504,"num_objects":5389,"num_object_clones":0,"num_object_copies":16167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5389,"num_write_kb":344896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":85,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400300+0000","last_change":"2023-06-10T21:46:14.277139+0000","last_active":"2023-06-10T21:46:45.400300+0000","last_peered":"2023-06-10T21:46:45.400300+0000","last_clean":"2023-06-10T21:46:45.400300+0000","last_became_active":"2023-06-10T21:46:14.276750+0000","last_became_peered":"2023-06-10T21:46:14.276750+0000","last_unstale":"2023-06-10T21:46:45.400300+0000","last_undegraded":"2023-06-10T21:46:45.400300+0000","last_fullsized":"2023-06-10T21:46:45.400300+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:51.450990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402418+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:46:45.402418+0000","last_peered":"2023-06-10T21:46:45.402418+0000","last_clean":"2023-06-10T21:46:45.402418+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:46:45.402418+0000","last_undegraded":"2023-06-10T21:46:45.402418+0000","last_fullsized":"2023-06-10T21:46:45.402418+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.6","version":"435'5204","reported_seq":5229,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.312511+0000","last_change":"2023-06-10T21:45:44.478058+0000","last_active":"2023-06-10T21:46:56.312511+0000","last_peered":"2023-06-10T21:46:56.312511+0000","last_clean":"2023-06-10T21:46:56.312511+0000","last_became_active":"2023-06-10T21:45:44.477758+0000","last_became_peered":"2023-06-10T21:45:44.477758+0000","last_unstale":"2023-06-10T21:46:56.312511+0000","last_undegraded":"2023-06-10T21:46:56.312511+0000","last_fullsized":"2023-06-10T21:46:56.312511+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:39:43.723763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341049344,"num_objects":5204,"num_object_clones":0,"num_object_copies":15612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5204,"num_write_kb":333056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":93,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402320+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:46:45.402320+0000","last_peered":"2023-06-10T21:46:45.402320+0000","last_clean":"2023-06-10T21:46:45.402320+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:46:45.402320+0000","last_undegraded":"2023-06-10T21:46:45.402320+0000","last_fullsized":"2023-06-10T21:46:45.402320+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":89,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400396+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:46:45.400396+0000","last_peered":"2023-06-10T21:46:45.400396+0000","last_clean":"2023-06-10T21:46:45.400396+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:46:45.400396+0000","last_undegraded":"2023-06-10T21:46:45.400396+0000","last_fullsized":"2023-06-10T21:46:45.400396+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"435'5134","reported_seq":5165,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:54.264901+0000","last_change":"2023-06-10T21:46:14.286123+0000","last_active":"2023-06-10T21:46:54.264901+0000","last_peered":"2023-06-10T21:46:54.264901+0000","last_clean":"2023-06-10T21:46:54.264901+0000","last_became_active":"2023-06-10T21:46:14.283059+0000","last_became_peered":"2023-06-10T21:46:14.283059+0000","last_unstale":"2023-06-10T21:46:54.264901+0000","last_undegraded":"2023-06-10T21:46:54.264901+0000","last_fullsized":"2023-06-10T21:46:54.264901+0000","mapping_epoch":426,"log_start":"424'800","ondisk_log_start":"424'800","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4334,"log_dups_size":0,"ondisk_log_size":4334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5134,"num_write_kb":328576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":257,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.403988+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:46:45.403988+0000","last_peered":"2023-06-10T21:46:45.403988+0000","last_clean":"2023-06-10T21:46:45.403988+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:46:45.403988+0000","last_undegraded":"2023-06-10T21:46:45.403988+0000","last_fullsized":"2023-06-10T21:46:45.403988+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":475,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215570+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:46:50.215570+0000","last_peered":"2023-06-10T21:46:50.215570+0000","last_clean":"2023-06-10T21:46:50.215570+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:46:50.215570+0000","last_undegraded":"2023-06-10T21:46:50.215570+0000","last_fullsized":"2023-06-10T21:46:50.215570+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"435'5113","reported_seq":5134,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:54.293818+0000","last_change":"2023-06-10T21:45:44.480834+0000","last_active":"2023-06-10T21:46:54.293818+0000","last_peered":"2023-06-10T21:46:54.293818+0000","last_clean":"2023-06-10T21:46:54.293818+0000","last_became_active":"2023-06-10T21:45:44.480535+0000","last_became_peered":"2023-06-10T21:45:44.480535+0000","last_unstale":"2023-06-10T21:46:54.293818+0000","last_undegraded":"2023-06-10T21:46:54.293818+0000","last_fullsized":"2023-06-10T21:46:54.293818+0000","mapping_epoch":420,"log_start":"424'800","ondisk_log_start":"424'800","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4313,"log_dups_size":0,"ondisk_log_size":4313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335085568,"num_objects":5113,"num_object_clones":0,"num_object_copies":15339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5113,"num_write_kb":327232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":452,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.216861+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:46:50.216861+0000","last_peered":"2023-06-10T21:46:50.216861+0000","last_clean":"2023-06-10T21:46:50.216861+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:46:50.216861+0000","last_undegraded":"2023-06-10T21:46:50.216861+0000","last_fullsized":"2023-06-10T21:46:50.216861+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402346+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:46:45.402346+0000","last_peered":"2023-06-10T21:46:45.402346+0000","last_clean":"2023-06-10T21:46:45.402346+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:46:45.402346+0000","last_undegraded":"2023-06-10T21:46:45.402346+0000","last_fullsized":"2023-06-10T21:46:45.402346+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"435'5249","reported_seq":5272,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:54.262119+0000","last_change":"2023-06-10T21:45:44.482366+0000","last_active":"2023-06-10T21:46:54.262119+0000","last_peered":"2023-06-10T21:46:54.262119+0000","last_clean":"2023-06-10T21:46:54.262119+0000","last_became_active":"2023-06-10T21:45:44.482065+0000","last_became_peered":"2023-06-10T21:45:44.482065+0000","last_unstale":"2023-06-10T21:46:54.262119+0000","last_undegraded":"2023-06-10T21:46:54.262119+0000","last_fullsized":"2023-06-10T21:46:54.262119+0000","mapping_epoch":420,"log_start":"424'900","ondisk_log_start":"424'900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4349,"log_dups_size":0,"ondisk_log_size":4349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":335936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.405167+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:46:45.405167+0000","last_peered":"2023-06-10T21:46:45.405167+0000","last_clean":"2023-06-10T21:46:45.405167+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:46:45.405167+0000","last_undegraded":"2023-06-10T21:46:45.405167+0000","last_fullsized":"2023-06-10T21:46:45.405167+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":383,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401922+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:46:45.401922+0000","last_peered":"2023-06-10T21:46:45.401922+0000","last_clean":"2023-06-10T21:46:45.401922+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:46:45.401922+0000","last_undegraded":"2023-06-10T21:46:45.401922+0000","last_fullsized":"2023-06-10T21:46:45.401922+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"435'5277","reported_seq":5302,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.307288+0000","last_change":"2023-06-10T21:45:44.477475+0000","last_active":"2023-06-10T21:46:56.307288+0000","last_peered":"2023-06-10T21:46:56.307288+0000","last_clean":"2023-06-10T21:46:56.307288+0000","last_became_active":"2023-06-10T21:45:44.477218+0000","last_became_peered":"2023-06-10T21:45:44.477218+0000","last_unstale":"2023-06-10T21:46:56.307288+0000","last_undegraded":"2023-06-10T21:46:56.307288+0000","last_fullsized":"2023-06-10T21:46:56.307288+0000","mapping_epoch":420,"log_start":"423'100","ondisk_log_start":"423'100","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:15:26.255634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345833472,"num_objects":5277,"num_object_clones":0,"num_object_copies":15831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5277,"num_write_kb":337728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":218,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.394852+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:46:45.394852+0000","last_peered":"2023-06-10T21:46:45.394852+0000","last_clean":"2023-06-10T21:46:45.394852+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:46:45.394852+0000","last_undegraded":"2023-06-10T21:46:45.394852+0000","last_fullsized":"2023-06-10T21:46:45.394852+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":66,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400789+0000","last_change":"2023-06-10T21:46:14.277043+0000","last_active":"2023-06-10T21:46:45.400789+0000","last_peered":"2023-06-10T21:46:45.400789+0000","last_clean":"2023-06-10T21:46:45.400789+0000","last_became_active":"2023-06-10T21:46:14.276737+0000","last_became_peered":"2023-06-10T21:46:14.276737+0000","last_unstale":"2023-06-10T21:46:45.400789+0000","last_undegraded":"2023-06-10T21:46:45.400789+0000","last_fullsized":"2023-06-10T21:46:45.400789+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:03.540332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":203,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215124+0000","last_change":"2023-06-10T21:46:14.279638+0000","last_active":"2023-06-10T21:46:50.215124+0000","last_peered":"2023-06-10T21:46:50.215124+0000","last_clean":"2023-06-10T21:46:50.215124+0000","last_became_active":"2023-06-10T21:46:14.279320+0000","last_became_peered":"2023-06-10T21:46:14.279320+0000","last_unstale":"2023-06-10T21:46:50.215124+0000","last_undegraded":"2023-06-10T21:46:50.215124+0000","last_fullsized":"2023-06-10T21:46:50.215124+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:05:18.656099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"435'5418","reported_seq":5443,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.299550+0000","last_change":"2023-06-10T21:45:44.480812+0000","last_active":"2023-06-10T21:46:56.299550+0000","last_peered":"2023-06-10T21:46:56.299550+0000","last_clean":"2023-06-10T21:46:56.299550+0000","last_became_active":"2023-06-10T21:45:44.479574+0000","last_became_peered":"2023-06-10T21:45:44.479574+0000","last_unstale":"2023-06-10T21:46:56.299550+0000","last_undegraded":"2023-06-10T21:46:56.299550+0000","last_fullsized":"2023-06-10T21:46:56.299550+0000","mapping_epoch":420,"log_start":"423'200","ondisk_log_start":"423'200","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5218,"log_dups_size":0,"ondisk_log_size":5218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:01:32.420364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355074048,"num_objects":5418,"num_object_clones":0,"num_object_copies":16254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5418,"num_write_kb":346752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":85,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.403872+0000","last_change":"2023-06-10T21:46:14.279110+0000","last_active":"2023-06-10T21:46:45.403872+0000","last_peered":"2023-06-10T21:46:45.403872+0000","last_clean":"2023-06-10T21:46:45.403872+0000","last_became_active":"2023-06-10T21:46:14.278624+0000","last_became_peered":"2023-06-10T21:46:14.278624+0000","last_unstale":"2023-06-10T21:46:45.403872+0000","last_undegraded":"2023-06-10T21:46:45.403872+0000","last_fullsized":"2023-06-10T21:46:45.403872+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:59:27.858698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":363,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402219+0000","last_change":"2023-06-10T21:46:14.278053+0000","last_active":"2023-06-10T21:46:45.402219+0000","last_peered":"2023-06-10T21:46:45.402219+0000","last_clean":"2023-06-10T21:46:45.402219+0000","last_became_active":"2023-06-10T21:46:14.277736+0000","last_became_peered":"2023-06-10T21:46:14.277736+0000","last_unstale":"2023-06-10T21:46:45.402219+0000","last_undegraded":"2023-06-10T21:46:45.402219+0000","last_fullsized":"2023-06-10T21:46:45.402219+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:48:27.204561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":302,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217055+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:46:50.217055+0000","last_peered":"2023-06-10T21:46:50.217055+0000","last_clean":"2023-06-10T21:46:50.217055+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:46:50.217055+0000","last_undegraded":"2023-06-10T21:46:50.217055+0000","last_fullsized":"2023-06-10T21:46:50.217055+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"435'5357","reported_seq":5389,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.295646+0000","last_change":"2023-06-10T21:46:14.285886+0000","last_active":"2023-06-10T21:46:56.295646+0000","last_peered":"2023-06-10T21:46:56.295646+0000","last_clean":"2023-06-10T21:46:56.295646+0000","last_became_active":"2023-06-10T21:46:14.280572+0000","last_became_peered":"2023-06-10T21:46:14.280572+0000","last_unstale":"2023-06-10T21:46:56.295646+0000","last_undegraded":"2023-06-10T21:46:56.295646+0000","last_fullsized":"2023-06-10T21:46:56.295646+0000","mapping_epoch":426,"log_start":"423'100","ondisk_log_start":"423'100","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5257,"log_dups_size":0,"ondisk_log_size":5257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:21:14.063231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351076352,"num_objects":5357,"num_object_clones":0,"num_object_copies":16071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5357,"num_write_kb":342848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":322,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.216785+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:46:50.216785+0000","last_peered":"2023-06-10T21:46:50.216785+0000","last_clean":"2023-06-10T21:46:50.216785+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:46:50.216785+0000","last_undegraded":"2023-06-10T21:46:50.216785+0000","last_fullsized":"2023-06-10T21:46:50.216785+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.218385+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:46:50.218385+0000","last_peered":"2023-06-10T21:46:50.218385+0000","last_clean":"2023-06-10T21:46:50.218385+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:46:50.218385+0000","last_undegraded":"2023-06-10T21:46:50.218385+0000","last_fullsized":"2023-06-10T21:46:50.218385+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393822+0000","last_change":"2023-06-10T21:46:14.282107+0000","last_active":"2023-06-10T21:46:45.393822+0000","last_peered":"2023-06-10T21:46:45.393822+0000","last_clean":"2023-06-10T21:46:45.393822+0000","last_became_active":"2023-06-10T21:46:14.281805+0000","last_became_peered":"2023-06-10T21:46:14.281805+0000","last_unstale":"2023-06-10T21:46:45.393822+0000","last_undegraded":"2023-06-10T21:46:45.393822+0000","last_fullsized":"2023-06-10T21:46:45.393822+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.f","version":"435'5176","reported_seq":5199,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:54.265154+0000","last_change":"2023-06-10T21:45:44.480803+0000","last_active":"2023-06-10T21:46:54.265154+0000","last_peered":"2023-06-10T21:46:54.265154+0000","last_clean":"2023-06-10T21:46:54.265154+0000","last_became_active":"2023-06-10T21:45:44.480472+0000","last_became_peered":"2023-06-10T21:45:44.480472+0000","last_unstale":"2023-06-10T21:46:54.265154+0000","last_undegraded":"2023-06-10T21:46:54.265154+0000","last_fullsized":"2023-06-10T21:46:54.265154+0000","mapping_epoch":420,"log_start":"424'800","ondisk_log_start":"424'800","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4376,"log_dups_size":0,"ondisk_log_size":4376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339214336,"num_objects":5176,"num_object_clones":0,"num_object_copies":15528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5176,"num_write_kb":331264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":392,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215226+0000","last_change":"2023-06-10T21:46:14.280999+0000","last_active":"2023-06-10T21:46:50.215226+0000","last_peered":"2023-06-10T21:46:50.215226+0000","last_clean":"2023-06-10T21:46:50.215226+0000","last_became_active":"2023-06-10T21:46:14.280409+0000","last_became_peered":"2023-06-10T21:46:14.280409+0000","last_unstale":"2023-06-10T21:46:50.215226+0000","last_undegraded":"2023-06-10T21:46:50.215226+0000","last_fullsized":"2023-06-10T21:46:50.215226+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:24:46.683725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":635,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.392761+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:46:45.392761+0000","last_peered":"2023-06-10T21:46:45.392761+0000","last_clean":"2023-06-10T21:46:45.392761+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:46:45.392761+0000","last_undegraded":"2023-06-10T21:46:45.392761+0000","last_fullsized":"2023-06-10T21:46:45.392761+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":105,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:46.139439+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:46:46.139439+0000","last_peered":"2023-06-10T21:46:46.139439+0000","last_clean":"2023-06-10T21:46:46.139439+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:46:46.139439+0000","last_undegraded":"2023-06-10T21:46:46.139439+0000","last_fullsized":"2023-06-10T21:46:46.139439+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":318,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395768+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:46:45.395768+0000","last_peered":"2023-06-10T21:46:45.395768+0000","last_clean":"2023-06-10T21:46:45.395768+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:46:45.395768+0000","last_undegraded":"2023-06-10T21:46:45.395768+0000","last_fullsized":"2023-06-10T21:46:45.395768+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"435'5259","reported_seq":5283,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.295923+0000","last_change":"2023-06-10T21:45:44.478367+0000","last_active":"2023-06-10T21:46:56.295923+0000","last_peered":"2023-06-10T21:46:56.295923+0000","last_clean":"2023-06-10T21:46:56.295923+0000","last_became_active":"2023-06-10T21:45:44.476855+0000","last_became_peered":"2023-06-10T21:45:44.476855+0000","last_unstale":"2023-06-10T21:46:56.295923+0000","last_undegraded":"2023-06-10T21:46:56.295923+0000","last_fullsized":"2023-06-10T21:46:56.295923+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5259,"log_dups_size":0,"ondisk_log_size":5259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:43.311166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344653824,"num_objects":5259,"num_object_clones":0,"num_object_copies":15777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5259,"num_write_kb":336576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":487,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.392842+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:46:45.392842+0000","last_peered":"2023-06-10T21:46:45.392842+0000","last_clean":"2023-06-10T21:46:45.392842+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:46:45.392842+0000","last_undegraded":"2023-06-10T21:46:45.392842+0000","last_fullsized":"2023-06-10T21:46:45.392842+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":105,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:46.139230+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:46:46.139230+0000","last_peered":"2023-06-10T21:46:46.139230+0000","last_clean":"2023-06-10T21:46:46.139230+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:46:46.139230+0000","last_undegraded":"2023-06-10T21:46:46.139230+0000","last_fullsized":"2023-06-10T21:46:46.139230+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":188,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.219222+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:46:50.219222+0000","last_peered":"2023-06-10T21:46:50.219222+0000","last_clean":"2023-06-10T21:46:50.219222+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:46:50.219222+0000","last_undegraded":"2023-06-10T21:46:50.219222+0000","last_fullsized":"2023-06-10T21:46:50.219222+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"435'5277","reported_seq":5306,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.132844+0000","last_change":"2023-06-10T21:46:14.282840+0000","last_active":"2023-06-10T21:46:56.132844+0000","last_peered":"2023-06-10T21:46:56.132844+0000","last_clean":"2023-06-10T21:46:56.132844+0000","last_became_active":"2023-06-10T21:46:14.280350+0000","last_became_peered":"2023-06-10T21:46:14.280350+0000","last_unstale":"2023-06-10T21:46:56.132844+0000","last_undegraded":"2023-06-10T21:46:56.132844+0000","last_fullsized":"2023-06-10T21:46:56.132844+0000","mapping_epoch":426,"log_start":"424'1000","ondisk_log_start":"424'1000","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4277,"log_dups_size":0,"ondisk_log_size":4277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:35:53.086699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345833472,"num_objects":5277,"num_object_clones":0,"num_object_copies":15831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5277,"num_write_kb":337728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":466,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395701+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:46:45.395701+0000","last_peered":"2023-06-10T21:46:45.395701+0000","last_clean":"2023-06-10T21:46:45.395701+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:46:45.395701+0000","last_undegraded":"2023-06-10T21:46:45.395701+0000","last_fullsized":"2023-06-10T21:46:45.395701+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":317,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217156+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:46:50.217156+0000","last_peered":"2023-06-10T21:46:50.217156+0000","last_clean":"2023-06-10T21:46:50.217156+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:46:50.217156+0000","last_undegraded":"2023-06-10T21:46:50.217156+0000","last_fullsized":"2023-06-10T21:46:50.217156+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":206,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395225+0000","last_change":"2023-06-10T21:46:14.283198+0000","last_active":"2023-06-10T21:46:45.395225+0000","last_peered":"2023-06-10T21:46:45.395225+0000","last_clean":"2023-06-10T21:46:45.395225+0000","last_became_active":"2023-06-10T21:46:14.282871+0000","last_became_peered":"2023-06-10T21:46:14.282871+0000","last_unstale":"2023-06-10T21:46:45.395225+0000","last_undegraded":"2023-06-10T21:46:45.395225+0000","last_fullsized":"2023-06-10T21:46:45.395225+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.a","version":"435'5290","reported_seq":5320,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.139843+0000","last_change":"2023-06-10T21:46:14.285180+0000","last_active":"2023-06-10T21:46:56.139843+0000","last_peered":"2023-06-10T21:46:56.139843+0000","last_clean":"2023-06-10T21:46:56.139843+0000","last_became_active":"2023-06-10T21:46:14.279416+0000","last_became_peered":"2023-06-10T21:46:14.279416+0000","last_unstale":"2023-06-10T21:46:56.139843+0000","last_undegraded":"2023-06-10T21:46:56.139843+0000","last_fullsized":"2023-06-10T21:46:56.139843+0000","mapping_epoch":426,"log_start":"424'1000","ondisk_log_start":"424'1000","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4290,"log_dups_size":0,"ondisk_log_size":4290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:05:25.130576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346685440,"num_objects":5290,"num_object_clones":0,"num_object_copies":15870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5290,"num_write_kb":338560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":446,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401778+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:46:45.401778+0000","last_peered":"2023-06-10T21:46:45.401778+0000","last_clean":"2023-06-10T21:46:45.401778+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:46:45.401778+0000","last_undegraded":"2023-06-10T21:46:45.401778+0000","last_fullsized":"2023-06-10T21:46:45.401778+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":257,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402627+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:46:45.402627+0000","last_peered":"2023-06-10T21:46:45.402627+0000","last_clean":"2023-06-10T21:46:45.402627+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:46:45.402627+0000","last_undegraded":"2023-06-10T21:46:45.402627+0000","last_fullsized":"2023-06-10T21:46:45.402627+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":105,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.405089+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:46:45.405089+0000","last_peered":"2023-06-10T21:46:45.405089+0000","last_clean":"2023-06-10T21:46:45.405089+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:46:45.405089+0000","last_undegraded":"2023-06-10T21:46:45.405089+0000","last_fullsized":"2023-06-10T21:46:45.405089+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"435'5261","reported_seq":5286,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.306778+0000","last_change":"2023-06-10T21:45:44.477091+0000","last_active":"2023-06-10T21:46:56.306778+0000","last_peered":"2023-06-10T21:46:56.306778+0000","last_clean":"2023-06-10T21:46:56.306778+0000","last_became_active":"2023-06-10T21:45:44.476764+0000","last_became_peered":"2023-06-10T21:45:44.476764+0000","last_unstale":"2023-06-10T21:46:56.306778+0000","last_undegraded":"2023-06-10T21:46:56.306778+0000","last_fullsized":"2023-06-10T21:46:56.306778+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5261,"log_dups_size":0,"ondisk_log_size":5261,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:40:54.289558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344784896,"num_objects":5261,"num_object_clones":0,"num_object_copies":15783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5261,"num_write_kb":336704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":183,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393735+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:46:45.393735+0000","last_peered":"2023-06-10T21:46:45.393735+0000","last_clean":"2023-06-10T21:46:45.393735+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:46:45.393735+0000","last_undegraded":"2023-06-10T21:46:45.393735+0000","last_fullsized":"2023-06-10T21:46:45.393735+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":297,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.393596+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:46:45.393596+0000","last_peered":"2023-06-10T21:46:45.393596+0000","last_clean":"2023-06-10T21:46:45.393596+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:46:45.393596+0000","last_undegraded":"2023-06-10T21:46:45.393596+0000","last_fullsized":"2023-06-10T21:46:45.393596+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":314,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.396790+0000","last_change":"2023-06-10T21:46:14.280962+0000","last_active":"2023-06-10T21:46:45.396790+0000","last_peered":"2023-06-10T21:46:45.396790+0000","last_clean":"2023-06-10T21:46:45.396790+0000","last_became_active":"2023-06-10T21:46:14.280651+0000","last_became_peered":"2023-06-10T21:46:14.280651+0000","last_unstale":"2023-06-10T21:46:45.396790+0000","last_undegraded":"2023-06-10T21:46:45.396790+0000","last_fullsized":"2023-06-10T21:46:45.396790+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":134,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.395202+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:46:45.395202+0000","last_peered":"2023-06-10T21:46:45.395202+0000","last_clean":"2023-06-10T21:46:45.395202+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:46:45.395202+0000","last_undegraded":"2023-06-10T21:46:45.395202+0000","last_fullsized":"2023-06-10T21:46:45.395202+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":333,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217231+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:46:50.217231+0000","last_peered":"2023-06-10T21:46:50.217231+0000","last_clean":"2023-06-10T21:46:50.217231+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:46:50.217231+0000","last_undegraded":"2023-06-10T21:46:50.217231+0000","last_fullsized":"2023-06-10T21:46:50.217231+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":103,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.394921+0000","last_change":"2023-06-10T21:46:14.283333+0000","last_active":"2023-06-10T21:46:45.394921+0000","last_peered":"2023-06-10T21:46:45.394921+0000","last_clean":"2023-06-10T21:46:45.394921+0000","last_became_active":"2023-06-10T21:46:14.282987+0000","last_became_peered":"2023-06-10T21:46:14.282987+0000","last_unstale":"2023-06-10T21:46:45.394921+0000","last_undegraded":"2023-06-10T21:46:45.394921+0000","last_fullsized":"2023-06-10T21:46:45.394921+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":340,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402502+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:46:45.402502+0000","last_peered":"2023-06-10T21:46:45.402502+0000","last_clean":"2023-06-10T21:46:45.402502+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:46:45.402502+0000","last_undegraded":"2023-06-10T21:46:45.402502+0000","last_fullsized":"2023-06-10T21:46:45.402502+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":82,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.400919+0000","last_change":"2023-06-10T21:46:14.279723+0000","last_active":"2023-06-10T21:46:45.400919+0000","last_peered":"2023-06-10T21:46:45.400919+0000","last_clean":"2023-06-10T21:46:45.400919+0000","last_became_active":"2023-06-10T21:46:14.279371+0000","last_became_peered":"2023-06-10T21:46:14.279371+0000","last_unstale":"2023-06-10T21:46:45.400919+0000","last_undegraded":"2023-06-10T21:46:45.400919+0000","last_fullsized":"2023-06-10T21:46:45.400919+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:43.249586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":411,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.404017+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:46:45.404017+0000","last_peered":"2023-06-10T21:46:45.404017+0000","last_clean":"2023-06-10T21:46:45.404017+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:46:45.404017+0000","last_undegraded":"2023-06-10T21:46:45.404017+0000","last_fullsized":"2023-06-10T21:46:45.404017+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":90,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.402471+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:46:45.402471+0000","last_peered":"2023-06-10T21:46:45.402471+0000","last_clean":"2023-06-10T21:46:45.402471+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:46:45.402471+0000","last_undegraded":"2023-06-10T21:46:45.402471+0000","last_fullsized":"2023-06-10T21:46:45.402471+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.401228+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:46:45.401228+0000","last_peered":"2023-06-10T21:46:45.401228+0000","last_clean":"2023-06-10T21:46:45.401228+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:46:45.401228+0000","last_undegraded":"2023-06-10T21:46:45.401228+0000","last_fullsized":"2023-06-10T21:46:45.401228+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":107,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215916+0000","last_change":"2023-06-10T21:46:14.280900+0000","last_active":"2023-06-10T21:46:50.215916+0000","last_peered":"2023-06-10T21:46:50.215916+0000","last_clean":"2023-06-10T21:46:50.215916+0000","last_became_active":"2023-06-10T21:46:14.280572+0000","last_became_peered":"2023-06-10T21:46:14.280572+0000","last_unstale":"2023-06-10T21:46:50.215916+0000","last_undegraded":"2023-06-10T21:46:50.215916+0000","last_fullsized":"2023-06-10T21:46:50.215916+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:13.502992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":434,"state":"active+clean","last_fresh":"2023-06-10T21:46:45.405239+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:46:45.405239+0000","last_peered":"2023-06-10T21:46:45.405239+0000","last_clean":"2023-06-10T21:46:45.405239+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:46:45.405239+0000","last_undegraded":"2023-06-10T21:46:45.405239+0000","last_fullsized":"2023-06-10T21:46:45.405239+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":5524684800,"num_objects":84300,"num_object_clones":0,"num_object_copies":252900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":84300,"num_write_kb":5395200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16491479040,"data_stored":16491479040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75400,"ondisk_log_size":75400,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430798,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4681032,"kb_used_data":3644480,"kb_used_omap":0,"kb_used_meta":1036544,"kb_avail":89064120,"statfs":{"total":95995035648,"available":91201658880,"internally_reserved":0,"allocated":3731947520,"data_stored":3731229645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1061421056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.006,"15min":2.504},"min":{"1min":0.253,"5min":0.253,"15min":0.224},"max":{"1min":4.048,"5min":8.5129999999999999,"15min":184.83699999999999},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.339,"5min":1.3720000000000001,"15min":2.8540000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.25},"max":{"1min":9.5050000000000008,"5min":20.568999999999999,"15min":185.58000000000001},"last":0.55800000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.2430000000000001,"15min":2.6299999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":2.1099999999999999,"5min":8.1500000000000004,"15min":87.417000000000002},"last":0.37},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.488,"15min":2.9609999999999999},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":3.1230000000000002,"5min":10.484,"15min":87.438000000000002},"last":2.6099999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:46:44 2023","interfaces":[{"interface":"back","average":{"1min":29.420000000000002,"5min":9.1989999999999998,"15min":6.5279999999999996},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":218.90600000000001,"5min":218.90600000000001,"15min":218.90600000000001},"last":4.0890000000000004},{"interface":"front","average":{"1min":29.443000000000001,"5min":9.3659999999999997,"15min":6.5599999999999996},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.30499999999999999},"max":{"1min":218.851,"5min":218.851,"15min":218.851},"last":5.423}]},{"osd":3,"last update":"Sat Jun 10 21:46:33 2023","interfaces":[{"interface":"back","average":{"1min":16.815000000000001,"5min":6.7800000000000002,"15min":5.7309999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27000000000000002},"max":{"1min":218.35300000000001,"5min":218.35300000000001,"15min":218.35300000000001},"last":4.1609999999999996},{"interface":"front","average":{"1min":17.359999999999999,"5min":6.7149999999999999,"15min":5.569},"min":{"1min":0.38800000000000001,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":218.74600000000001,"5min":218.74600000000001,"15min":218.74600000000001},"last":2.5800000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.1950000000000001,"15min":0.70199999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":3.6000000000000001,"5min":13.429,"15min":13.429},"last":0.311},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.3129999999999999,"15min":0.85499999999999998},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":3.9580000000000002,"5min":12.279999999999999,"15min":22.77},"last":0.51900000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:46:07 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":8.0039999999999996,"15min":10.786},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.3230000000000004,"5min":131.619,"15min":131.619},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.6870000000000001,"5min":7.944,"15min":10.647},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.603999999999999,"5min":134.928,"15min":134.928},"last":0.80600000000000005}]},{"osd":6,"last update":"Sat Jun 10 21:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0840000000000001,"15min":0.75900000000000001},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.9780000000000002,"5min":7.9660000000000002,"15min":10.890000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.878,"5min":1.292,"15min":0.79000000000000004},"min":{"1min":0.38400000000000001,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":3.5499999999999998,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":2.6400000000000001}]}]},{"osd":6,"up_from":0,"seq":180388627026,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566469,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020445,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075857,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4471276,"kb_used_data":3382500,"kb_used_omap":7,"kb_used_meta":1088760,"kb_avail":89273876,"statfs":{"total":95995035648,"available":91416449024,"internally_reserved":0,"allocated":3463680000,"data_stored":3462291538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1114890321},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.4850000000000001,"15min":4.0039999999999996},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":5.8849999999999998,"5min":8.5899999999999999,"15min":171.33699999999999},"last":0.309},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.2170000000000001,"15min":3.7050000000000001},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":6.415,"5min":6.415,"15min":171.56200000000001},"last":0.35599999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.4089999999999998,"15min":5.0590000000000002},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":11.398999999999999,"5min":85.709999999999994,"15min":171.286},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.478,"5min":2.71,"15min":3.7530000000000001},"min":{"1min":0.311,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.9119999999999999,"5min":85.501999999999995,"15min":147.81299999999999},"last":0.375}]},{"osd":3,"last update":"Sat Jun 10 21:46:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3949999999999996,"5min":8.4849999999999994,"15min":6.7590000000000003},"min":{"1min":0.29099999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":62.441000000000003,"5min":145.07599999999999,"15min":171.78299999999999},"last":1.1759999999999999},{"interface":"front","average":{"1min":4.2889999999999997,"5min":8.0690000000000008,"15min":6.5750000000000002},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.24399999999999999},"max":{"1min":62.517000000000003,"5min":144.99000000000001,"15min":171.65100000000001},"last":1.1950000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.3140000000000001,"15min":2.1859999999999999},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.28100000000000003},"max":{"1min":6.3259999999999996,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.438},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.423,"15min":2.9969999999999999},"min":{"1min":0.41699999999999998,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.8150000000000004,"5min":5.8150000000000004,"15min":171.18100000000001},"last":2.5529999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:46:04 2023","interfaces":[{"interface":"back","average":{"1min":7.7290000000000001,"5min":6.1479999999999997,"15min":6.7039999999999997},"min":{"1min":0.46000000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":117.562,"5min":117.562,"15min":117.562},"last":2.7679999999999998},{"interface":"front","average":{"1min":7.3940000000000001,"5min":5.6790000000000003,"15min":6.1210000000000004},"min":{"1min":0.42899999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":117.535,"5min":117.535,"15min":117.535},"last":2.645}]},{"osd":6,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":1.603,"15min":2.3290000000000002},"min":{"1min":0.44400000000000001,"5min":0.41999999999999998,"15min":0.307},"max":{"1min":6.3879999999999999,"5min":9.0139999999999993,"15min":171.441},"last":5.6340000000000003},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.55,"15min":3.0819999999999999},"min":{"1min":0.46500000000000002,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":3.6320000000000001,"5min":4.9199999999999999,"15min":171.24600000000001},"last":5.6470000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:45:54 2023","interfaces":[{"interface":"back","average":{"1min":8.1129999999999995,"5min":3.2330000000000001,"15min":4.4729999999999999},"min":{"1min":0.46600000000000003,"5min":0.46500000000000002,"15min":0.34200000000000003},"max":{"1min":124.071,"5min":124.071,"15min":219.131},"last":2.4460000000000002},{"interface":"front","average":{"1min":8.0570000000000004,"5min":3.008,"15min":4.3579999999999997},"min":{"1min":0.504,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":125.203,"5min":125.203,"15min":219.19},"last":2.4609999999999999}]}]},{"osd":3,"up_from":250,"seq":1073741824217,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4832380,"kb_used_data":4032948,"kb_used_omap":7,"kb_used_meta":799416,"kb_avail":88912772,"statfs":{"total":95995035648,"available":91046678528,"internally_reserved":0,"allocated":4129738752,"data_stored":4128214944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":818602003},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.76800000000000002,"15min":2.843},"min":{"1min":0.216,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":6.46,"5min":6.46,"15min":105.09},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.042,"5min":1.573,"15min":3.097},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.20799999999999999},"max":{"1min":6.5519999999999996,"5min":6.6429999999999998,"15min":99.852999999999994},"last":4.4109999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.252,"15min":3},"min":{"1min":0.247,"5min":0.247,"15min":0.20499999999999999},"max":{"1min":2.6259999999999999,"5min":6.6459999999999999,"15min":94.766000000000005},"last":4.7489999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.88500000000000001,"15min":3.048},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.651,"5min":5.4109999999999996,"15min":94.691000000000003},"last":0.64000000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":3.4630000000000001,"15min":4.04},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.23300000000000001},"max":{"1min":4.7720000000000002,"5min":126.913,"15min":231.07900000000001},"last":4.5739999999999998},{"interface":"front","average":{"1min":0.92200000000000004,"5min":3.6459999999999999,"15min":5.0039999999999996},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.22800000000000001},"max":{"1min":4.157,"5min":128.52199999999999,"15min":231.58600000000001},"last":4.6820000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.591,"15min":1.472},"min":{"1min":0.34100000000000003,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":2.754,"5min":9.5150000000000006,"15min":13.417},"last":3.702},{"interface":"front","average":{"1min":1.351,"5min":1.613,"15min":2.0739999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":6.8479999999999999,"5min":8.8379999999999992,"15min":19.244},"last":2.6110000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.0109999999999999,"15min":1.0629999999999999},"min":{"1min":0.39300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.7370000000000001,"5min":6.532,"15min":6.532},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.855,"5min":2.4470000000000001,"15min":2.6179999999999999},"min":{"1min":0.41799999999999998,"5min":0.378,"15min":0.378},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":3.7530000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.7629999999999999,"15min":2.137},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8620000000000001,"5min":9.0250000000000004,"15min":19.164999999999999},"last":2.6549999999999998},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.214,"15min":1.5349999999999999},"min":{"1min":0.434,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1019999999999999,"5min":6.0750000000000002,"15min":17.327000000000002},"last":1.0529999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":4.6829999999999998,"5min":2.6779999999999999,"15min":2.6960000000000002},"min":{"1min":0.39800000000000002,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":79.984999999999999},"last":2.7090000000000001},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1099999999999999,"15min":2.0939999999999999},"min":{"1min":0.48199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":56.731999999999999,"5min":56.731999999999999,"15min":80.272999999999996},"last":1.087}]}]},{"osd":4,"up_from":0,"seq":128849019482,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245488,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5312384,"kb_used_data":4968376,"kb_used_omap":11,"kb_used_meta":343988,"kb_avail":88432768,"statfs":{"total":95995035648,"available":90555154432,"internally_reserved":0,"allocated":5087617024,"data_stored":5086906946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":352244214},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.8100000000000001,"15min":1.99},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.573,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.069,"5min":1.3109999999999999,"15min":1.3180000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":5.0490000000000004},"last":0.88500000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":2.2829999999999999,"15min":2.5329999999999999},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.9610000000000003,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.069,"5min":1.9810000000000001,"15min":2.181},"min":{"1min":0.315,"5min":0.308,"15min":0.308},"max":{"1min":4.9980000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.69099999999999995}]},{"osd":2,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":3.294,"15min":2.339},"min":{"1min":0.31900000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3499999999999996,"5min":159.94999999999999,"15min":159.94999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.004,"5min":3.4489999999999998,"15min":2.6869999999999998},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.8279999999999998,"5min":159.851,"15min":159.851},"last":2.1840000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":6.109,"15min":7.5250000000000004},"min":{"1min":0.36499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.2549999999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.878},{"interface":"front","average":{"1min":1.4430000000000001,"5min":6.0919999999999996,"15min":7.2939999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.5049999999999999,"5min":133.03800000000001,"15min":133.03800000000001},"last":0.85599999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.97499999999999998,"15min":0.93100000000000005},"min":{"1min":0.33100000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.274,"5min":5.173,"15min":5.173},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.431,"5min":1.9339999999999999,"15min":2.1829999999999998},"min":{"1min":0.33400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.0700000000000003,"5min":7.476,"15min":7.476},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.5900000000000001,"15min":1.788},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.0810000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.302,"15min":1.3300000000000001},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.5660000000000001,"15min":1.5960000000000001},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6210000000000004,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":2.754},{"interface":"front","average":{"1min":1.974,"5min":2.722,"15min":2.8340000000000001},"min":{"1min":0.46200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.5370000000000008,"5min":9.5370000000000008,"15min":9.5370000000000008},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3475767296,"data_stored":3475767296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4166385664,"data_stored":4166385664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5115871232,"data_stored":5115871232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3733454848,"data_stored":3733454848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:47:00.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:47:00.859 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:00.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:00.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:00.971 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:01.004 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:01.072 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:01.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:01.180 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:01.210 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 67 16 89528 89512 83.4501 60.375 0.0107675 0.0119783 2023-06-10T21:47:01.264 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:01.402 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:01.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:01.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:01.546 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:01.617 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:01.617 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:47:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:01.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:01.670 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:47:01.700 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:01.710 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:47:01.726 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:01.742 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:47:01.743 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:47:01.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:47:01.743 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104502 12.3 12:c60ebc2b:::benchmark_data_smithi064_442287_object16452:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 520.52970307299995, 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2634145729999999, 2023-06-10T21:47:01.744 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104502 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.745 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198009+0000", 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198012+0000", 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7499999999999999e-06 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198017+0000", 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8730000000000004e-06 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.198022+0000", 2023-06-10T21:47:01.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.899e-06 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.200726+0000", 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0027034200000000002 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.200875+0000", 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014875899999999999 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.201009+0000", 2023-06-10T21:47:01.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013431900000000001 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.207565+0000", 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065557020000000001 2023-06-10T21:47:01.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.208794+0000", 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012298840000000001 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229308+0000", 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020513547 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229322+0000", 2023-06-10T21:47:01.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3995999999999999e-05 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203125+0000", 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97380280500000005 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203136+0000", 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1531e-05 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461328+0000", 2023-06-10T21:47:01.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2581918000000001 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461424+0000", 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.569e-05 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.757 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:104504 12.3 12:c95ca6d2:::benchmark_data_smithi064_442287_object16454:head [delete] snapc 0=[] ondisk+write+known_if_redirected e329)", 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 520.52544947700005, 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.25957418, 2023-06-10T21:47:01.758 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 104504 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202263+0000", 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202261+0000", 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:01.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202268+0000", 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1489999999999997e-06 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202282+0000", 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3679000000000001e-05 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.202290+0000", 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5410000000000003e-06 2023-06-10T21:47:01.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.203820+0000", 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001530954 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.203940+0000", 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119145 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.204054+0000", 2023-06-10T21:47:01.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114466 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.208801+0000", 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047466110000000004 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.210128+0000", 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013276329999999999 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229351+0000", 2023-06-10T21:47:01.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019222966000000001 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:21.229368+0000", 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6806000000000001e-05 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203250+0000", 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97388200400000002 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:22.203261+0000", 2023-06-10T21:47:01.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0981e-05 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461757+0000", 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.258495736 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:23.461837+0000", 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0174999999999997e-05 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:110588 12.c 12:32a597f0:::benchmark_data_smithi064_442287_object22538:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 500.619036634, 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32362693999999997, 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.772 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 110588 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108676+0000", 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108675+0000", 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108678+0000", 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0230000000000001e-06 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108687+0000", 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2900000000000002e-06 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108692+0000", 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0270000000000003e-06 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108740+0000", 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7923000000000002e-05 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.108896+0000", 2023-06-10T21:47:01.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015683099999999999 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,5,7", 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.109018+0000", 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122058 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.120560+0000", 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011541332 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.129815+0000", 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092551660000000004 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.129838+0000", 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3451999999999998e-05 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432143+0000", 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30230479700000001 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432195+0000", 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1430000000000001e-05 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:41.432303+0000", 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108051 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.783 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111271 12.e 12:71b77781:::benchmark_data_smithi064_442287_object23221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.55883214099998, 2023-06-10T21:47:01.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34391442799999999, 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111271 2023-06-10T21:47:01.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168880+0000", 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168883+0000", 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2780000000000002e-06 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168889+0000", 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8139999999999997e-06 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168894+0000", 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.626e-06 2023-06-10T21:47:01.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.169018+0000", 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124146 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.181820+0000", 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012801554999999999 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476308+0000", 2023-06-10T21:47:01.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.29448750000000001 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476432+0000", 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012422299999999999 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.476567+0000", 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013533600000000001 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.489518+0000", 2023-06-10T21:47:01.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012950721 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.489561+0000", 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2778999999999998e-05 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509183+0000", 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019622695999999999 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509231+0000", 2023-06-10T21:47:01.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7969000000000001e-05 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509276+0000", 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5182999999999998e-05 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512795+0000", 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035181790000000002 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111272 12.e 12:7d0d8fe8:::benchmark_data_smithi064_442287_object23222:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.55880617999998, 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34743270999999998, 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.798 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111272 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:47:01.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168906+0000", 2023-06-10T21:47:01.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168908+0000", 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.289e-06 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168913+0000", 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2440000000000001e-06 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.168918+0000", 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2799999999999997e-06 2023-06-10T21:47:01.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.181864+0000", 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012946077 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182630+0000", 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076583 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486615+0000", 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.30398561099999999 2023-06-10T21:47:01.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486722+0000", 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106419 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.486862+0000", 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140097 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506259+0000", 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.019397116999999998 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509129+0000", 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028700869999999999 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512883+0000", 2023-06-10T21:47:01.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037540400000000002 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512915+0000", 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2101999999999999e-05 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512960+0000", 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4907999999999998e-05 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516339+0000", 2023-06-10T21:47:01.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033789279999999998 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111276 12.e 12:70f74f42:::benchmark_data_smithi064_442287_object23226:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:47:01.811 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.55121021100001, 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34345777999999999, 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111276 2023-06-10T21:47:01.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176502+0000", 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176501+0000", 2023-06-10T21:47:01.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176508+0000", 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9410000000000001e-06 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176521+0000", 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3344e-05 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.176527+0000", 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.959e-06 2023-06-10T21:47:01.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182668+0000", 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061415300000000001 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:01.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.183316+0000", 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00064711 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496369+0000", 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.31305350599999998 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496497+0000", 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000128102 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.496626+0000", 2023-06-10T21:47:01.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012860200000000001 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.509144+0000", 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012518184999999999 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.512849+0000", 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003704948 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516399+0000", 2023-06-10T21:47:01.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035501859999999999 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516435+0000", 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6162999999999999e-05 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.516493+0000", 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7615000000000003e-05 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.519960+0000", 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034670529999999999 2023-06-10T21:47:01.823 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111279 12.e 12:79bea3d8:::benchmark_data_smithi064_442287_object23229:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:47:01.824 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.54526712900002, 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.34643435099999997, 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111279 2023-06-10T21:47:01.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182445+0000", 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182444+0000", 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:01.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182451+0000", 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3679999999999996e-06 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182464+0000", 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2527999999999999e-05 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.182472+0000", 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0909999999999993e-06 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.184408+0000", 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019360619999999999 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:01.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.185139+0000", 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073120499999999996 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506300+0000", 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32116106999999999 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506438+0000", 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013793599999999999 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.506589+0000", 2023-06-10T21:47:01.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000151132 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525267+0000", 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018678178 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525276+0000", 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9209999999999998e-06 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525311+0000", 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4999000000000002e-05 2023-06-10T21:47:01.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525348+0000", 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6294000000000002e-05 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.525399+0000", 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1554999999999997e-05 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.528880+0000", 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034804789999999999 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.836 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:111285 12.3 12:c9e541db:::benchmark_data_smithi064_442287_object23235:head [delete] snapc 0=[] ondisk+write+known_if_redirected e337)", 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 498.53372997700001, 2023-06-10T21:47:01.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.33020181300000001, 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 111285 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.838 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193982+0000", 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193980+0000", 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.193992+0000", 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1773000000000001e-05 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.470758+0000", 2023-06-10T21:47:01.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.27676603100000002 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.470772+0000", 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3966e-05 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471014+0000", 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000242725 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471167+0000", 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000152967 2023-06-10T21:47:01.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.471277+0000", 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109799 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.474344+0000", 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003067354 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.481669+0000", 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0073244490000000002 2023-06-10T21:47:01.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.481676+0000", 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7789999999999998e-06 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524025+0000", 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042349554999999997 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524076+0000", 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0383000000000002e-05 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:38:43.524184+0000", 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108587 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117664 12.0 e342/336 12:0001adf0:::benchmark_data_smithi064_442287_object29614:head v 342'7257, mlcod=316'1811)", 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.58051312700002, 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.45376187699999998, 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:47:01.849 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147199+0000", 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147203+0000", 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1980000000000003e-06 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147209+0000", 2023-06-10T21:47:01.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5380000000000002e-06 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.147213+0000", 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7040000000000002e-06 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.151470+0000", 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042562950000000002 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.151505+0000", 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4841000000000001e-05 2023-06-10T21:47:01.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.600961+0000", 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.44945654200000001 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.855 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117672 12.0 e342/336 12:08d95f39:::benchmark_data_smithi064_442287_object29622:head v 342'7258, mlcod=316'1811)", 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.54930500500001, 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.42256061299999997, 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.856 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178407+0000", 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178412+0000", 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.118e-06 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.859 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178421+0000", 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.755e-06 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.178427+0000", 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1700000000000002e-06 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.182568+0000", 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041405349999999999 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.182601+0000", 2023-06-10T21:47:01.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3775000000000001e-05 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.600968+0000", 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.418366511 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.862 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117681 12.13 12:cf7cb710:::benchmark_data_smithi064_442287_object29631:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.51638959100001, 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.889619234, 2023-06-10T21:47:01.863 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117681 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211323+0000", 2023-06-10T21:47:01.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211320+0000", 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211331+0000", 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0591000000000001e-05 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211352+0000", 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0720000000000002e-05 2023-06-10T21:47:01.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211370+0000", 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8448e-05 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424732+0000", 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.213361354 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424740+0000", 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6810000000000004e-06 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.078401+0000", 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.653661219 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.078414+0000", 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2357999999999999e-05 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.100864+0000", 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022449875 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.100942+0000", 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8227999999999997e-05 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117686 12.13 12:cc0e1ca3:::benchmark_data_smithi064_442287_object29636:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e343)", 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.51633224099999, 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9062216680000001, 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.873 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117686 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:47:01.874 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211380+0000", 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211383+0000", 2023-06-10T21:47:01.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6179999999999999e-06 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211391+0000", 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5780000000000001e-06 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.211397+0000", 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0630000000000002e-06 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424757+0000", 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.21336037299999999 2023-06-10T21:47:01.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.424762+0000", 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1919999999999996e-06 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.083725+0000", 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.658962869 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:06.083737+0000", 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1284000000000001e-05 2023-06-10T21:47:01.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101248+0000", 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.017511311 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101387+0000", 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013950599999999999 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 3,5,7", 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.101678+0000", 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029038499999999999 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.103061+0000", 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001383553 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.113231+0000", 2023-06-10T21:47:01.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010170102 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.114525+0000", 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0012936619999999999 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117436+0000", 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029114499999999999 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117492+0000", 2023-06-10T21:47:01.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5167000000000002e-05 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:07.117602+0000", 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000110105 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.886 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:117687 12.a e343/331 12:50ee7d02:::benchmark_data_smithi064_442287_object29637:head v 343'7458, mlcod=316'1040)", 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.494613754, 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32658561000000003, 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.887 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.888 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233099+0000", 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233097+0000", 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:01.889 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233111+0000", 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4912000000000001e-05 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.890 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233134+0000", 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2920000000000001e-05 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.233149+0000", 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4758e-05 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.235997+0000", 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028478689999999998 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.236030+0000", 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3021000000000001e-05 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.559594+0000", 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.32356415199999999 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.559684+0000", 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9957999999999995e-05 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:117694 12.16 12:6ae122fd:::benchmark_data_smithi064_442287_object29644:head [delete] snapc 0=[] ondisk+write+known_if_redirected e343)", 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 477.05461611200002, 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.56191629600000004, 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:01.895 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 117694 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:47:01.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673096+0000", 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673094+0000", 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673106+0000", 2023-06-10T21:47:01.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2442e-05 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673128+0000", 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2512000000000001e-05 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673142+0000", 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3696000000000001e-05 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673202+0000", 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9373999999999998e-05 2023-06-10T21:47:01.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:04.673218+0000", 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6918999999999998e-05 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:01.901 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.217815+0000", 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.54459649300000001 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.217922+0000", 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010657100000000001 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,7", 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.218094+0000", 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017210200000000001 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.220622+0000", 2023-06-10T21:47:01.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025278940000000001 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.220645+0000", 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3152000000000001e-05 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.234923+0000", 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.014277817999999999 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.234971+0000", 2023-06-10T21:47:01.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8021e-05 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:05.235012+0000", 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1964e-05 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.906 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.907 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:01.907 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:01.907 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:47:01.907 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:47:01.908 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:01.908 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:47:01.908 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:01.909 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:47:01.909 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:47:01.918 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.935 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:47:01.936 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:47:02.209 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 68 16 90483 90467 83.1008 59.6875 0.014582 0.0120288 2023-06-10T21:47:02.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3399584091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:02.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3399584091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:02.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3399584091' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:02.948 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:02.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:03.053 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:03.113 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:03.145 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:02] "GET /metrics HTTP/1.1" 200 224094 "" "Prometheus/2.33.4" 2023-06-10T21:47:03.208 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 69 16 91341 91325 82.6737 53.625 0.00383889 0.0120854 2023-06-10T21:47:03.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:03 smithi064 ceph-mon[186706]: pgmap v1606: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:03.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:03.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:03 smithi064 ceph-mon[180297]: pgmap v1606: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:03.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:03 smithi152 ceph-mon[156819]: pgmap v1606: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:03.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:03.931 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:03.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:04.032 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:04.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:04.208 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 70 16 92464 92448 82.4953 70.1875 0.0144879 0.0121172 2023-06-10T21:47:04.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:47:04.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:04 smithi064 ceph-mon[180297]: pgmap v1607: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:04.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:47:04.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:04 smithi064 ceph-mon[186706]: pgmap v1607: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:04.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:47:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:47:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:04 smithi152 ceph-mon[156819]: pgmap v1607: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2023-06-10T21:47:04.852 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:04.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:04.958 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:05.036 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:05.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 71 16 93370 93354 82.1284 56.625 0.00824764 0.0121711 2023-06-10T21:47:05.341 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:05.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:05.446 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:05.475 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:05.569 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:05.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:05.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:05.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:05.729 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.729 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.734 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:05.891 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:05.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:05.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:05.938 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:47:06.003 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:06.052 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:06.153 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:06.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:06.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:06.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 72 16 94191 94175 81.7006 51.3125 0.0109513 0.0122355 2023-06-10T21:47:06.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:06.304 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:06.375 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:06.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:06.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:06.540 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:06.604 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:06.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:06.708 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:06.760 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:06.858 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:06.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:06.866 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:06.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:07.011 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:07.089 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:07.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:07.189 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:07.202 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:07.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 73 16 95041 95025 81.3093 53.125 0.0101296 0.0122943 2023-06-10T21:47:07.317 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:07.376 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:47:07.385 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:47:07.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:07.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:07.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:07.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:07 smithi064 ceph-mon[180297]: pgmap v1608: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 975 op/s 2023-06-10T21:47:07.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:07.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:07 smithi064 ceph-mon[186706]: pgmap v1608: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 975 op/s 2023-06-10T21:47:07.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:07.661 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:47:07.669 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:07 smithi152 ceph-mon[156819]: pgmap v1608: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 975 op/s 2023-06-10T21:47:07.669 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:07.675 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:07.803 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:07.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:07.919 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:07.981 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:08.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 74 16 96076 96060 81.0847 64.6875 0.014127 0.0123285 2023-06-10T21:47:08.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:08.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:47:08.288 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:08.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: osdmap e437: 8 total, 8 up, 4 in 2023-06-10T21:47:08.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:08.289 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:08.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:47:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: osdmap e437: 8 total, 8 up, 4 in 2023-06-10T21:47:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:08.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:08 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:08.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:47:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3802358571' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: osdmap e437: 8 total, 8 up, 4 in 2023-06-10T21:47:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:08.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:08 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 75 16 97177 97161 80.921 68.8125 0.0364199 0.0123524 2023-06-10T21:47:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:47:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: pgmap v1610: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:47:09.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: pgmap v1610: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.483 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:09.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:09.590 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:09.611 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1608,"stamp":"2023-06-10T21:47:06.254427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6014965646,"num_objects":91796,"num_object_clones":0,"num_object_copies":275388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91796,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":92033,"num_write_kb":5875854,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":2377742,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75567,"ondisk_log_size":75567,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":20987736,"kb_used_data":17654648,"kb_used_omap":27,"kb_used_meta":3333028,"kb_avail":353992872,"statfs":{"total":383980142592,"available":362488700928,"internally_reserved":0,"allocated":18078359552,"data_stored":18074009811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":28070,"internal_metadata":3413021274},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":639369216,"num_objects":9756,"num_object_clones":0,"num_object_copies":29268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9756,"num_write_kb":624384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003350"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":476,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.214972+0000","last_change":"2023-06-10T21:43:01.240255+0000","last_active":"2023-06-10T21:46:50.214972+0000","last_peered":"2023-06-10T21:46:50.214972+0000","last_clean":"2023-06-10T21:46:50.214972+0000","last_became_active":"2023-06-10T21:43:01.239915+0000","last_became_peered":"2023-06-10T21:43:01.239915+0000","last_unstale":"2023-06-10T21:46:50.214972+0000","last_undegraded":"2023-06-10T21:46:50.214972+0000","last_fullsized":"2023-06-10T21:46:50.214972+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:43:08.293267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225271+0000","last_change":"2023-06-10T21:43:01.203830+0000","last_active":"2023-06-10T21:46:50.225271+0000","last_peered":"2023-06-10T21:46:50.225271+0000","last_clean":"2023-06-10T21:46:50.225271+0000","last_became_active":"2023-06-10T21:43:01.203468+0000","last_became_peered":"2023-06-10T21:43:01.203468+0000","last_unstale":"2023-06-10T21:46:50.225271+0000","last_undegraded":"2023-06-10T21:46:50.225271+0000","last_fullsized":"2023-06-10T21:46:50.225271+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:01.613126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":368,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221821+0000","last_change":"2023-06-10T21:39:50.428028+0000","last_active":"2023-06-10T21:46:50.221821+0000","last_peered":"2023-06-10T21:46:50.221821+0000","last_clean":"2023-06-10T21:46:50.221821+0000","last_became_active":"2023-06-10T21:39:50.427681+0000","last_became_peered":"2023-06-10T21:39:50.427681+0000","last_unstale":"2023-06-10T21:46:50.221821+0000","last_undegraded":"2023-06-10T21:46:50.221821+0000","last_fullsized":"2023-06-10T21:46:50.221821+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:39:45.637930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229728+0000","last_change":"2023-06-10T21:43:01.267377+0000","last_active":"2023-06-10T21:46:50.229728+0000","last_peered":"2023-06-10T21:46:50.229728+0000","last_clean":"2023-06-10T21:46:50.229728+0000","last_became_active":"2023-06-10T21:43:01.266072+0000","last_became_peered":"2023-06-10T21:43:01.266072+0000","last_unstale":"2023-06-10T21:46:50.229728+0000","last_undegraded":"2023-06-10T21:46:50.229728+0000","last_fullsized":"2023-06-10T21:46:50.229728+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225111+0000","last_change":"2023-06-10T21:40:30.279344+0000","last_active":"2023-06-10T21:46:50.225111+0000","last_peered":"2023-06-10T21:46:50.225111+0000","last_clean":"2023-06-10T21:46:50.225111+0000","last_became_active":"2023-06-10T21:40:30.278694+0000","last_became_peered":"2023-06-10T21:40:30.278694+0000","last_unstale":"2023-06-10T21:46:50.225111+0000","last_undegraded":"2023-06-10T21:46:50.225111+0000","last_fullsized":"2023-06-10T21:46:50.225111+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:37:57.641767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":85,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.223741+0000","last_change":"2023-06-10T21:46:14.280677+0000","last_active":"2023-06-10T21:46:50.223741+0000","last_peered":"2023-06-10T21:46:50.223741+0000","last_clean":"2023-06-10T21:46:50.223741+0000","last_became_active":"2023-06-10T21:46:14.280384+0000","last_became_peered":"2023-06-10T21:46:14.280384+0000","last_unstale":"2023-06-10T21:46:50.223741+0000","last_undegraded":"2023-06-10T21:46:50.223741+0000","last_fullsized":"2023-06-10T21:46:50.223741+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:20:53.330571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.223614+0000","last_change":"2023-06-10T21:46:14.280309+0000","last_active":"2023-06-10T21:46:50.223614+0000","last_peered":"2023-06-10T21:46:50.223614+0000","last_clean":"2023-06-10T21:46:50.223614+0000","last_became_active":"2023-06-10T21:46:14.280033+0000","last_became_peered":"2023-06-10T21:46:14.280033+0000","last_unstale":"2023-06-10T21:46:50.223614+0000","last_undegraded":"2023-06-10T21:46:50.223614+0000","last_fullsized":"2023-06-10T21:46:50.223614+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:44:39.412406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225336+0000","last_change":"2023-06-10T21:43:01.203778+0000","last_active":"2023-06-10T21:46:50.225336+0000","last_peered":"2023-06-10T21:46:50.225336+0000","last_clean":"2023-06-10T21:46:50.225336+0000","last_became_active":"2023-06-10T21:43:01.203330+0000","last_became_peered":"2023-06-10T21:43:01.203330+0000","last_unstale":"2023-06-10T21:46:50.225336+0000","last_undegraded":"2023-06-10T21:46:50.225336+0000","last_fullsized":"2023-06-10T21:46:50.225336+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:24:52.787379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":96,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229868+0000","last_change":"2023-06-10T21:46:14.280460+0000","last_active":"2023-06-10T21:46:50.229868+0000","last_peered":"2023-06-10T21:46:50.229868+0000","last_clean":"2023-06-10T21:46:50.229868+0000","last_became_active":"2023-06-10T21:46:14.280171+0000","last_became_peered":"2023-06-10T21:46:14.280171+0000","last_unstale":"2023-06-10T21:46:50.229868+0000","last_undegraded":"2023-06-10T21:46:50.229868+0000","last_fullsized":"2023-06-10T21:46:50.229868+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.147223+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:46:56.147223+0000","last_peered":"2023-06-10T21:46:56.147223+0000","last_clean":"2023-06-10T21:46:56.147223+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:46:56.147223+0000","last_undegraded":"2023-06-10T21:46:56.147223+0000","last_fullsized":"2023-06-10T21:46:56.147223+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":193,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229757+0000","last_change":"2023-06-10T21:39:30.672957+0000","last_active":"2023-06-10T21:46:50.229757+0000","last_peered":"2023-06-10T21:46:50.229757+0000","last_clean":"2023-06-10T21:46:50.229757+0000","last_became_active":"2023-06-10T21:39:30.231073+0000","last_became_peered":"2023-06-10T21:39:30.231073+0000","last_unstale":"2023-06-10T21:46:50.229757+0000","last_undegraded":"2023-06-10T21:46:50.229757+0000","last_fullsized":"2023-06-10T21:46:50.229757+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":436,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215052+0000","last_change":"2023-06-10T21:43:01.240554+0000","last_active":"2023-06-10T21:46:50.215052+0000","last_peered":"2023-06-10T21:46:50.215052+0000","last_clean":"2023-06-10T21:46:50.215052+0000","last_became_active":"2023-06-10T21:43:01.240248+0000","last_became_peered":"2023-06-10T21:43:01.240248+0000","last_unstale":"2023-06-10T21:46:50.215052+0000","last_undegraded":"2023-06-10T21:46:50.215052+0000","last_fullsized":"2023-06-10T21:46:50.215052+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:45:43.889738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":359,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229169+0000","last_change":"2023-06-10T21:43:01.235509+0000","last_active":"2023-06-10T21:46:50.229169+0000","last_peered":"2023-06-10T21:46:50.229169+0000","last_clean":"2023-06-10T21:46:50.229169+0000","last_became_active":"2023-06-10T21:43:01.235129+0000","last_became_peered":"2023-06-10T21:43:01.235129+0000","last_unstale":"2023-06-10T21:46:50.229169+0000","last_undegraded":"2023-06-10T21:46:50.229169+0000","last_fullsized":"2023-06-10T21:46:50.229169+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14148,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215156+0000","last_change":"2023-06-10T21:46:14.281135+0000","last_active":"2023-06-10T21:46:50.215156+0000","last_peered":"2023-06-10T21:46:50.215156+0000","last_clean":"2023-06-10T21:46:50.215156+0000","last_became_active":"2023-06-10T21:46:14.280752+0000","last_became_peered":"2023-06-10T21:46:14.280752+0000","last_unstale":"2023-06-10T21:46:50.215156+0000","last_undegraded":"2023-06-10T21:46:50.215156+0000","last_fullsized":"2023-06-10T21:46:50.215156+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:30:54.587316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":495,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229090+0000","last_change":"2023-06-10T21:43:01.230432+0000","last_active":"2023-06-10T21:46:50.229090+0000","last_peered":"2023-06-10T21:46:50.229090+0000","last_clean":"2023-06-10T21:46:50.229090+0000","last_became_active":"2023-06-10T21:43:01.230129+0000","last_became_peered":"2023-06-10T21:43:01.230129+0000","last_unstale":"2023-06-10T21:46:50.229090+0000","last_undegraded":"2023-06-10T21:46:50.229090+0000","last_fullsized":"2023-06-10T21:46:50.229090+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":384,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224814+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:46:50.224814+0000","last_peered":"2023-06-10T21:46:50.224814+0000","last_clean":"2023-06-10T21:46:50.224814+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:46:50.224814+0000","last_undegraded":"2023-06-10T21:46:50.224814+0000","last_fullsized":"2023-06-10T21:46:50.224814+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":306,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229401+0000","last_change":"2023-06-10T21:40:30.397569+0000","last_active":"2023-06-10T21:46:50.229401+0000","last_peered":"2023-06-10T21:46:50.229401+0000","last_clean":"2023-06-10T21:46:50.229401+0000","last_became_active":"2023-06-10T21:40:30.393710+0000","last_became_peered":"2023-06-10T21:40:30.393710+0000","last_unstale":"2023-06-10T21:46:50.229401+0000","last_undegraded":"2023-06-10T21:46:50.229401+0000","last_fullsized":"2023-06-10T21:46:50.229401+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":303,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.230127+0000","last_change":"2023-06-10T21:46:14.281414+0000","last_active":"2023-06-10T21:46:50.230127+0000","last_peered":"2023-06-10T21:46:50.230127+0000","last_clean":"2023-06-10T21:46:50.230127+0000","last_became_active":"2023-06-10T21:46:14.280518+0000","last_became_peered":"2023-06-10T21:46:14.280518+0000","last_unstale":"2023-06-10T21:46:50.230127+0000","last_undegraded":"2023-06-10T21:46:50.230127+0000","last_fullsized":"2023-06-10T21:46:50.230127+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":448,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229314+0000","last_change":"2023-06-10T21:40:30.285921+0000","last_active":"2023-06-10T21:46:50.229314+0000","last_peered":"2023-06-10T21:46:50.229314+0000","last_clean":"2023-06-10T21:46:50.229314+0000","last_became_active":"2023-06-10T21:40:30.285569+0000","last_became_peered":"2023-06-10T21:40:30.285569+0000","last_unstale":"2023-06-10T21:46:50.229314+0000","last_undegraded":"2023-06-10T21:46:50.229314+0000","last_fullsized":"2023-06-10T21:46:50.229314+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":90,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224280+0000","last_change":"2023-06-10T21:40:30.351628+0000","last_active":"2023-06-10T21:46:50.224280+0000","last_peered":"2023-06-10T21:46:50.224280+0000","last_clean":"2023-06-10T21:46:50.224280+0000","last_became_active":"2023-06-10T21:40:30.350062+0000","last_became_peered":"2023-06-10T21:40:30.350062+0000","last_unstale":"2023-06-10T21:46:50.224280+0000","last_undegraded":"2023-06-10T21:46:50.224280+0000","last_fullsized":"2023-06-10T21:46:50.224280+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:54:15.200663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":416,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225493+0000","last_change":"2023-06-10T21:46:14.279613+0000","last_active":"2023-06-10T21:46:50.225493+0000","last_peered":"2023-06-10T21:46:50.225493+0000","last_clean":"2023-06-10T21:46:50.225493+0000","last_became_active":"2023-06-10T21:46:14.279252+0000","last_became_peered":"2023-06-10T21:46:14.279252+0000","last_unstale":"2023-06-10T21:46:50.225493+0000","last_undegraded":"2023-06-10T21:46:50.225493+0000","last_fullsized":"2023-06-10T21:46:50.225493+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:58:42.469928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224110+0000","last_change":"2023-06-10T21:40:30.253321+0000","last_active":"2023-06-10T21:46:50.224110+0000","last_peered":"2023-06-10T21:46:50.224110+0000","last_clean":"2023-06-10T21:46:50.224110+0000","last_became_active":"2023-06-10T21:40:30.253012+0000","last_became_peered":"2023-06-10T21:40:30.253012+0000","last_unstale":"2023-06-10T21:46:50.224110+0000","last_undegraded":"2023-06-10T21:46:50.224110+0000","last_fullsized":"2023-06-10T21:46:50.224110+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:44:06.900735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"436'5895","reported_seq":5920,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:06.154340+0000","last_change":"2023-06-10T21:45:44.480492+0000","last_active":"2023-06-10T21:47:06.154340+0000","last_peered":"2023-06-10T21:47:06.154340+0000","last_clean":"2023-06-10T21:47:06.154340+0000","last_became_active":"2023-06-10T21:45:44.480237+0000","last_became_peered":"2023-06-10T21:45:44.480237+0000","last_unstale":"2023-06-10T21:47:06.154340+0000","last_undegraded":"2023-06-10T21:47:06.154340+0000","last_fullsized":"2023-06-10T21:47:06.154340+0000","mapping_epoch":420,"log_start":"424'1600","ondisk_log_start":"424'1600","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4295,"log_dups_size":0,"ondisk_log_size":4295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:15:36.524573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386334720,"num_objects":5895,"num_object_clones":0,"num_object_copies":17685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":228,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229767+0000","last_change":"2023-06-10T21:43:01.243009+0000","last_active":"2023-06-10T21:46:50.229767+0000","last_peered":"2023-06-10T21:46:50.229767+0000","last_clean":"2023-06-10T21:46:50.229767+0000","last_became_active":"2023-06-10T21:43:01.242599+0000","last_became_peered":"2023-06-10T21:43:01.242599+0000","last_unstale":"2023-06-10T21:46:50.229767+0000","last_undegraded":"2023-06-10T21:46:50.229767+0000","last_fullsized":"2023-06-10T21:46:50.229767+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":114,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229218+0000","last_change":"2023-06-10T21:46:14.279497+0000","last_active":"2023-06-10T21:46:50.229218+0000","last_peered":"2023-06-10T21:46:50.229218+0000","last_clean":"2023-06-10T21:46:50.229218+0000","last_became_active":"2023-06-10T21:46:14.279186+0000","last_became_peered":"2023-06-10T21:46:14.279186+0000","last_unstale":"2023-06-10T21:46:50.229218+0000","last_undegraded":"2023-06-10T21:46:50.229218+0000","last_fullsized":"2023-06-10T21:46:50.229218+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"436'5873","reported_seq":5898,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:06.146139+0000","last_change":"2023-06-10T21:45:44.477018+0000","last_active":"2023-06-10T21:47:06.146139+0000","last_peered":"2023-06-10T21:47:06.146139+0000","last_clean":"2023-06-10T21:47:06.146139+0000","last_became_active":"2023-06-10T21:45:44.476687+0000","last_became_peered":"2023-06-10T21:45:44.476687+0000","last_unstale":"2023-06-10T21:47:06.146139+0000","last_undegraded":"2023-06-10T21:47:06.146139+0000","last_fullsized":"2023-06-10T21:47:06.146139+0000","mapping_epoch":420,"log_start":"424'1600","ondisk_log_start":"424'1600","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4273,"log_dups_size":0,"ondisk_log_size":4273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:10:48.652326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384892928,"num_objects":5873,"num_object_clones":0,"num_object_copies":17619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":375872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":228,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229926+0000","last_change":"2023-06-10T21:43:01.267770+0000","last_active":"2023-06-10T21:46:50.229926+0000","last_peered":"2023-06-10T21:46:50.229926+0000","last_clean":"2023-06-10T21:46:50.229926+0000","last_became_active":"2023-06-10T21:43:01.267040+0000","last_became_peered":"2023-06-10T21:43:01.267040+0000","last_unstale":"2023-06-10T21:46:50.229926+0000","last_undegraded":"2023-06-10T21:46:50.229926+0000","last_fullsized":"2023-06-10T21:46:50.229926+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.309809+0000","last_change":"2023-06-10T21:40:30.316394+0000","last_active":"2023-06-10T21:46:56.309809+0000","last_peered":"2023-06-10T21:46:56.309809+0000","last_clean":"2023-06-10T21:46:56.309809+0000","last_became_active":"2023-06-10T21:40:30.315218+0000","last_became_peered":"2023-06-10T21:40:30.315218+0000","last_unstale":"2023-06-10T21:46:56.309809+0000","last_undegraded":"2023-06-10T21:46:56.309809+0000","last_fullsized":"2023-06-10T21:46:56.309809+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:33:07.609662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.0","version":"436'5620","reported_seq":5645,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.306562+0000","last_change":"2023-06-10T21:45:44.477367+0000","last_active":"2023-06-10T21:47:01.306562+0000","last_peered":"2023-06-10T21:47:01.306562+0000","last_clean":"2023-06-10T21:47:01.306562+0000","last_became_active":"2023-06-10T21:45:44.477107+0000","last_became_peered":"2023-06-10T21:45:44.477107+0000","last_unstale":"2023-06-10T21:47:01.306562+0000","last_undegraded":"2023-06-10T21:47:01.306562+0000","last_fullsized":"2023-06-10T21:47:01.306562+0000","mapping_epoch":420,"log_start":"424'400","ondisk_log_start":"424'400","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5220,"log_dups_size":0,"ondisk_log_size":5220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:35:06.980899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368312320,"num_objects":5620,"num_object_clones":0,"num_object_copies":16860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":359680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":515,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229844+0000","last_change":"2023-06-10T21:43:01.231170+0000","last_active":"2023-06-10T21:46:50.229844+0000","last_peered":"2023-06-10T21:46:50.229844+0000","last_clean":"2023-06-10T21:46:50.229844+0000","last_became_active":"2023-06-10T21:43:01.230787+0000","last_became_peered":"2023-06-10T21:43:01.230787+0000","last_unstale":"2023-06-10T21:46:50.229844+0000","last_undegraded":"2023-06-10T21:46:50.229844+0000","last_fullsized":"2023-06-10T21:46:50.229844+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":199,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229187+0000","last_change":"2023-06-10T21:43:01.234229+0000","last_active":"2023-06-10T21:46:50.229187+0000","last_peered":"2023-06-10T21:46:50.229187+0000","last_clean":"2023-06-10T21:46:50.229187+0000","last_became_active":"2023-06-10T21:43:01.233913+0000","last_became_peered":"2023-06-10T21:43:01.233913+0000","last_unstale":"2023-06-10T21:46:50.229187+0000","last_undegraded":"2023-06-10T21:46:50.229187+0000","last_fullsized":"2023-06-10T21:46:50.229187+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"436'6003","reported_seq":6028,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:06.153537+0000","last_change":"2023-06-10T21:45:44.476967+0000","last_active":"2023-06-10T21:47:06.153537+0000","last_peered":"2023-06-10T21:47:06.153537+0000","last_clean":"2023-06-10T21:47:06.153537+0000","last_became_active":"2023-06-10T21:45:44.476670+0000","last_became_peered":"2023-06-10T21:45:44.476670+0000","last_unstale":"2023-06-10T21:47:06.153537+0000","last_undegraded":"2023-06-10T21:47:06.153537+0000","last_fullsized":"2023-06-10T21:47:06.153537+0000","mapping_epoch":420,"log_start":"424'1700","ondisk_log_start":"424'1700","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4303,"log_dups_size":0,"ondisk_log_size":4303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:11:49.819095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393412608,"num_objects":6003,"num_object_clones":0,"num_object_copies":18009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":86,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224655+0000","last_change":"2023-06-10T21:46:14.277139+0000","last_active":"2023-06-10T21:46:50.224655+0000","last_peered":"2023-06-10T21:46:50.224655+0000","last_clean":"2023-06-10T21:46:50.224655+0000","last_became_active":"2023-06-10T21:46:14.276750+0000","last_became_peered":"2023-06-10T21:46:14.276750+0000","last_unstale":"2023-06-10T21:46:50.224655+0000","last_undegraded":"2023-06-10T21:46:50.224655+0000","last_fullsized":"2023-06-10T21:46:50.224655+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:51.450990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221834+0000","last_change":"2023-06-10T21:43:01.213003+0000","last_active":"2023-06-10T21:46:50.221834+0000","last_peered":"2023-06-10T21:46:50.221834+0000","last_clean":"2023-06-10T21:46:50.221834+0000","last_became_active":"2023-06-10T21:43:01.212675+0000","last_became_peered":"2023-06-10T21:43:01.212675+0000","last_unstale":"2023-06-10T21:46:50.221834+0000","last_undegraded":"2023-06-10T21:46:50.221834+0000","last_fullsized":"2023-06-10T21:46:50.221834+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:38:43.535453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.6","version":"436'5511","reported_seq":5536,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.298439+0000","last_change":"2023-06-10T21:45:44.478058+0000","last_active":"2023-06-10T21:47:01.298439+0000","last_peered":"2023-06-10T21:47:01.298439+0000","last_clean":"2023-06-10T21:47:01.298439+0000","last_became_active":"2023-06-10T21:45:44.477758+0000","last_became_peered":"2023-06-10T21:45:44.477758+0000","last_unstale":"2023-06-10T21:47:01.298439+0000","last_undegraded":"2023-06-10T21:47:01.298439+0000","last_fullsized":"2023-06-10T21:47:01.298439+0000","mapping_epoch":420,"log_start":"423'300","ondisk_log_start":"423'300","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5211,"log_dups_size":0,"ondisk_log_size":5211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:39:43.723763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361168896,"num_objects":5511,"num_object_clones":0,"num_object_copies":16533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5511,"num_write_kb":352704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221754+0000","last_change":"2023-06-10T21:40:30.490559+0000","last_active":"2023-06-10T21:46:50.221754+0000","last_peered":"2023-06-10T21:46:50.221754+0000","last_clean":"2023-06-10T21:46:50.221754+0000","last_became_active":"2023-06-10T21:40:30.271487+0000","last_became_peered":"2023-06-10T21:40:30.271487+0000","last_unstale":"2023-06-10T21:46:50.221754+0000","last_undegraded":"2023-06-10T21:46:50.221754+0000","last_fullsized":"2023-06-10T21:46:50.221754+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:10:23.675521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":90,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224738+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:46:50.224738+0000","last_peered":"2023-06-10T21:46:50.224738+0000","last_clean":"2023-06-10T21:46:50.224738+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:46:50.224738+0000","last_undegraded":"2023-06-10T21:46:50.224738+0000","last_fullsized":"2023-06-10T21:46:50.224738+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"436'5711","reported_seq":5743,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:04.240187+0000","last_change":"2023-06-10T21:46:14.286123+0000","last_active":"2023-06-10T21:47:04.240187+0000","last_peered":"2023-06-10T21:47:04.240187+0000","last_clean":"2023-06-10T21:47:04.240187+0000","last_became_active":"2023-06-10T21:46:14.283059+0000","last_became_peered":"2023-06-10T21:46:14.283059+0000","last_unstale":"2023-06-10T21:47:04.240187+0000","last_undegraded":"2023-06-10T21:47:04.240187+0000","last_fullsized":"2023-06-10T21:47:04.240187+0000","mapping_epoch":426,"log_start":"424'1400","ondisk_log_start":"424'1400","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4311,"log_dups_size":0,"ondisk_log_size":4311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374276096,"num_objects":5711,"num_object_clones":0,"num_object_copies":17133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5711,"num_write_kb":365504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":258,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225783+0000","last_change":"2023-06-10T21:43:01.221763+0000","last_active":"2023-06-10T21:46:50.225783+0000","last_peered":"2023-06-10T21:46:50.225783+0000","last_clean":"2023-06-10T21:46:50.225783+0000","last_became_active":"2023-06-10T21:43:01.221264+0000","last_became_peered":"2023-06-10T21:43:01.221264+0000","last_unstale":"2023-06-10T21:46:50.225783+0000","last_undegraded":"2023-06-10T21:46:50.225783+0000","last_fullsized":"2023-06-10T21:46:50.225783+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:40:19.464587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":475,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215570+0000","last_change":"2023-06-10T21:43:01.210944+0000","last_active":"2023-06-10T21:46:50.215570+0000","last_peered":"2023-06-10T21:46:50.215570+0000","last_clean":"2023-06-10T21:46:50.215570+0000","last_became_active":"2023-06-10T21:43:01.210621+0000","last_became_peered":"2023-06-10T21:43:01.210621+0000","last_unstale":"2023-06-10T21:46:50.215570+0000","last_undegraded":"2023-06-10T21:46:50.215570+0000","last_fullsized":"2023-06-10T21:46:50.215570+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:48:11.515983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"436'5670","reported_seq":5694,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:04.228651+0000","last_change":"2023-06-10T21:45:44.480834+0000","last_active":"2023-06-10T21:47:04.228651+0000","last_peered":"2023-06-10T21:47:04.228651+0000","last_clean":"2023-06-10T21:47:04.228651+0000","last_became_active":"2023-06-10T21:45:44.480535+0000","last_became_peered":"2023-06-10T21:45:44.480535+0000","last_unstale":"2023-06-10T21:47:04.228651+0000","last_undegraded":"2023-06-10T21:47:04.228651+0000","last_fullsized":"2023-06-10T21:47:04.228651+0000","mapping_epoch":420,"log_start":"424'1300","ondisk_log_start":"424'1300","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4370,"log_dups_size":0,"ondisk_log_size":4370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":362880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":452,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.216861+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:46:50.216861+0000","last_peered":"2023-06-10T21:46:50.216861+0000","last_clean":"2023-06-10T21:46:50.216861+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:46:50.216861+0000","last_undegraded":"2023-06-10T21:46:50.216861+0000","last_fullsized":"2023-06-10T21:46:50.216861+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224188+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:46:50.224188+0000","last_peered":"2023-06-10T21:46:50.224188+0000","last_clean":"2023-06-10T21:46:50.224188+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:46:50.224188+0000","last_undegraded":"2023-06-10T21:46:50.224188+0000","last_fullsized":"2023-06-10T21:46:50.224188+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"436'5846","reported_seq":5870,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:04.208097+0000","last_change":"2023-06-10T21:45:44.482366+0000","last_active":"2023-06-10T21:47:04.208097+0000","last_peered":"2023-06-10T21:47:04.208097+0000","last_clean":"2023-06-10T21:47:04.208097+0000","last_became_active":"2023-06-10T21:45:44.482065+0000","last_became_peered":"2023-06-10T21:45:44.482065+0000","last_unstale":"2023-06-10T21:47:04.208097+0000","last_undegraded":"2023-06-10T21:47:04.208097+0000","last_fullsized":"2023-06-10T21:47:04.208097+0000","mapping_epoch":420,"log_start":"424'1500","ondisk_log_start":"424'1500","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4346,"log_dups_size":0,"ondisk_log_size":4346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383123456,"num_objects":5846,"num_object_clones":0,"num_object_copies":17538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5846,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":106,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.309781+0000","last_change":"2023-06-10T21:43:01.211810+0000","last_active":"2023-06-10T21:46:56.309781+0000","last_peered":"2023-06-10T21:46:56.309781+0000","last_clean":"2023-06-10T21:46:56.309781+0000","last_became_active":"2023-06-10T21:43:01.211431+0000","last_became_peered":"2023-06-10T21:43:01.211431+0000","last_unstale":"2023-06-10T21:46:56.309781+0000","last_undegraded":"2023-06-10T21:46:56.309781+0000","last_fullsized":"2023-06-10T21:46:56.309781+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:03:54.150938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":385,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:46:56.315103+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:46:56.315103+0000","last_peered":"2023-06-10T21:46:56.315103+0000","last_clean":"2023-06-10T21:46:56.315103+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:46:56.315103+0000","last_undegraded":"2023-06-10T21:46:56.315103+0000","last_fullsized":"2023-06-10T21:46:56.315103+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"436'5585","reported_seq":5610,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.248437+0000","last_change":"2023-06-10T21:45:44.477475+0000","last_active":"2023-06-10T21:47:01.248437+0000","last_peered":"2023-06-10T21:47:01.248437+0000","last_clean":"2023-06-10T21:47:01.248437+0000","last_became_active":"2023-06-10T21:45:44.477218+0000","last_became_peered":"2023-06-10T21:45:44.477218+0000","last_unstale":"2023-06-10T21:47:01.248437+0000","last_undegraded":"2023-06-10T21:47:01.248437+0000","last_fullsized":"2023-06-10T21:47:01.248437+0000","mapping_epoch":420,"log_start":"424'400","ondisk_log_start":"424'400","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5185,"log_dups_size":0,"ondisk_log_size":5185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:15:26.255634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366018560,"num_objects":5585,"num_object_clones":0,"num_object_copies":16755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5585,"num_write_kb":357440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":219,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229322+0000","last_change":"2023-06-10T21:40:30.286887+0000","last_active":"2023-06-10T21:46:50.229322+0000","last_peered":"2023-06-10T21:46:50.229322+0000","last_clean":"2023-06-10T21:46:50.229322+0000","last_became_active":"2023-06-10T21:40:30.286542+0000","last_became_peered":"2023-06-10T21:40:30.286542+0000","last_unstale":"2023-06-10T21:46:50.229322+0000","last_undegraded":"2023-06-10T21:46:50.229322+0000","last_fullsized":"2023-06-10T21:46:50.229322+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":67,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224895+0000","last_change":"2023-06-10T21:46:14.277043+0000","last_active":"2023-06-10T21:46:50.224895+0000","last_peered":"2023-06-10T21:46:50.224895+0000","last_clean":"2023-06-10T21:46:50.224895+0000","last_became_active":"2023-06-10T21:46:14.276737+0000","last_became_peered":"2023-06-10T21:46:14.276737+0000","last_unstale":"2023-06-10T21:46:50.224895+0000","last_undegraded":"2023-06-10T21:46:50.224895+0000","last_fullsized":"2023-06-10T21:46:50.224895+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:03.540332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":203,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215124+0000","last_change":"2023-06-10T21:46:14.279638+0000","last_active":"2023-06-10T21:46:50.215124+0000","last_peered":"2023-06-10T21:46:50.215124+0000","last_clean":"2023-06-10T21:46:50.215124+0000","last_became_active":"2023-06-10T21:46:14.279320+0000","last_became_peered":"2023-06-10T21:46:14.279320+0000","last_unstale":"2023-06-10T21:46:50.215124+0000","last_undegraded":"2023-06-10T21:46:50.215124+0000","last_fullsized":"2023-06-10T21:46:50.215124+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:05:18.656099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.e","version":"436'5749","reported_seq":5774,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.284767+0000","last_change":"2023-06-10T21:45:44.480812+0000","last_active":"2023-06-10T21:47:01.284767+0000","last_peered":"2023-06-10T21:47:01.284767+0000","last_clean":"2023-06-10T21:47:01.284767+0000","last_became_active":"2023-06-10T21:45:44.479574+0000","last_became_peered":"2023-06-10T21:45:44.479574+0000","last_unstale":"2023-06-10T21:47:01.284767+0000","last_undegraded":"2023-06-10T21:47:01.284767+0000","last_fullsized":"2023-06-10T21:47:01.284767+0000","mapping_epoch":420,"log_start":"424'500","ondisk_log_start":"424'500","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5249,"log_dups_size":0,"ondisk_log_size":5249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:01:32.420364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376766464,"num_objects":5749,"num_object_clones":0,"num_object_copies":17247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":86,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225558+0000","last_change":"2023-06-10T21:46:14.279110+0000","last_active":"2023-06-10T21:46:50.225558+0000","last_peered":"2023-06-10T21:46:50.225558+0000","last_clean":"2023-06-10T21:46:50.225558+0000","last_became_active":"2023-06-10T21:46:14.278624+0000","last_became_peered":"2023-06-10T21:46:14.278624+0000","last_unstale":"2023-06-10T21:46:50.225558+0000","last_undegraded":"2023-06-10T21:46:50.225558+0000","last_fullsized":"2023-06-10T21:46:50.225558+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:59:27.858698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":364,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221655+0000","last_change":"2023-06-10T21:46:14.278053+0000","last_active":"2023-06-10T21:46:50.221655+0000","last_peered":"2023-06-10T21:46:50.221655+0000","last_clean":"2023-06-10T21:46:50.221655+0000","last_became_active":"2023-06-10T21:46:14.277736+0000","last_became_peered":"2023-06-10T21:46:14.277736+0000","last_unstale":"2023-06-10T21:46:50.221655+0000","last_undegraded":"2023-06-10T21:46:50.221655+0000","last_fullsized":"2023-06-10T21:46:50.221655+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:48:27.204561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":302,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217055+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:46:50.217055+0000","last_peered":"2023-06-10T21:46:50.217055+0000","last_clean":"2023-06-10T21:46:50.217055+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:46:50.217055+0000","last_undegraded":"2023-06-10T21:46:50.217055+0000","last_fullsized":"2023-06-10T21:46:50.217055+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"436'5680","reported_seq":5712,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.311038+0000","last_change":"2023-06-10T21:46:14.285886+0000","last_active":"2023-06-10T21:47:01.311038+0000","last_peered":"2023-06-10T21:47:01.311038+0000","last_clean":"2023-06-10T21:47:01.311038+0000","last_became_active":"2023-06-10T21:46:14.280572+0000","last_became_peered":"2023-06-10T21:46:14.280572+0000","last_unstale":"2023-06-10T21:47:01.311038+0000","last_undegraded":"2023-06-10T21:47:01.311038+0000","last_fullsized":"2023-06-10T21:47:01.311038+0000","mapping_epoch":426,"log_start":"424'500","ondisk_log_start":"424'500","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5180,"log_dups_size":0,"ondisk_log_size":5180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:21:14.063231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372244480,"num_objects":5680,"num_object_clones":0,"num_object_copies":17040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5680,"num_write_kb":363520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":322,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.216785+0000","last_change":"2023-06-10T21:40:30.252694+0000","last_active":"2023-06-10T21:46:50.216785+0000","last_peered":"2023-06-10T21:46:50.216785+0000","last_clean":"2023-06-10T21:46:50.216785+0000","last_became_active":"2023-06-10T21:40:30.252365+0000","last_became_peered":"2023-06-10T21:40:30.252365+0000","last_unstale":"2023-06-10T21:46:50.216785+0000","last_undegraded":"2023-06-10T21:46:50.216785+0000","last_fullsized":"2023-06-10T21:46:50.216785+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:04:43.291684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":326,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.218385+0000","last_change":"2023-06-10T21:43:01.202873+0000","last_active":"2023-06-10T21:46:50.218385+0000","last_peered":"2023-06-10T21:46:50.218385+0000","last_clean":"2023-06-10T21:46:50.218385+0000","last_became_active":"2023-06-10T21:43:01.202574+0000","last_became_peered":"2023-06-10T21:43:01.202574+0000","last_unstale":"2023-06-10T21:46:50.218385+0000","last_undegraded":"2023-06-10T21:46:50.218385+0000","last_fullsized":"2023-06-10T21:46:50.218385+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:47.723259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":85,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229910+0000","last_change":"2023-06-10T21:46:14.282107+0000","last_active":"2023-06-10T21:46:50.229910+0000","last_peered":"2023-06-10T21:46:50.229910+0000","last_clean":"2023-06-10T21:46:50.229910+0000","last_became_active":"2023-06-10T21:46:14.281805+0000","last_became_peered":"2023-06-10T21:46:14.281805+0000","last_unstale":"2023-06-10T21:46:50.229910+0000","last_undegraded":"2023-06-10T21:46:50.229910+0000","last_fullsized":"2023-06-10T21:46:50.229910+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.f","version":"436'5777","reported_seq":5801,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:04.254132+0000","last_change":"2023-06-10T21:45:44.480803+0000","last_active":"2023-06-10T21:47:04.254132+0000","last_peered":"2023-06-10T21:47:04.254132+0000","last_clean":"2023-06-10T21:47:04.254132+0000","last_became_active":"2023-06-10T21:45:44.480472+0000","last_became_peered":"2023-06-10T21:45:44.480472+0000","last_unstale":"2023-06-10T21:47:04.254132+0000","last_undegraded":"2023-06-10T21:47:04.254132+0000","last_fullsized":"2023-06-10T21:47:04.254132+0000","mapping_epoch":420,"log_start":"424'1400","ondisk_log_start":"424'1400","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4377,"log_dups_size":0,"ondisk_log_size":4377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378601472,"num_objects":5777,"num_object_clones":0,"num_object_copies":17331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5777,"num_write_kb":369728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":392,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215226+0000","last_change":"2023-06-10T21:46:14.280999+0000","last_active":"2023-06-10T21:46:50.215226+0000","last_peered":"2023-06-10T21:46:50.215226+0000","last_clean":"2023-06-10T21:46:50.215226+0000","last_became_active":"2023-06-10T21:46:14.280409+0000","last_became_peered":"2023-06-10T21:46:14.280409+0000","last_unstale":"2023-06-10T21:46:50.215226+0000","last_undegraded":"2023-06-10T21:46:50.215226+0000","last_fullsized":"2023-06-10T21:46:50.215226+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:24:46.683725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":636,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.228815+0000","last_change":"2023-06-10T21:40:32.136192+0000","last_active":"2023-06-10T21:46:50.228815+0000","last_peered":"2023-06-10T21:46:50.228815+0000","last_clean":"2023-06-10T21:46:50.228815+0000","last_became_active":"2023-06-10T21:40:32.135821+0000","last_became_peered":"2023-06-10T21:40:32.135821+0000","last_unstale":"2023-06-10T21:46:50.228815+0000","last_undegraded":"2023-06-10T21:46:50.228815+0000","last_fullsized":"2023-06-10T21:46:50.228815+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225642+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:46:50.225642+0000","last_peered":"2023-06-10T21:46:50.225642+0000","last_clean":"2023-06-10T21:46:50.225642+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:46:50.225642+0000","last_undegraded":"2023-06-10T21:46:50.225642+0000","last_fullsized":"2023-06-10T21:46:50.225642+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":319,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.230057+0000","last_change":"2023-06-10T21:43:01.231012+0000","last_active":"2023-06-10T21:46:50.230057+0000","last_peered":"2023-06-10T21:46:50.230057+0000","last_clean":"2023-06-10T21:46:50.230057+0000","last_became_active":"2023-06-10T21:43:01.230398+0000","last_became_peered":"2023-06-10T21:43:01.230398+0000","last_unstale":"2023-06-10T21:46:50.230057+0000","last_undegraded":"2023-06-10T21:46:50.230057+0000","last_fullsized":"2023-06-10T21:46:50.230057+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"436'5541","reported_seq":5566,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.311031+0000","last_change":"2023-06-10T21:45:44.478367+0000","last_active":"2023-06-10T21:47:01.311031+0000","last_peered":"2023-06-10T21:47:01.311031+0000","last_clean":"2023-06-10T21:47:01.311031+0000","last_became_active":"2023-06-10T21:45:44.476855+0000","last_became_peered":"2023-06-10T21:45:44.476855+0000","last_unstale":"2023-06-10T21:47:01.311031+0000","last_undegraded":"2023-06-10T21:47:01.311031+0000","last_fullsized":"2023-06-10T21:47:01.311031+0000","mapping_epoch":420,"log_start":"423'300","ondisk_log_start":"423'300","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5241,"log_dups_size":0,"ondisk_log_size":5241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:20:43.311166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363134976,"num_objects":5541,"num_object_clones":0,"num_object_copies":16623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5541,"num_write_kb":354624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":488,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229013+0000","last_change":"2023-06-10T21:43:01.270731+0000","last_active":"2023-06-10T21:46:50.229013+0000","last_peered":"2023-06-10T21:46:50.229013+0000","last_clean":"2023-06-10T21:46:50.229013+0000","last_became_active":"2023-06-10T21:43:01.270380+0000","last_became_peered":"2023-06-10T21:43:01.270380+0000","last_unstale":"2023-06-10T21:46:50.229013+0000","last_undegraded":"2023-06-10T21:46:50.229013+0000","last_fullsized":"2023-06-10T21:46:50.229013+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":106,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221578+0000","last_change":"2023-06-10T21:43:01.210106+0000","last_active":"2023-06-10T21:46:50.221578+0000","last_peered":"2023-06-10T21:46:50.221578+0000","last_clean":"2023-06-10T21:46:50.221578+0000","last_became_active":"2023-06-10T21:43:01.209743+0000","last_became_peered":"2023-06-10T21:43:01.209743+0000","last_unstale":"2023-06-10T21:46:50.221578+0000","last_undegraded":"2023-06-10T21:46:50.221578+0000","last_fullsized":"2023-06-10T21:46:50.221578+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:46:09.494557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":188,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.219222+0000","last_change":"2023-06-10T21:43:01.210278+0000","last_active":"2023-06-10T21:46:50.219222+0000","last_peered":"2023-06-10T21:46:50.219222+0000","last_clean":"2023-06-10T21:46:50.219222+0000","last_became_active":"2023-06-10T21:43:01.209988+0000","last_became_peered":"2023-06-10T21:43:01.209988+0000","last_unstale":"2023-06-10T21:46:50.219222+0000","last_undegraded":"2023-06-10T21:46:50.219222+0000","last_fullsized":"2023-06-10T21:46:50.219222+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:15:55.439650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"436'5872","reported_seq":5902,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:06.122808+0000","last_change":"2023-06-10T21:46:14.282840+0000","last_active":"2023-06-10T21:47:06.122808+0000","last_peered":"2023-06-10T21:47:06.122808+0000","last_clean":"2023-06-10T21:47:06.122808+0000","last_became_active":"2023-06-10T21:46:14.280350+0000","last_became_peered":"2023-06-10T21:46:14.280350+0000","last_unstale":"2023-06-10T21:47:06.122808+0000","last_undegraded":"2023-06-10T21:47:06.122808+0000","last_fullsized":"2023-06-10T21:47:06.122808+0000","mapping_epoch":426,"log_start":"424'1600","ondisk_log_start":"424'1600","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4272,"log_dups_size":0,"ondisk_log_size":4272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:35:53.086699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384827392,"num_objects":5872,"num_object_clones":0,"num_object_copies":17616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5872,"num_write_kb":375808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":467,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229931+0000","last_change":"2023-06-10T21:39:50.249960+0000","last_active":"2023-06-10T21:46:50.229931+0000","last_peered":"2023-06-10T21:46:50.229931+0000","last_clean":"2023-06-10T21:46:50.229931+0000","last_became_active":"2023-06-10T21:39:50.249538+0000","last_became_peered":"2023-06-10T21:39:50.249538+0000","last_unstale":"2023-06-10T21:46:50.229931+0000","last_undegraded":"2023-06-10T21:46:50.229931+0000","last_fullsized":"2023-06-10T21:46:50.229931+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":317,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217156+0000","last_change":"2023-06-10T21:43:01.272083+0000","last_active":"2023-06-10T21:46:50.217156+0000","last_peered":"2023-06-10T21:46:50.217156+0000","last_clean":"2023-06-10T21:46:50.217156+0000","last_became_active":"2023-06-10T21:43:01.271287+0000","last_became_peered":"2023-06-10T21:43:01.271287+0000","last_unstale":"2023-06-10T21:46:50.217156+0000","last_undegraded":"2023-06-10T21:46:50.217156+0000","last_fullsized":"2023-06-10T21:46:50.217156+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:40.354914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":207,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229510+0000","last_change":"2023-06-10T21:46:14.283198+0000","last_active":"2023-06-10T21:46:50.229510+0000","last_peered":"2023-06-10T21:46:50.229510+0000","last_clean":"2023-06-10T21:46:50.229510+0000","last_became_active":"2023-06-10T21:46:14.282871+0000","last_became_peered":"2023-06-10T21:46:14.282871+0000","last_unstale":"2023-06-10T21:46:50.229510+0000","last_undegraded":"2023-06-10T21:46:50.229510+0000","last_fullsized":"2023-06-10T21:46:50.229510+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.a","version":"436'5873","reported_seq":5905,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:06.111617+0000","last_change":"2023-06-10T21:46:14.285180+0000","last_active":"2023-06-10T21:47:06.111617+0000","last_peered":"2023-06-10T21:47:06.111617+0000","last_clean":"2023-06-10T21:47:06.111617+0000","last_became_active":"2023-06-10T21:46:14.279416+0000","last_became_peered":"2023-06-10T21:46:14.279416+0000","last_unstale":"2023-06-10T21:47:06.111617+0000","last_undegraded":"2023-06-10T21:47:06.111617+0000","last_fullsized":"2023-06-10T21:47:06.111617+0000","mapping_epoch":426,"log_start":"424'1600","ondisk_log_start":"424'1600","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4273,"log_dups_size":0,"ondisk_log_size":4273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:05:25.130576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384892928,"num_objects":5873,"num_object_clones":0,"num_object_copies":17619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":375872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":447,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.223861+0000","last_change":"2023-06-10T21:43:01.212540+0000","last_active":"2023-06-10T21:46:50.223861+0000","last_peered":"2023-06-10T21:46:50.223861+0000","last_clean":"2023-06-10T21:46:50.223861+0000","last_became_active":"2023-06-10T21:43:01.211969+0000","last_became_peered":"2023-06-10T21:43:01.211969+0000","last_unstale":"2023-06-10T21:46:50.223861+0000","last_undegraded":"2023-06-10T21:46:50.223861+0000","last_fullsized":"2023-06-10T21:46:50.223861+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:54:40.266971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":258,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224540+0000","last_change":"2023-06-10T21:43:01.204758+0000","last_active":"2023-06-10T21:46:50.224540+0000","last_peered":"2023-06-10T21:46:50.224540+0000","last_clean":"2023-06-10T21:46:50.224540+0000","last_became_active":"2023-06-10T21:43:01.204262+0000","last_became_peered":"2023-06-10T21:43:01.204262+0000","last_unstale":"2023-06-10T21:46:50.224540+0000","last_undegraded":"2023-06-10T21:46:50.224540+0000","last_fullsized":"2023-06-10T21:46:50.224540+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:32:53.650738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225620+0000","last_change":"2023-06-10T21:43:01.211907+0000","last_active":"2023-06-10T21:46:50.225620+0000","last_peered":"2023-06-10T21:46:50.225620+0000","last_clean":"2023-06-10T21:46:50.225620+0000","last_became_active":"2023-06-10T21:43:01.211655+0000","last_became_peered":"2023-06-10T21:43:01.211655+0000","last_unstale":"2023-06-10T21:46:50.225620+0000","last_undegraded":"2023-06-10T21:46:50.225620+0000","last_fullsized":"2023-06-10T21:46:50.225620+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:45:46.997776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"436'5566","reported_seq":5591,"reported_epoch":436,"state":"active+clean","last_fresh":"2023-06-10T21:47:01.284672+0000","last_change":"2023-06-10T21:45:44.477091+0000","last_active":"2023-06-10T21:47:01.284672+0000","last_peered":"2023-06-10T21:47:01.284672+0000","last_clean":"2023-06-10T21:47:01.284672+0000","last_became_active":"2023-06-10T21:45:44.476764+0000","last_became_peered":"2023-06-10T21:45:44.476764+0000","last_unstale":"2023-06-10T21:47:01.284672+0000","last_undegraded":"2023-06-10T21:47:01.284672+0000","last_fullsized":"2023-06-10T21:47:01.284672+0000","mapping_epoch":420,"log_start":"423'300","ondisk_log_start":"423'300","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5266,"log_dups_size":0,"ondisk_log_size":5266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:40:54.289558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364773376,"num_objects":5566,"num_object_clones":0,"num_object_copies":16698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5566,"num_write_kb":356224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":184,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229814+0000","last_change":"2023-06-10T21:40:30.285823+0000","last_active":"2023-06-10T21:46:50.229814+0000","last_peered":"2023-06-10T21:46:50.229814+0000","last_clean":"2023-06-10T21:46:50.229814+0000","last_became_active":"2023-06-10T21:40:30.285507+0000","last_became_peered":"2023-06-10T21:40:30.285507+0000","last_unstale":"2023-06-10T21:46:50.229814+0000","last_undegraded":"2023-06-10T21:46:50.229814+0000","last_fullsized":"2023-06-10T21:46:50.229814+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":298,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229513+0000","last_change":"2023-06-10T21:43:01.267576+0000","last_active":"2023-06-10T21:46:50.229513+0000","last_peered":"2023-06-10T21:46:50.229513+0000","last_clean":"2023-06-10T21:46:50.229513+0000","last_became_active":"2023-06-10T21:43:01.266578+0000","last_became_peered":"2023-06-10T21:43:01.266578+0000","last_unstale":"2023-06-10T21:46:50.229513+0000","last_undegraded":"2023-06-10T21:46:50.229513+0000","last_fullsized":"2023-06-10T21:46:50.229513+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":315,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229114+0000","last_change":"2023-06-10T21:46:14.280962+0000","last_active":"2023-06-10T21:46:50.229114+0000","last_peered":"2023-06-10T21:46:50.229114+0000","last_clean":"2023-06-10T21:46:50.229114+0000","last_became_active":"2023-06-10T21:46:14.280651+0000","last_became_peered":"2023-06-10T21:46:14.280651+0000","last_unstale":"2023-06-10T21:46:50.229114+0000","last_undegraded":"2023-06-10T21:46:50.229114+0000","last_fullsized":"2023-06-10T21:46:50.229114+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":135,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229430+0000","last_change":"2023-06-10T21:40:30.286275+0000","last_active":"2023-06-10T21:46:50.229430+0000","last_peered":"2023-06-10T21:46:50.229430+0000","last_clean":"2023-06-10T21:46:50.229430+0000","last_became_active":"2023-06-10T21:40:30.285565+0000","last_became_peered":"2023-06-10T21:40:30.285565+0000","last_unstale":"2023-06-10T21:46:50.229430+0000","last_undegraded":"2023-06-10T21:46:50.229430+0000","last_fullsized":"2023-06-10T21:46:50.229430+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":333,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.217231+0000","last_change":"2023-06-10T21:43:01.211634+0000","last_active":"2023-06-10T21:46:50.217231+0000","last_peered":"2023-06-10T21:46:50.217231+0000","last_clean":"2023-06-10T21:46:50.217231+0000","last_became_active":"2023-06-10T21:43:01.211337+0000","last_became_peered":"2023-06-10T21:43:01.211337+0000","last_unstale":"2023-06-10T21:46:50.217231+0000","last_undegraded":"2023-06-10T21:46:50.217231+0000","last_fullsized":"2023-06-10T21:46:50.217231+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:08:03.220922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":104,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.229408+0000","last_change":"2023-06-10T21:46:14.283333+0000","last_active":"2023-06-10T21:46:50.229408+0000","last_peered":"2023-06-10T21:46:50.229408+0000","last_clean":"2023-06-10T21:46:50.229408+0000","last_became_active":"2023-06-10T21:46:14.282987+0000","last_became_peered":"2023-06-10T21:46:14.282987+0000","last_unstale":"2023-06-10T21:46:50.229408+0000","last_undegraded":"2023-06-10T21:46:50.229408+0000","last_fullsized":"2023-06-10T21:46:50.229408+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":341,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221923+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:46:50.221923+0000","last_peered":"2023-06-10T21:46:50.221923+0000","last_clean":"2023-06-10T21:46:50.221923+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:46:50.221923+0000","last_undegraded":"2023-06-10T21:46:50.221923+0000","last_fullsized":"2023-06-10T21:46:50.221923+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":83,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225038+0000","last_change":"2023-06-10T21:46:14.279723+0000","last_active":"2023-06-10T21:46:50.225038+0000","last_peered":"2023-06-10T21:46:50.225038+0000","last_clean":"2023-06-10T21:46:50.225038+0000","last_became_active":"2023-06-10T21:46:14.279371+0000","last_became_peered":"2023-06-10T21:46:14.279371+0000","last_unstale":"2023-06-10T21:46:50.225038+0000","last_undegraded":"2023-06-10T21:46:50.225038+0000","last_fullsized":"2023-06-10T21:46:50.225038+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:43.249586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":412,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225859+0000","last_change":"2023-06-10T21:43:01.209042+0000","last_active":"2023-06-10T21:46:50.225859+0000","last_peered":"2023-06-10T21:46:50.225859+0000","last_clean":"2023-06-10T21:46:50.225859+0000","last_became_active":"2023-06-10T21:43:01.208691+0000","last_became_peered":"2023-06-10T21:43:01.208691+0000","last_unstale":"2023-06-10T21:46:50.225859+0000","last_undegraded":"2023-06-10T21:46:50.225859+0000","last_fullsized":"2023-06-10T21:46:50.225859+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:00:03.454293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":91,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.224401+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:46:50.224401+0000","last_peered":"2023-06-10T21:46:50.224401+0000","last_clean":"2023-06-10T21:46:50.224401+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:46:50.224401+0000","last_undegraded":"2023-06-10T21:46:50.224401+0000","last_fullsized":"2023-06-10T21:46:50.224401+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.221655+0000","last_change":"2023-06-10T21:43:01.202260+0000","last_active":"2023-06-10T21:46:50.221655+0000","last_peered":"2023-06-10T21:46:50.221655+0000","last_clean":"2023-06-10T21:46:50.221655+0000","last_became_active":"2023-06-10T21:43:01.201964+0000","last_became_peered":"2023-06-10T21:43:01.201964+0000","last_unstale":"2023-06-10T21:46:50.221655+0000","last_undegraded":"2023-06-10T21:46:50.221655+0000","last_fullsized":"2023-06-10T21:46:50.221655+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:41:45.099670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":107,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.215916+0000","last_change":"2023-06-10T21:46:14.280900+0000","last_active":"2023-06-10T21:46:50.215916+0000","last_peered":"2023-06-10T21:46:50.215916+0000","last_clean":"2023-06-10T21:46:50.215916+0000","last_became_active":"2023-06-10T21:46:14.280572+0000","last_became_peered":"2023-06-10T21:46:14.280572+0000","last_unstale":"2023-06-10T21:46:50.215916+0000","last_undegraded":"2023-06-10T21:46:50.215916+0000","last_fullsized":"2023-06-10T21:46:50.215916+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:24:13.502992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":105,"reported_epoch":435,"state":"active+clean","last_fresh":"2023-06-10T21:46:50.225769+0000","last_change":"2023-06-10T21:43:01.204106+0000","last_active":"2023-06-10T21:46:50.225769+0000","last_peered":"2023-06-10T21:46:50.225769+0000","last_clean":"2023-06-10T21:46:50.225769+0000","last_became_active":"2023-06-10T21:43:01.203878+0000","last_became_peered":"2023-06-10T21:43:01.203878+0000","last_unstale":"2023-06-10T21:46:50.225769+0000","last_undegraded":"2023-06-10T21:46:50.225769+0000","last_fullsized":"2023-06-10T21:46:50.225769+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:11.397692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":6014369792,"num_objects":91772,"num_object_clones":0,"num_object_copies":275316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91772,"num_write_kb":5873408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18181390336,"data_stored":18181390336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75372,"ondisk_log_size":75372,"up":48,"acting":48,"num_store_stats":4},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16270,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430800,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5072016,"kb_used_data":4024712,"kb_used_omap":0,"kb_used_meta":1047296,"kb_avail":88673136,"statfs":{"total":95995035648,"available":90801291264,"internally_reserved":0,"allocated":4121305088,"data_stored":4120586745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1072431104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.92100000000000004,"15min":1.724},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.224},"max":{"1min":2.0800000000000001,"5min":8.5129999999999999,"15min":56.639000000000003},"last":1.927},{"interface":"front","average":{"1min":1.081,"5min":1.369,"15min":2.1299999999999999},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.25},"max":{"1min":5.1440000000000001,"5min":20.568999999999999,"15min":56.628},"last":1.423}]},{"osd":1,"last update":"Sat Jun 10 21:46:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":1.2430000000000001,"15min":2.6299999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.25600000000000001},"max":{"1min":2.1099999999999999,"5min":8.1500000000000004,"15min":87.417000000000002},"last":1.974},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.488,"15min":2.9609999999999999},"min":{"1min":0.33700000000000002,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":3.1230000000000002,"5min":10.484,"15min":87.438000000000002},"last":2.681}]},{"osd":2,"last update":"Sat Jun 10 21:46:44 2023","interfaces":[{"interface":"back","average":{"1min":29.420000000000002,"5min":9.1989999999999998,"15min":6.5279999999999996},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":218.90600000000001,"5min":218.90600000000001,"15min":218.90600000000001},"last":3.0670000000000002},{"interface":"front","average":{"1min":29.443000000000001,"5min":9.3659999999999997,"15min":6.5599999999999996},"min":{"1min":0.443,"5min":0.35099999999999998,"15min":0.30499999999999999},"max":{"1min":218.851,"5min":218.851,"15min":218.851},"last":2.0150000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:46:33 2023","interfaces":[{"interface":"back","average":{"1min":16.815000000000001,"5min":6.7800000000000002,"15min":5.7309999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.27000000000000002},"max":{"1min":218.35300000000001,"5min":218.35300000000001,"15min":218.35300000000001},"last":3.1440000000000001},{"interface":"front","average":{"1min":17.359999999999999,"5min":6.7149999999999999,"15min":5.569},"min":{"1min":0.38800000000000001,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":218.74600000000001,"5min":218.74600000000001,"15min":218.74600000000001},"last":2.073}]},{"osd":4,"last update":"Sat Jun 10 21:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.1950000000000001,"15min":0.70199999999999996},"min":{"1min":0.308,"5min":0.27500000000000002,"15min":0.249},"max":{"1min":3.6000000000000001,"5min":13.429,"15min":13.429},"last":2.387},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.3129999999999999,"15min":0.85499999999999998},"min":{"1min":0.32900000000000001,"5min":0.28699999999999998,"15min":0.245},"max":{"1min":3.9580000000000002,"5min":12.279999999999999,"15min":22.77},"last":2.1269999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:46:07 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":8.0039999999999996,"15min":10.786},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.3230000000000004,"5min":131.619,"15min":131.619},"last":5.3040000000000003},{"interface":"front","average":{"1min":1.6870000000000001,"5min":7.944,"15min":10.647},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":10.603999999999999,"5min":134.928,"15min":134.928},"last":2.5990000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.0840000000000001,"15min":0.75900000000000001},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.9780000000000002,"5min":7.9660000000000002,"15min":10.890000000000001},"last":2.2559999999999998},{"interface":"front","average":{"1min":0.878,"5min":1.292,"15min":0.79000000000000004},"min":{"1min":0.38400000000000001,"5min":0.34799999999999998,"15min":0.30599999999999999},"max":{"1min":3.5499999999999998,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":2.0470000000000002}]}]},{"osd":6,"up_from":0,"seq":180388627028,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566471,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":0,"seq":1047972020447,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":120,"seq":515396075858,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4659768,"kb_used_data":3579184,"kb_used_omap":7,"kb_used_meta":1080568,"kb_avail":89085384,"statfs":{"total":95995035648,"available":91223433216,"internally_reserved":0,"allocated":3665084416,"data_stored":3663691663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1106501713},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:04 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.4850000000000001,"15min":4.0039999999999996},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19},"max":{"1min":5.8849999999999998,"5min":8.5899999999999999,"15min":171.33699999999999},"last":3.6600000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.2170000000000001,"15min":3.7050000000000001},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.20799999999999999},"max":{"1min":6.415,"5min":6.415,"15min":171.56200000000001},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:46:56 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.4590000000000001,"15min":5.0709999999999997},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":5.5179999999999998,"5min":11.398999999999999,"15min":171.286},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.641,"15min":3.7040000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":4.3390000000000004,"5min":9.0500000000000007,"15min":147.81299999999999},"last":3.6040000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:46:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3949999999999996,"5min":8.4849999999999994,"15min":6.7590000000000003},"min":{"1min":0.29099999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":62.441000000000003,"5min":145.07599999999999,"15min":171.78299999999999},"last":3.6960000000000002},{"interface":"front","average":{"1min":4.2889999999999997,"5min":8.0690000000000008,"15min":6.5750000000000002},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.24399999999999999},"max":{"1min":62.517000000000003,"5min":144.99000000000001,"15min":171.65100000000001},"last":2.169}]},{"osd":4,"last update":"Sat Jun 10 21:46:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.3839999999999999,"15min":2.2000000000000002},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.28100000000000003},"max":{"1min":3.2570000000000001,"5min":8.9629999999999992,"15min":171.48699999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.5509999999999999,"15min":3.0470000000000002},"min":{"1min":0.38300000000000001,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":5.5629999999999997,"5min":5.8150000000000004,"15min":171.18100000000001},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:46:04 2023","interfaces":[{"interface":"back","average":{"1min":7.7290000000000001,"5min":6.1479999999999997,"15min":6.7039999999999997},"min":{"1min":0.46000000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":117.562,"5min":117.562,"15min":117.562},"last":3.7850000000000001},{"interface":"front","average":{"1min":7.3940000000000001,"5min":5.6790000000000003,"15min":6.1210000000000004},"min":{"1min":0.42899999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":117.535,"5min":117.535,"15min":117.535},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:46:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.722,"15min":2.3610000000000002},"min":{"1min":0.42999999999999999,"5min":0.41999999999999998,"15min":0.307},"max":{"1min":5.6340000000000003,"5min":9.0139999999999993,"15min":171.441},"last":0.629},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.708,"15min":3.1419999999999999},"min":{"1min":0.39100000000000001,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":5.6470000000000002,"5min":5.6470000000000002,"15min":171.24600000000001},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:46:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0369999999999999,"5min":3.625,"15min":4.6289999999999996},"min":{"1min":0.497,"5min":0.46500000000000002,"15min":0.34200000000000003},"max":{"1min":23.789999999999999,"5min":124.071,"15min":219.131},"last":3.7450000000000001},{"interface":"front","average":{"1min":3.0859999999999999,"5min":3.4550000000000001,"15min":4.4950000000000001},"min":{"1min":0.42199999999999999,"5min":0.375,"15min":0.32200000000000001},"max":{"1min":23.77,"5min":125.203,"15min":219.19},"last":0.57199999999999995}]}]},{"osd":3,"up_from":250,"seq":1073741824219,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5363464,"kb_used_data":4504128,"kb_used_omap":7,"kb_used_meta":859320,"kb_avail":88381688,"statfs":{"total":95995035648,"available":90502848512,"internally_reserved":0,"allocated":4612227072,"data_stored":4610698973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":879943699},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.76800000000000002,"15min":2.843},"min":{"1min":0.216,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":6.46,"5min":6.46,"15min":105.09},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.042,"5min":1.573,"15min":3.097},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.20799999999999999},"max":{"1min":6.5519999999999996,"5min":6.6429999999999998,"15min":99.852999999999994},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.252,"15min":3},"min":{"1min":0.247,"5min":0.247,"15min":0.20499999999999999},"max":{"1min":2.6259999999999999,"5min":6.6459999999999999,"15min":94.766000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.88500000000000001,"15min":3.048},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":1.651,"5min":5.4109999999999996,"15min":94.691000000000003},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":3.4630000000000001,"15min":4.04},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.23300000000000001},"max":{"1min":4.7720000000000002,"5min":126.913,"15min":231.07900000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.92200000000000004,"5min":3.6459999999999999,"15min":5.0039999999999996},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.22800000000000001},"max":{"1min":4.157,"5min":128.52199999999999,"15min":231.58600000000001},"last":0.504}]},{"osd":4,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.591,"15min":1.472},"min":{"1min":0.34100000000000003,"5min":0.30499999999999999,"15min":0.28499999999999998},"max":{"1min":2.754,"5min":9.5150000000000006,"15min":13.417},"last":5.8970000000000002},{"interface":"front","average":{"1min":1.351,"5min":1.613,"15min":2.0739999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":6.8479999999999999,"5min":8.8379999999999992,"15min":19.244},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.0109999999999999,"15min":1.0629999999999999},"min":{"1min":0.39300000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.7370000000000001,"5min":6.532,"15min":6.532},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.855,"5min":2.4470000000000001,"15min":2.6179999999999999},"min":{"1min":0.41799999999999998,"5min":0.378,"15min":0.378},"max":{"1min":7.8120000000000003,"5min":7.8120000000000003,"15min":7.8120000000000003},"last":0.96599999999999997}]},{"osd":6,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.7629999999999999,"15min":2.137},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":3.8620000000000001,"5min":9.0250000000000004,"15min":19.164999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.214,"15min":1.5349999999999999},"min":{"1min":0.434,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.1019999999999999,"5min":6.0750000000000002,"15min":17.327000000000002},"last":0.629}]},{"osd":7,"last update":"Sat Jun 10 21:46:36 2023","interfaces":[{"interface":"back","average":{"1min":4.6829999999999998,"5min":2.6779999999999999,"15min":2.6960000000000002},"min":{"1min":0.39800000000000002,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":56.317999999999998,"5min":56.317999999999998,"15min":79.984999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":4.7960000000000003,"5min":2.1099999999999999,"15min":2.0939999999999999},"min":{"1min":0.48199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":56.731999999999999,"5min":56.731999999999999,"15min":80.272999999999996},"last":0.88600000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019484,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245490,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5892488,"kb_used_data":5546624,"kb_used_omap":11,"kb_used_meta":345844,"kb_avail":87852664,"statfs":{"total":95995035648,"available":89961127936,"internally_reserved":0,"allocated":5679742976,"data_stored":5679032430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":354144758},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.8100000000000001,"15min":1.99},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.573,"5min":8.2590000000000003,"15min":8.2590000000000003},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.3109999999999999,"15min":1.3180000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.0490000000000004,"5min":5.0490000000000004,"15min":5.0490000000000004},"last":0.52000000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":2.2829999999999999,"15min":2.5329999999999999},"min":{"1min":0.29899999999999999,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.9610000000000003,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.069,"5min":1.9810000000000001,"15min":2.181},"min":{"1min":0.315,"5min":0.308,"15min":0.308},"max":{"1min":4.9980000000000002,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":0.58699999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":3.294,"15min":2.339},"min":{"1min":0.31900000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.3499999999999996,"5min":159.94999999999999,"15min":159.94999999999999},"last":2.177},{"interface":"front","average":{"1min":1.004,"5min":3.4489999999999998,"15min":2.6869999999999998},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.8279999999999998,"5min":159.851,"15min":159.851},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":6.109,"15min":7.5250000000000004},"min":{"1min":0.36499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":5.2549999999999999,"5min":133.07499999999999,"15min":133.07499999999999},"last":0.437},{"interface":"front","average":{"1min":1.4430000000000001,"5min":6.0919999999999996,"15min":7.2939999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.5049999999999999,"5min":133.03800000000001,"15min":133.03800000000001},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.97499999999999998,"15min":0.93100000000000005},"min":{"1min":0.33100000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.274,"5min":5.173,"15min":5.173},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.431,"5min":1.9339999999999999,"15min":2.1829999999999998},"min":{"1min":0.33400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.0700000000000003,"5min":7.476,"15min":7.476},"last":3.5369999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.5900000000000001,"15min":1.788},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.0810000000000004,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.302,"15min":1.3300000000000001},"min":{"1min":0.41999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Jun 10 21:46:06 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.5660000000000001,"15min":1.5960000000000001},"min":{"1min":0.35799999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.6210000000000004,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.974,"5min":2.722,"15min":2.8340000000000001},"min":{"1min":0.46200000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.5370000000000008,"5min":9.5370000000000008,"15min":9.5370000000000008},"last":3.569}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3675258880,"data_stored":3675258880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4633985024,"data_stored":4633985024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5707399168,"data_stored":5707399168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4164747264,"data_stored":4164747264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:47:09.617 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:47:09.617 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:47:09.618 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-06-10T21:47:09.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 0 2023-06-10T21:47:09.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: from='client.47255 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: failed to encode map e437 with expected crc 2023-06-10T21:47:09.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: pgmap v1610: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-10T21:47:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:09.989 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:09.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:09.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:10.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:10.143 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:10.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 76 16 98036 98020 80.5628 53.6875 0.0253062 0.0124071 2023-06-10T21:47:10.214 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:10.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:10.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:10.392 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:10.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:10 smithi152 ceph-mon[156819]: pgmap v1611: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:47:10.465 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:10.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:10.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:10.573 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:10.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:10 smithi064 ceph-mon[180297]: pgmap v1611: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:47:10.602 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:10 smithi064 ceph-mon[186706]: pgmap v1611: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2023-06-10T21:47:10.604 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:11.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 77 16 98766 98750 80.1092 45.625 0.018362 0.0124507 2023-06-10T21:47:11.279 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:11.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:11.387 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:11.451 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:11.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:11.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:11.695 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:11.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:11.792 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:12.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:47:12.158 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:12.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:12.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:12.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 78 16 99711 99695 79.8394 59.0625 0.0208684 0.0125202 2023-06-10T21:47:12.220 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:47:12.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:12.283 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:47:12.315 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:47:12.315 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:47:12.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.315 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99751 14.1 14:8b40bfc7:::benchmark_data_smithi064_559916_object99750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.315 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.254657+0000", 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.041772529000000003, 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.041806844000000003, 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.316 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99751 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254657+0000", 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254657+0000", 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254656+0000", 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254706+0000", 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9382999999999998e-05 2023-06-10T21:47:12.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254713+0000", 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9229999999999996e-06 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254718+0000", 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5319999999999997e-06 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.257568+0000", 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028497430000000001 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.257682+0000", 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011421099999999999 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.257821+0000", 2023-06-10T21:47:12.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013936499999999999 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.261420+0000", 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035983830000000001 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.263534+0000", 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0021143400000000001 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99753 14.1 14:8a685c32:::benchmark_data_smithi064_559916_object99752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.271669+0000", 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.024760411999999999, 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.024896068, 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.325 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99753 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271669+0000", 2023-06-10T21:47:12.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271669+0000", 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271666+0000", 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999976 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271726+0000", 2023-06-10T21:47:12.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0053000000000003e-05 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271748+0000", 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1262000000000001e-05 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.271758+0000", 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0696999999999999e-05 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.272030+0000", 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00027135100000000002 2023-06-10T21:47:12.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.272185+0000", 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015482699999999999 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:47:12.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.272344+0000", 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159841 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.275221+0000", 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028769300000000002 2023-06-10T21:47:12.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.277584+0000", 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023621549999999999 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:99747 14.1 e437/420)", 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.278977+0000", 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.017452220000000001, 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017671511000000001, 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.334 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.278977+0000", 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.278977+0000", 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.278976+0000", 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.278980+0000", 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2769999999999999e-06 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.278993+0000", 2023-06-10T21:47:12.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2680999999999999e-05 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.279001+0000", 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.498e-06 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.338 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:99751 14.1 e437/420)", 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.284198+0000", 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012231205, 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012497051, 2023-06-10T21:47:12.339 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284198+0000", 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284198+0000", 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284197+0000", 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284205+0000", 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4519999999999997e-06 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284218+0000", 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2578000000000001e-05 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284227+0000", 2023-06-10T21:47:12.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7660000000000004e-06 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:99756 14.0 e437/420)", 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.284247+0000", 2023-06-10T21:47:12.344 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012181888, 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012494851, 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284247+0000", 2023-06-10T21:47:12.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284247+0000", 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284247+0000", 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284276+0000", 2023-06-10T21:47:12.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9217000000000002e-05 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284286+0000", 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9830000000000008e-06 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284291+0000", 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4110000000000004e-06 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.349 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99759 14.a 14:5094f70a:::benchmark_data_smithi064_559916_object99758:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.284384+0000", 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.012044984999999999, 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012412116000000001, 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.350 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99759 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284384+0000", 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284384+0000", 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284383+0000", 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284408+0000", 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4921000000000001e-05 2023-06-10T21:47:12.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284421+0000", 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.33e-05 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284428+0000", 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.204e-06 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284478+0000", 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9322e-05 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284662+0000", 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018389799999999999 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.284813+0000", 2023-06-10T21:47:12.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015106599999999999 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.292853+0000", 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080405059999999993 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.292883+0000", 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9668000000000001e-05 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99760 14.a 14:591551c8:::benchmark_data_smithi064_559916_object99759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.291419+0000", 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0050103910000000003, 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054650339999999997, 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.359 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99760 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291419+0000", 2023-06-10T21:47:12.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291419+0000", 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291417+0000", 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291453+0000", 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6625999999999998e-05 2023-06-10T21:47:12.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291478+0000", 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4423e-05 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291489+0000", 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1582e-05 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.292903+0000", 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014141819999999999 2023-06-10T21:47:12.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.293071+0000", 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016794800000000001 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:47:12.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.293240+0000", 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016882 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:99753 14.1 e437/420)", 2023-06-10T21:47:12.366 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.291520+0000", 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0049092379999999998, 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005435446, 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291520+0000", 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291520+0000", 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291519+0000", 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291524+0000", 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4630000000000003e-06 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291536+0000", 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2012e-05 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291542+0000", 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0310000000000004e-06 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:99758 14.0 e437/420)", 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.291554+0000", 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0048748319999999999, 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054465390000000002, 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.372 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291554+0000", 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291554+0000", 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291554+0000", 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291578+0000", 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4267999999999999e-05 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291585+0000", 2023-06-10T21:47:12.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7859999999999999e-06 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.291590+0000", 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8389999999999997e-06 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.376 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99762 14.2 14:40ea843e:::benchmark_data_smithi064_559916_object99761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.294775+0000", 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.001654125, 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022786590000000002, 2023-06-10T21:47:12.377 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99762 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294775+0000", 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294775+0000", 2023-06-10T21:47:12.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294773+0000", 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294843+0000", 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0345000000000005e-05 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294864+0000", 2023-06-10T21:47:12.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0704e-05 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294876+0000", 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2035000000000001e-05 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295814+0000", 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00093772400000000002 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295947+0000", 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013316200000000001 2023-06-10T21:47:12.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.296079+0000", 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013228400000000001 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99768 14.3 14.897888a3 (undecoded) ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.294885+0000", 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0015440359999999999, 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002239909, 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.385 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99768 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294885+0000", 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294885+0000", 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294885+0000", 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294928+0000", 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3432e-05 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294939+0000", 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1171e-05 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.294945+0000", 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4280000000000004e-06 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:99761 14.0 e437/420)", 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.295510+0000", 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00091956700000000004, 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0016666700000000001, 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295510+0000", 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295510+0000", 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295509+0000", 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295533+0000", 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4816000000000001e-05 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295545+0000", 2023-06-10T21:47:12.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1537e-05 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295552+0000", 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9589999999999998e-06 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:99747 14.1 14:87b0024b:::benchmark_data_smithi064_559916_object99746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e437)", 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.254484+0000", 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.041945115999999998, 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.042745867, 2023-06-10T21:47:12.396 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 99747 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254484+0000", 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254484+0000", 2023-06-10T21:47:12.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254482+0000", 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254515+0000", 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3312999999999998e-05 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254539+0000", 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3601999999999999e-05 2023-06-10T21:47:12.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254551+0000", 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2167999999999999e-05 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.254886+0000", 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033521700000000001 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.255017+0000", 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000130961 2023-06-10T21:47:12.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.255158+0000", 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014078199999999999 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:12.403 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.261406+0000", 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062476739999999999 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.263505+0000", 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020997540000000001 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.404 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:99766 14.e e437/420 14:7e51dc46:::benchmark_data_smithi064_559916_object99765:head v 437'6369, mlcod=437'6367)", 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:12.295558+0000", 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00087063899999999998, 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0017558420000000001, 2023-06-10T21:47:12.405 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295558+0000", 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295558+0000", 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295558+0000", 2023-06-10T21:47:12.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295577+0000", 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9097e-05 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295583+0000", 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9909999999999999e-06 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295588+0000", 2023-06-10T21:47:12.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9910000000000002e-06 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295652+0000", 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3465000000000003e-05 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:12.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:12.295679+0000", 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7358000000000001e-05 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:47:12.411 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 14 2023-06-10T21:47:12.412 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:47:12.412 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:47:12.412 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:47:12.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1403086364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:12.413 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1403086364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:12.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:12.446 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:47:12.487 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:47:12.539 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:47:12.582 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:47:12.594 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.594 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.595 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:47:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1403086364' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:12] "GET /metrics HTTP/1.1" 200 224106 "" "Prometheus/2.33.4" 2023-06-10T21:47:13.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 79 16 100505 100489 79.4571 49.625 0.0145493 0.0125817 2023-06-10T21:47:13.591 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:13 smithi064 ceph-mon[180297]: pgmap v1612: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:13.592 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:13 smithi064 ceph-mon[186706]: pgmap v1612: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:13.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:13 smithi152 ceph-mon[156819]: pgmap v1612: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:14.211 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:2023-06-10T21:47:14.210211+0000 min lat: 0.00126485 max lat: 1.07351 avg lat: 0.0126183 2023-06-10T21:47:14.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-10T21:47:14.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 80 16 101476 101460 79.2225 60.6875 0.0134797 0.0126183 2023-06-10T21:47:14.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:47:14.375 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:14.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:14.488 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:14.527 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:14.599 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:14.603 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:14 smithi152 ceph-mon[156819]: pgmap v1613: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:14.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:14.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:14 smithi064 ceph-mon[186706]: pgmap v1613: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:14.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:14 smithi064 ceph-mon[180297]: pgmap v1613: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:14.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:14.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:15.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:15.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:15.212 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 81 16 102573 102557 79.0909 68.5625 0.0154274 0.0126385 2023-06-10T21:47:15.303 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:15.360 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:15.468 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:15.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:15.574 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:15.581 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:15.666 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:15.769 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:15.833 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:15.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:15.949 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:16.022 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:16.219 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:16.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 82 16 103523 103507 78.8419 59.375 0.0143654 0.0126792 2023-06-10T21:47:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:16.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:16.329 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:16.395 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:47:16.424 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:16.495 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:16 smithi064 conmon[180272]: 2023-06-10T21:47:16.297+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e437 definitely_dead 0 2023-06-10T21:47:17.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 83 16 104611 104595 78.7113 68 0.010828 0.0127003 2023-06-10T21:47:17.317 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.0. 2023-06-10T21:47:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[186706]: pgmap v1614: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1658900385' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:17.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[180297]: pgmap v1614: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:17.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1658900385' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:17 smithi152 ceph-mon[156819]: pgmap v1614: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 926 op/s 2023-06-10T21:47:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1658900385' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-06-10T21:47:18.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 84 16 104708 104692 77.8468 6.0625 0.00305727 0.0127026 2023-06-10T21:47:18.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:18.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:18.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:18.393 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:47:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: osdmap e438: 8 total, 8 up, 5 in 2023-06-10T21:47:18.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[186706]: pgmap v1616: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:47:18.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: osdmap e438: 8 total, 8 up, 5 in 2023-06-10T21:47:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:18.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:18 smithi064 ceph-mon[180297]: pgmap v1616: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2023-06-10T21:47:18.429 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-06-10T21:47:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: osdmap e438: 8 total, 8 up, 5 in 2023-06-10T21:47:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: failed to encode map e438 with expected crc 2023-06-10T21:47:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:18.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:18 smithi152 ceph-mon[156819]: pgmap v1616: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2023-06-10T21:47:19.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:19.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:19.137 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-06-10T21:47:19.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:19.217 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:19.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 85 16 104708 104692 76.9315 0 - 0.0127026 2023-06-10T21:47:19.267 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:19.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:19.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:19.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: failed to encode map e438 with expected crc 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: osdmap e439: 8 total, 8 up, 5 in 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: failed to encode map e438 with expected crc 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: osdmap e439: 8 total, 8 up, 5 in 2023-06-10T21:47:19.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.430 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: failed to encode map e438 with expected crc 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: osdmap e439: 8 total, 8 up, 5 in 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: failed to encode map e439 with expected crc 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:19.851 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:19.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:19.903 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:19.964 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:19.993 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:20.039 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:47:20.039 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:47:20.050 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:20.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:20.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:20.189 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:20.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 86 16 105086 105070 76.3119 11.8125 0.0221351 0.0130986 2023-06-10T21:47:20.246 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:20.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:20.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.331 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: osdmap e440: 8 total, 8 up, 5 in 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.2 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6523 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.9 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6561 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.5 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6604 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.b starting backfill to osd.0 from (0'0,0'0] MAX to 437'6454 2023-06-10T21:47:20.332 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.8 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6569 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.7 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6505 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 437'6700 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.c starting backfill to osd.0 from (0'0,0'0] MAX to 437'6525 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.6 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6424 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.3 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6540 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: 14.0 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6517 2023-06-10T21:47:20.333 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: pgmap v1619: 89 pgs: 1 remapped+peering, 88 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 53 MiB/s wr, 847 op/s; 902 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:47:20.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:47:20.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:47:20.334 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:20.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:20.416 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: osdmap e440: 8 total, 8 up, 5 in 2023-06-10T21:47:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.2 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6523 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.9 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6561 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.5 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6604 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.b starting backfill to osd.0 from (0'0,0'0] MAX to 437'6454 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.8 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6569 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.7 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6505 2023-06-10T21:47:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 437'6700 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.c starting backfill to osd.0 from (0'0,0'0] MAX to 437'6525 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.6 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6424 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.3 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6540 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: 14.0 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6517 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: pgmap v1619: 89 pgs: 1 remapped+peering, 88 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 53 MiB/s wr, 847 op/s; 902 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:47:20.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:47:20.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: osdmap e440: 8 total, 8 up, 5 in 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.2 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6523 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.9 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6561 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.5 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6604 2023-06-10T21:47:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.b starting backfill to osd.0 from (0'0,0'0] MAX to 437'6454 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.8 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6569 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.7 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6505 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.e starting backfill to osd.0 from (0'0,0'0] MAX to 437'6700 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.c starting backfill to osd.0 from (0'0,0'0] MAX to 437'6525 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.6 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6424 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.3 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6540 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: failed to encode map e440 with expected crc 2023-06-10T21:47:20.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: 14.0 starting backfill to osd.0 from (0'0,0'0] MAX to 437'6517 2023-06-10T21:47:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: pgmap v1619: 89 pgs: 1 remapped+peering, 88 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 53 MiB/s wr, 847 op/s; 902 B/s, 1 keys/s, 0 objects/s recovering 2023-06-10T21:47:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:47:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:47:20.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:21.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 87 16 105719 105703 75.8897 39.5625 0.0179188 0.0131723 2023-06-10T21:47:21.312 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:21.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:47:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:21.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:21.359 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:47:21.427 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:21.460 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:21.517 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:21.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:21.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: osdmap e441: 8 total, 8 up, 5 in 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/206348340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: osdmap e441: 8 total, 8 up, 5 in 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/206348340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: osdmap e441: 8 total, 8 up, 5 in 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: failed to encode map e441 with expected crc 2023-06-10T21:47:21.662 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/206348340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.663 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:47:21.665 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:21.732 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:21.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:21.839 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:21.873 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:21.933 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:21.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:22.040 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:22.079 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:22.120 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:22.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:22.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 88 16 106329 106313 75.4608 38.125 0.131524 0.0132283 2023-06-10T21:47:22.228 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:22.307 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: osdmap e442: 8 total, 8 up, 5 in 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[180297]: pgmap v1622: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 632 op/s; 72545/316944 objects misplaced (22.889%); 2.0 MiB/s, 2 keys/s, 32 objects/s recovering 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:47:22.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: osdmap e442: 8 total, 8 up, 5 in 2023-06-10T21:47:22.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:22 smithi064 ceph-mon[186706]: pgmap v1622: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 632 op/s; 72545/316944 objects misplaced (22.889%); 2.0 MiB/s, 2 keys/s, 32 objects/s recovering 2023-06-10T21:47:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:47:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: osdmap e442: 8 total, 8 up, 5 in 2023-06-10T21:47:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: failed to encode map e442 with expected crc 2023-06-10T21:47:22.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:22 smithi152 ceph-mon[156819]: pgmap v1622: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 40 MiB/s wr, 632 op/s; 72545/316944 objects misplaced (22.889%); 2.0 MiB/s, 2 keys/s, 32 objects/s recovering 2023-06-10T21:47:22.796 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:22.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:22.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:22.988 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:22] "GET /metrics HTTP/1.1" 200 224099 "" "Prometheus/2.33.4" 2023-06-10T21:47:23.221 INFO:tasks.radosbench.radosbench.0.smithi064.stdout: 89 16 107085 107069 75.1439 47.25 0.0262164 0.013304 2023-06-10T21:47:23.570 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:23.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:23.677 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:23.767 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:23.790 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:24.007 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:24.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:24.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:24.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:24.139 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5, 0] out_osds: [6, 4, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:47:24.139 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:47:24.139 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-06-10T21:47:24.139 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight-by-utilization 110 1.0 2023-06-10T21:47:24.154 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:24.200 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:24.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total time run: 90.011 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Total writes made: 107853 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Write size: 65536 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Object size: 65536 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Bandwidth (MB/sec): 74.8888 2023-06-10T21:47:24.222 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Bandwidth: 40.0477 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max bandwidth (MB/sec): 225.125 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min bandwidth (MB/sec): 0 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average IOPS: 1198 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev IOPS: 640.763 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max IOPS: 3602 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min IOPS: 0 2023-06-10T21:47:24.223 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Average Latency(s): 0.0133491 2023-06-10T21:47:24.224 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Stddev Latency(s): 0.0349808 2023-06-10T21:47:24.224 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Max latency(s): 2.36628 2023-06-10T21:47:24.224 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Min latency(s): 0.00126485 2023-06-10T21:47:24.228 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Cleaning up (deleting benchmark objects) 2023-06-10T21:47:24.307 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:24.338 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:24.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:24 smithi064 ceph-mon[180297]: pgmap v1623: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 27 MiB/s wr, 424 op/s; 72545/316944 objects misplaced (22.889%); 1.4 MiB/s, 1 keys/s, 21 objects/s recovering 2023-06-10T21:47:24.404 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:24.411 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:24 smithi152 ceph-mon[156819]: pgmap v1623: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 27 MiB/s wr, 424 op/s; 72545/316944 objects misplaced (22.889%); 1.4 MiB/s, 1 keys/s, 21 objects/s recovering 2023-06-10T21:47:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:24.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:24.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:24.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:47:24.547 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:24.555 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:47:24.592 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:47:24.607 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:24.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:24.631 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:47:24.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:24 smithi064 ceph-mon[186706]: pgmap v1623: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 1 remapped+peering, 77 active+clean; 6.4 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 27 MiB/s wr, 424 op/s; 72545/316944 objects misplaced (22.889%); 1.4 MiB/s, 1 keys/s, 21 objects/s recovering 2023-06-10T21:47:24.669 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:47:24.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:47:24.716 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:24.750 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:47:24.778 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:24.801 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:47:24.826 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.826 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.826 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.826 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.826 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.827 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.827 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:47:24.827 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:47:25.368 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:47:25.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4005848590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:25.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4005848590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4005848590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:26.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:26.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:26.177 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:26.223 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:26.300 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:26.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:26.408 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:26.443 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:26.509 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:26.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:26.615 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:26.648 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:26.716 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:26.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:26.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:26.827 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:26.857 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:26.914 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:26.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:27.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:27.039 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:27.039 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:27.131 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:27.188 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:47:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:47:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:47:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:47:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.189 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:136398 12.b 12:d51f51df:::benchmark_data_smithi064_442287_object48348:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 457.98803044599998, 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1895044019999998, 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.190 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 136398 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.191 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193986+0000", 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193984+0000", 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.193990+0000", 2023-06-10T21:47:27.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7289999999999997e-06 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194000+0000", 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0328e-05 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194007+0000", 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5259999999999999e-06 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194082+0000", 2023-06-10T21:47:27.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5372999999999998e-05 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194250+0000", 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000168278 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.194419+0000", 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016897799999999999 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.200597+0000", 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061777100000000003 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.206684+0000", 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060865779999999996 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.219819+0000", 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013135413 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:49.219832+0000", 2023-06-10T21:47:27.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3101e-05 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.216190+0000", 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99635775400000004 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:50.216208+0000", 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.8501000000000001e-05 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.383417+0000", 2023-06-10T21:47:27.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.16720892 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:39:51.383490+0000", 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2949999999999998e-05 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.202 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156294 12.4 12:2623a45a:::benchmark_data_smithi064_442287_object68244:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.214882+0000", 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.967133709, 2023-06-10T21:47:27.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96337366700000004, 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156294 2023-06-10T21:47:27.204 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214882+0000", 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214882+0000", 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214881+0000", 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214887+0000", 2023-06-10T21:47:27.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1630000000000001e-06 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214900+0000", 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2856e-05 2023-06-10T21:47:27.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214910+0000", 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0079e-05 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214965+0000", 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5461000000000003e-05 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214975+0000", 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0010000000000001e-05 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222083+0000", 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071072549999999998 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222094+0000", 2023-06-10T21:47:27.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1243000000000001e-05 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.178200+0000", 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95610619399999996 2023-06-10T21:47:27.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.178256+0000", 2023-06-10T21:47:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.609e-05 2023-06-10T21:47:27.214 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156295 12.14 12:2e87f5af:::benchmark_data_smithi064_442287_object68245:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:47:27.215 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.214924+0000", 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.96709222499999, 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96297794400000003, 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.216 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156295 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214924+0000", 2023-06-10T21:47:27.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214924+0000", 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.218 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214924+0000", 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214927+0000", 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3110000000000001e-06 2023-06-10T21:47:27.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214934+0000", 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1679999999999998e-06 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214941+0000", 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5220000000000004e-06 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215022+0000", 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1446999999999995e-05 2023-06-10T21:47:27.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215028+0000", 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.964e-06 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222116+0000", 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070882719999999996 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222125+0000", 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2830000000000001e-06 2023-06-10T21:47:27.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.177818+0000", 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95569378900000002 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.177902+0000", 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3609000000000005e-05 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.225 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156299 12.2 12:4770460a:::benchmark_data_smithi064_442287_object68249:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.96707076199999, 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.193165665, 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.226 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156299 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214947+0000", 2023-06-10T21:47:27.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3539999999999998e-06 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214953+0000", 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6590000000000004e-06 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214959+0000", 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1920000000000002e-06 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.231 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215018+0000", 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9209000000000002e-05 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215029+0000", 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-06-10T21:47:27.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222815+0000", 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077857710000000004 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222824+0000", 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2979999999999997e-06 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.391187+0000", 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1683627429999999 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.392188+0000", 2023-06-10T21:47:27.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010011899999999999 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408024+0000", 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015836071 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408111+0000", 2023-06-10T21:47:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7112000000000002e-05 2023-06-10T21:47:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.237 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156302 12.8 12:11489672:::benchmark_data_smithi064_442287_object68252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.214970+0000", 2023-06-10T21:47:27.238 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.96704575400003, 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0003238759999999, 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.239 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156302 2023-06-10T21:47:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214970+0000", 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214970+0000", 2023-06-10T21:47:27.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214970+0000", 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214973+0000", 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6769999999999999e-06 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214979+0000", 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.947e-06 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214984+0000", 2023-06-10T21:47:27.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2530000000000004e-06 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.224192+0000", 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092076380000000006 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.245 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.224201+0000", 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1710000000000008e-06 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.187309+0000", 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96310821999999996 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.187461+0000", 2023-06-10T21:47:27.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015176499999999999 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.187830+0000", 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000369133 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.193827+0000", 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059974570000000003 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.193837+0000", 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4029999999999996e-06 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.206478+0000", 2023-06-10T21:47:27.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012641536 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.215140+0000", 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086619790000000002 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.215181+0000", 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1109999999999998e-05 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.215294+0000", 2023-06-10T21:47:27.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112967 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156309 12.18 12:1fb29d12:::benchmark_data_smithi064_442287_object68259:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.215036+0000", 2023-06-10T21:47:27.253 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.96697992700001, 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0000620790000001, 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.254 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156309 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215036+0000", 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215036+0000", 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215036+0000", 2023-06-10T21:47:27.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215038+0000", 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4499999999999998e-06 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.257 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215043+0000", 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6460000000000001e-06 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215049+0000", 2023-06-10T21:47:27.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5260000000000001e-06 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.224239+0000", 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091903269999999999 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.224252+0000", 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2796000000000001e-05 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.189847+0000", 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96559575499999994 2023-06-10T21:47:27.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.189964+0000", 2023-06-10T21:47:27.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011697999999999999 2023-06-10T21:47:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:47:27.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.190296+0000", 2023-06-10T21:47:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033121099999999998 2023-06-10T21:47:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.193845+0000", 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035490000000000001 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.201491+0000", 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076463759999999999 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.208311+0000", 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0068202879999999999 2023-06-10T21:47:27.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.214917+0000", 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066059350000000003 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.214973+0000", 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5596000000000001e-05 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.215098+0000", 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012553 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156315 12.12 12:486ff777:::benchmark_data_smithi064_442287_object68265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:47:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:47:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.777456035, 2023-06-10T21:47:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0549169319999998, 2023-06-10T21:47:27.269 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156315 2023-06-10T21:47:27.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:47:27.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:47:27.272 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404558+0000", 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:27.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404567+0000", 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3510000000000002e-06 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404580+0000", 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3585e-05 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404597+0000", 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6884999999999998e-05 2023-06-10T21:47:27.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404651+0000", 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4268000000000003e-05 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404663+0000", 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1511999999999999e-05 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408138+0000", 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0034749550000002 2023-06-10T21:47:27.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430486+0000", 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022347895 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440618+0000", 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010132474000000001 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440713+0000", 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4409999999999999e-05 2023-06-10T21:47:27.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.441046+0000", 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033325899999999997 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.445516+0000", 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044697950000000004 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457707+0000", 2023-06-10T21:47:27.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012191306000000001 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457796+0000", 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8623999999999997e-05 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.458924+0000", 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011283 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.458960+0000", 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5641e-05 2023-06-10T21:47:27.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459477+0000", 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00051747900000000005 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156317 12.12 12:4887ecdb:::benchmark_data_smithi064_442287_object68267:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.77224375600002, 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0511120819999999, 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156317 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409771+0000", 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:27.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409779+0000", 2023-06-10T21:47:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4189999999999996e-06 2023-06-10T21:47:27.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409792+0000", 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3642000000000001e-05 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409801+0000", 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1789999999999997e-06 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409855+0000", 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3746999999999999e-05 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409865+0000", 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0232e-05 2023-06-10T21:47:27.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430525+0000", 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0206594729999998 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.432926+0000", 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002401481 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.448701+0000", 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015774939000000002 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.448802+0000", 2023-06-10T21:47:27.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100634 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.449172+0000", 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037020999999999999 2023-06-10T21:47:27.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451840+0000", 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002668241 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459503+0000", 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076622160000000003 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.298 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459525+0000", 2023-06-10T21:47:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2195e-05 2023-06-10T21:47:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.299 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460331+0000", 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080666600000000005 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460406+0000", 2023-06-10T21:47:27.300 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4512999999999996e-05 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460885+0000", 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047858500000000001 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.301 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156321 12.12 12:4965cc64:::benchmark_data_smithi064_442287_object68271:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.70843313799998, 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.989335101, 2023-06-10T21:47:27.302 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156321 2023-06-10T21:47:27.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473581+0000", 2023-06-10T21:47:27.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473589+0000", 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4819999999999997e-06 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473603+0000", 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4447000000000001e-05 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473614+0000", 2023-06-10T21:47:27.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1249e-05 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473664+0000", 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9842999999999997e-05 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473675+0000", 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0786e-05 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.432987+0000", 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.959312341 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433685+0000", 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069741300000000003 2023-06-10T21:47:27.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451862+0000", 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018176859 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451922+0000", 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0197e-05 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.452223+0000", 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030085300000000001 2023-06-10T21:47:27.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457715+0000", 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054923799999999998 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459543+0000", 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018279150000000001 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459560+0000", 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7337999999999999e-05 2023-06-10T21:47:27.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462361+0000", 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002800641 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.315 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462413+0000", 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1984000000000002e-05 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462918+0000", 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050521099999999998 2023-06-10T21:47:27.316 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156335 12.a e360/358 12:540f4c4f:::benchmark_data_smithi064_442287_object68285:head v 360'9865, mlcod=0'0)", 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.96096996199998, 2023-06-10T21:47:27.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1858911519999999, 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221045+0000", 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221053+0000", 2023-06-10T21:47:27.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7559999999999999e-06 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221064+0000", 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0668e-05 2023-06-10T21:47:27.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221072+0000", 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3250000000000008e-06 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.401905+0000", 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.180832348 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.401927+0000", 2023-06-10T21:47:27.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2634e-05 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406910+0000", 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049828210000000001 2023-06-10T21:47:27.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406937+0000", 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7396e-05 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.325 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156351 12.a e360/358 12:57b5438e:::benchmark_data_smithi064_442287_object68301:head v 360'9866, mlcod=0'0)", 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.89968002900002, 2023-06-10T21:47:27.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.124617532, 2023-06-10T21:47:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.328 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:47:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.329 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282335+0000", 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282343+0000", 2023-06-10T21:47:27.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6960000000000005e-06 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287460+0000", 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051169090000000002 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287467+0000", 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.272e-06 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.403319+0000", 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.115852015 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.403339+0000", 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9918000000000001e-05 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406942+0000", 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003602821 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406954+0000", 2023-06-10T21:47:27.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1768e-05 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156352 12.a e360/358 12:57f76687:::benchmark_data_smithi064_442287_object68302:head v 360'9867, mlcod=0'0)", 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:47:27.335 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.89452671800001, 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.143004788, 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287496+0000", 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7859999999999999e-06 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287504+0000", 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9799999999999998e-06 2023-06-10T21:47:27.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287509+0000", 2023-06-10T21:47:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4240000000000001e-06 2023-06-10T21:47:27.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.404422+0000", 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1169133019999999 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.341 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.404444+0000", 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1957e-05 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430421+0000", 2023-06-10T21:47:27.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025976448999999999 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430494+0000", 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3517999999999996e-05 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.343 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156358 12.a e360/358 12:5a95b6f1:::benchmark_data_smithi064_442287_object68308:head v 360'9868, mlcod=0'0)", 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.87856449600002, 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.127077511, 2023-06-10T21:47:27.344 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303451+0000", 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303458+0000", 2023-06-10T21:47:27.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1550000000000001e-06 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303471+0000", 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3037000000000001e-05 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303479+0000", 2023-06-10T21:47:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8900000000000007e-06 2023-06-10T21:47:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.405165+0000", 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.101685818 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.405186+0000", 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0908999999999999e-05 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430498+0000", 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02531286 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430529+0000", 2023-06-10T21:47:27.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0797999999999999e-05 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156385 12.2 12:42ca3731:::benchmark_data_smithi064_442287_object68335:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-06-10T21:47:27.353 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.427492+0000", 2023-06-10T21:47:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.75452430500002, 2023-06-10T21:47:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0207177439999999, 2023-06-10T21:47:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.354 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156385 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.355 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427492+0000", 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427492+0000", 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427491+0000", 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427497+0000", 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3300000000000004e-06 2023-06-10T21:47:27.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427508+0000", 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0974e-05 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.427516+0000", 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8949999999999998e-06 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.432972+0000", 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0054560859999999 2023-06-10T21:47:27.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433110+0000", 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013768600000000001 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433522+0000", 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041261500000000002 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.439438+0000", 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059155379999999997 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440552+0000", 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011138649999999999 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.443476+0000", 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029242320000000001 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.447406+0000", 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039299679999999998 2023-06-10T21:47:27.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.447464+0000", 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8369999999999998e-05 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.448210+0000", 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000745468 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.366 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156393 12.a e360/358 12:5b39e110:::benchmark_data_smithi064_442287_object68343:head v 360'9869, mlcod=0'0)", 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.488256+0000", 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.693759817, 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94548128899999995, 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488256+0000", 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488256+0000", 2023-06-10T21:47:27.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488255+0000", 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.369 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488263+0000", 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1409999999999999e-06 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488272+0000", 2023-06-10T21:47:27.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0459999999999994e-06 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.488278+0000", 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3080000000000004e-06 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406160+0000", 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91788199299999995 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406174+0000", 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3614e-05 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433693+0000", 2023-06-10T21:47:27.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.027519660000000001 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433738+0000", 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4471999999999997e-05 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.374 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156399 12.12 12:486b9e0e:::benchmark_data_smithi064_442287_object68349:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.499052+0000", 2023-06-10T21:47:27.375 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.68296445499999, 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96438187600000003, 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.376 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156399 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499052+0000", 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499052+0000", 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499051+0000", 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499056+0000", 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5620000000000003e-06 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499066+0000", 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3659999999999998e-06 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.499071+0000", 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7320000000000003e-06 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433826+0000", 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93475425199999995 2023-06-10T21:47:27.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.434563+0000", 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00073788800000000002 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.382 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.454831+0000", 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020267072000000001 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.454933+0000", 2023-06-10T21:47:27.383 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102414 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.455308+0000", 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037465699999999997 2023-06-10T21:47:27.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457721+0000", 2023-06-10T21:47:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024133739999999998 2023-06-10T21:47:27.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459578+0000", 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018570559999999999 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459601+0000", 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3152000000000001e-05 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462999+0000", 2023-06-10T21:47:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033981129999999999 2023-06-10T21:47:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.463057+0000", 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7683000000000003e-05 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.463434+0000", 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037664300000000002 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156407 12.2 12:461b8517:::benchmark_data_smithi064_442287_object68357:head [delete] snapc 0=[] ondisk+write+known_if_redirected e360)", 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.516085+0000", 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 415.665930715, 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94157304500000005, 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.391 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156407 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516085+0000", 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516085+0000", 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516084+0000", 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516090+0000", 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.93e-06 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516100+0000", 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8630000000000003e-06 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.395 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.516106+0000", 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.942e-06 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.396 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.435348+0000", 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91924222499999997 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.435498+0000", 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000149381 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.397 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.435884+0000", 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038616900000000001 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.398 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440589+0000", 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047053640000000001 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.441484+0000", 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00089458099999999998 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.399 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.443483+0000", 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019990989999999998 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.456845+0000", 2023-06-10T21:47:27.400 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013362634999999999 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.456901+0000", 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5299e-05 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.401 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457659+0000", 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00075786099999999997 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.402 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106691 13.3 13:cd7d9036:::benchmark_data_smithi064_501823_object106690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 266.66191734699999, 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.983989778, 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.403 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:47:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:47:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106691 2023-06-10T21:47:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.404 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.405 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520098+0000", 2023-06-10T21:47:27.406 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520145+0000", 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6650999999999998e-05 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.407 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520153+0000", 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6559999999999992e-06 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.408 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520159+0000", 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4359999999999999e-06 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.409 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520202+0000", 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3389000000000001e-05 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520324+0000", 2023-06-10T21:47:27.410 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121074 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520472+0000", 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148715 2023-06-10T21:47:27.411 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.526115+0000", 2023-06-10T21:47:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056431629999999997 2023-06-10T21:47:27.412 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.413 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547106+0000", 2023-06-10T21:47:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020990175 2023-06-10T21:47:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.414 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547117+0000", 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.182e-05 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.415 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189400+0000", 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64228294200000002 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189417+0000", 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6770999999999999e-05 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.416 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.503987+0000", 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314569638 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.417 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.504089+0000", 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101869 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106703 13.3 13:caa34abe:::benchmark_data_smithi064_501823_object106702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:47:27.418 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 266.64283266500001, 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.981275578, 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:47:27.419 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106703 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.420 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.421 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539182+0000", 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539209+0000", 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6724999999999999e-05 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.422 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542176+0000", 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002966991 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542190+0000", 2023-06-10T21:47:27.423 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4246999999999999e-05 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547147+0000", 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049566059999999997 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.424 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547155+0000", 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3969999999999994e-06 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189447+0000", 2023-06-10T21:47:27.425 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64229122500000002 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189456+0000", 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4609999999999997e-06 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.426 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505480+0000", 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.316023876 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505680+0000", 2023-06-10T21:47:27.427 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000200366 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505915+0000", 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000234653 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.428 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.508394+0000", 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024786389999999999 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.517040+0000", 2023-06-10T21:47:27.429 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086467249999999992 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.518965+0000", 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001924469 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.430 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519013+0000", 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000002e-05 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.431 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.520459+0000", 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001446073 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.432 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106693 13.2 13:481d9832:::benchmark_data_smithi064_501823_object106692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e391)", 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 265.99111807200001, 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3284778340000001, 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:47:27.433 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106693 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.434 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:47:27.435 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190896+0000", 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:47:27.436 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190926+0000", 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9950999999999998e-05 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:47:27.437 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190941+0000", 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5157000000000001e-05 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190954+0000", 2023-06-10T21:47:27.438 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2877999999999999e-05 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193380+0000", 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024257100000000002 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.439 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193393+0000", 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2578000000000001e-05 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:47:27.440 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519272+0000", 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325879748 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519376+0000", 2023-06-10T21:47:27.441 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010365300000000001 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:47:27.442 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:47:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:27.444 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:27.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:27.445 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:27 smithi064 ceph-mon[186706]: pgmap v1624: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 34 MiB/s wr, 620 op/s; 72704/318777 objects misplaced (22.807%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-06-10T21:47:27.445 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:27 smithi064 ceph-mon[180297]: pgmap v1624: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 34 MiB/s wr, 620 op/s; 72704/318777 objects misplaced (22.807%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-06-10T21:47:27.546 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:27.572 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:27 smithi152 ceph-mon[156819]: pgmap v1624: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 34 MiB/s wr, 620 op/s; 72704/318777 objects misplaced (22.807%); 1.2 MiB/s, 1 keys/s, 20 objects/s recovering 2023-06-10T21:47:27.575 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:27.640 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:27.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:27.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:27.749 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:27.840 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:28.598 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:28 smithi064 ceph-mon[180297]: pgmap v1625: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 34 MiB/s wr, 716 op/s; 72483/319539 objects misplaced (22.684%); 4.7 MiB/s, 73 objects/s recovering 2023-06-10T21:47:28.598 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:28 smithi064 ceph-mon[186706]: pgmap v1625: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 34 MiB/s wr, 716 op/s; 72483/319539 objects misplaced (22.684%); 4.7 MiB/s, 73 objects/s recovering 2023-06-10T21:47:28.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:47:28.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:28 smithi152 ceph-mon[156819]: pgmap v1625: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 34 MiB/s wr, 716 op/s; 72483/319539 objects misplaced (22.684%); 4.7 MiB/s, 73 objects/s recovering 2023-06-10T21:47:28.899 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:28.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:28.992 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:47:29.021 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:29.083 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:29.156 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:29.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:29.265 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:29.349 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:29.482 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:47:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:30 smithi064 ceph-mon[186706]: pgmap v1626: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 103 B/s rd, 28 MiB/s wr, 627 op/s; 72483/318060 objects misplaced (22.789%); 3.8 MiB/s, 59 objects/s recovering 2023-06-10T21:47:30.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:30 smithi064 ceph-mon[180297]: pgmap v1626: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 103 B/s rd, 28 MiB/s wr, 627 op/s; 72483/318060 objects misplaced (22.789%); 3.8 MiB/s, 59 objects/s recovering 2023-06-10T21:47:30.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:30 smithi152 ceph-mon[156819]: pgmap v1626: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.5 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 103 B/s rd, 28 MiB/s wr, 627 op/s; 72483/318060 objects misplaced (22.789%); 3.8 MiB/s, 59 objects/s recovering 2023-06-10T21:47:30.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:30.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:30.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:30.913 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:31.044 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:31.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:31.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:31.160 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:31.202 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:31.370 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:31.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:31.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:31.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:31.518 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:31.588 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:31.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:31.603 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:47:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:47:31.604 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113067 14.f e442/420 14:f371ebf2:::benchmark_data_smithi064_559916_object5211:head v 442'7115, mlcod=442'7113)", 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.591028+0000", 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0056495340000000003, 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056788860000000002, 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.605 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591028+0000", 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.606 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591028+0000", 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591027+0000", 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3650000000000004e-06 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591035+0000", 2023-06-10T21:47:31.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2114e-05 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591048+0000", 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.446e-06 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591053+0000", 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2284999999999998e-05 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591105+0000", 2023-06-10T21:47:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6761e-05 2023-06-10T21:47:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591132+0000", 2023-06-10T21:47:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103865 2023-06-10T21:47:31.612 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.613 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113069 14.a e442/426 14:5151fbcc:::benchmark_data_smithi064_559916_object5213:head v 442'7074, mlcod=442'7072)", 2023-06-10T21:47:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.591390+0000", 2023-06-10T21:47:31.614 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0052876140000000004, 2023-06-10T21:47:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053876180000000003, 2023-06-10T21:47:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.615 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591390+0000", 2023-06-10T21:47:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591390+0000", 2023-06-10T21:47:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591390+0000", 2023-06-10T21:47:31.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3339999999999998e-06 2023-06-10T21:47:31.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591396+0000", 2023-06-10T21:47:31.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0376e-05 2023-06-10T21:47:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591406+0000", 2023-06-10T21:47:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2679999999999996e-06 2023-06-10T21:47:31.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591411+0000", 2023-06-10T21:47:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0817999999999999e-05 2023-06-10T21:47:31.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591471+0000", 2023-06-10T21:47:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0013999999999998e-05 2023-06-10T21:47:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.591491+0000", 2023-06-10T21:47:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101272 2023-06-10T21:47:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.632 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113070 14.a e442/426 14:55fcae30:::benchmark_data_smithi064_559916_object5214:head v 442'7075, mlcod=442'7072)", 2023-06-10T21:47:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.593555+0000", 2023-06-10T21:47:31.634 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0031228129999999999, 2023-06-10T21:47:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0032859970000000001, 2023-06-10T21:47:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.635 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593555+0000", 2023-06-10T21:47:31.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593555+0000", 2023-06-10T21:47:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:47:31.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593554+0000", 2023-06-10T21:47:31.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3869999999999995e-06 2023-06-10T21:47:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593562+0000", 2023-06-10T21:47:31.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0936e-05 2023-06-10T21:47:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593573+0000", 2023-06-10T21:47:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3589999999999996e-06 2023-06-10T21:47:31.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.647 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593580+0000", 2023-06-10T21:47:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5281999999999998e-05 2023-06-10T21:47:31.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593625+0000", 2023-06-10T21:47:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6838999999999999e-05 2023-06-10T21:47:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.593642+0000", 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6835999999999993e-05 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113064 14.8 e442/439 14:1e9d40df:::benchmark_data_smithi064_559916_object5208:head v 442'7075, mlcod=442'7071)", 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.594101+0000", 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0025771140000000001, 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002804034, 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.653 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594101+0000", 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594101+0000", 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594100+0000", 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5899999999999996e-06 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594107+0000", 2023-06-10T21:47:31.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2439999999999999e-06 2023-06-10T21:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594116+0000", 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4029999999999998e-06 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594121+0000", 2023-06-10T21:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0010000000000003e-05 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594181+0000", 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.0091e-05 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.594211+0000", 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011066899999999999 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113072 14.4 e442/420 14:2caf9282:::benchmark_data_smithi064_559916_object5216:head v 442'6991, mlcod=442'6989)", 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.595125+0000", 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.0015523220000000001, 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0018423199999999999, 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.661 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595125+0000", 2023-06-10T21:47:31.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595125+0000", 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595124+0000", 2023-06-10T21:47:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4699999999999996e-06 2023-06-10T21:47:31.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595132+0000", 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2738000000000002e-05 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595155+0000", 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3620000000000002e-06 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595161+0000", 2023-06-10T21:47:31.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4610000000000001e-05 2023-06-10T21:47:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595206+0000", 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9139000000000001e-05 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595225+0000", 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9359999999999997e-05 2023-06-10T21:47:31.668 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113068 14.7 e442/439 14:ecc87202:::benchmark_data_smithi064_559916_object5212:head v 442'7017, mlcod=442'7014)", 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.595656+0000", 2023-06-10T21:47:31.669 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.001021931, 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001380928, 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595656+0000", 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595656+0000", 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:31.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595655+0000", 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7579999999999992e-06 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595663+0000", 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1935e-05 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595675+0000", 2023-06-10T21:47:31.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2590000000000002e-06 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595679+0000", 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.066e-05 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.674 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595720+0000", 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2416999999999999e-05 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595742+0000", 2023-06-10T21:47:31.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6351e-05 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:113071 14.d e442/426 14:b2d56779:::benchmark_data_smithi064_559916_object5215:head v 442'7090, mlcod=442'7088)", 2023-06-10T21:47:31.676 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:47:31.595691+0000", 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 0.00098697499999999996, 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001406753, 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595691+0000", 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595691+0000", 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:47:31.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595690+0000", 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0090000000000001e-06 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595694+0000", 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6590000000000004e-06 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:47:31.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595700+0000", 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8999999999999999e-06 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595704+0000", 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8517999999999998e-05 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595763+0000", 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8e-05 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:47:31.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:47:31.595781+0000", 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9772999999999996e-05 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: ], 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 7 2023-06-10T21:47:31.683 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:47:31.697 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:31.739 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:31.817 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:31.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:31.922 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:31.970 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:32.205 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:32.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:32.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:32.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:32.335 INFO:teuthology.orchestra.run.smithi064.stdout:moved 38 / 267 (14.2322%) 2023-06-10T21:47:32.336 INFO:teuthology.orchestra.run.smithi064.stdout:avg 53.4 2023-06-10T21:47:32.336 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 6.3435 -> 7.46994 (expected baseline 6.53605) 2023-06-10T21:47:32.336 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.2 with 43 -> 48 pgs (0.805243 -> 0.898876 * mean) 2023-06-10T21:47:32.336 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.3 with 59 -> 53 pgs (1.10487 -> 0.992509 * mean) 2023-06-10T21:47:32.336 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 1 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0516 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0568 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 1.0000 -> 0.8560 2023-06-10T21:47:32.337 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 1.0000 -> 0.8053 2023-06-10T21:47:32.338 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 1.0000 -> 0.9007 2023-06-10T21:47:32.338 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 1.0000 -> 0.7307 2023-06-10T21:47:32.351 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:47:32.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:47:32.421 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:32.476 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:32.478 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:32.478 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[180297]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:32.478 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]: dispatch 2023-06-10T21:47:32.478 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[180297]: pgmap v1627: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 93 B/s rd, 20 MiB/s wr, 753 op/s; 69639/309192 objects misplaced (22.523%); 6.7 MiB/s, 106 objects/s recovering 2023-06-10T21:47:32.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[186706]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]: dispatch 2023-06-10T21:47:32.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:32 smithi064 ceph-mon[186706]: pgmap v1627: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 93 B/s rd, 20 MiB/s wr, 753 op/s; 69639/309192 objects misplaced (22.523%); 6.7 MiB/s, 106 objects/s recovering 2023-06-10T21:47:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:32.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:32.581 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:32.611 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:32.664 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:32.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:32 smithi152 ceph-mon[156819]: from='client.57046 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:32 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]: dispatch 2023-06-10T21:47:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:32 smithi152 ceph-mon[156819]: pgmap v1627: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 93 B/s rd, 20 MiB/s wr, 753 op/s; 69639/309192 objects misplaced (22.523%); 6.7 MiB/s, 106 objects/s recovering 2023-06-10T21:47:32.748 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:47:32.770 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:32.799 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:32] "GET /metrics HTTP/1.1" 200 224099 "" "Prometheus/2.33.4" 2023-06-10T21:47:33.635 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]': finished 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: osdmap e443: 8 total, 8 up, 5 in 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:33.636 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]': finished 2023-06-10T21:47:33.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: osdmap e443: 8 total, 8 up, 5 in 2023-06-10T21:47:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:33.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2570304797' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'59027','3':'52779','5':'47890','7':'56097'}"}]': finished 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: osdmap e443: 8 total, 8 up, 5 in 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: failed to encode map e443 with expected crc 2023-06-10T21:47:33.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:33.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:34.265 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:47:34.265 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:47:34.266 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:47:34.266 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:47:34.266 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:47:34.356 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:34.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:34.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:34.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:34.503 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:34.547 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: osdmap e444: 8 total, 8 up, 5 in 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.549 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: 14.3 continuing backfill to osd.0 from (425'2300,442'7070] MIN to 442'7070 2023-06-10T21:47:34.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: 14.c continuing backfill to osd.0 from (424'1300,442'7057] MIN to 442'7057 2023-06-10T21:47:34.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: 14.7 continuing backfill to osd.0 from (425'2200,442'7045] 14:e3376b7a:::benchmark_data_smithi064_559916_object60193:head to 442'7045 2023-06-10T21:47:34.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: 14.7 starting backfill to osd.3 from (0'0,0'0] MAX to 442'7045 2023-06-10T21:47:34.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:34.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:34 smithi152 ceph-mon[156819]: pgmap v1630: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 8.0 MiB/s wr, 656 op/s; 69639/309192 objects misplaced (22.523%); 8.1 MiB/s, 129 objects/s recovering 2023-06-10T21:47:34.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:34.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:34.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: osdmap e444: 8 total, 8 up, 5 in 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: 14.3 continuing backfill to osd.0 from (425'2300,442'7070] MIN to 442'7070 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: 14.c continuing backfill to osd.0 from (424'1300,442'7057] MIN to 442'7057 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: 14.7 continuing backfill to osd.0 from (425'2200,442'7045] 14:e3376b7a:::benchmark_data_smithi064_559916_object60193:head to 442'7045 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: 14.7 starting backfill to osd.3 from (0'0,0'0] MAX to 442'7045 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[180297]: pgmap v1630: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 8.0 MiB/s wr, 656 op/s; 69639/309192 objects misplaced (22.523%); 8.1 MiB/s, 129 objects/s recovering 2023-06-10T21:47:34.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: osdmap e444: 8 total, 8 up, 5 in 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: failed to encode map e444 with expected crc 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: 14.3 continuing backfill to osd.0 from (425'2300,442'7070] MIN to 442'7070 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: 14.c continuing backfill to osd.0 from (424'1300,442'7057] MIN to 442'7057 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: 14.7 continuing backfill to osd.0 from (425'2200,442'7045] 14:e3376b7a:::benchmark_data_smithi064_559916_object60193:head to 442'7045 2023-06-10T21:47:34.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: 14.7 starting backfill to osd.3 from (0'0,0'0] MAX to 442'7045 2023-06-10T21:47:34.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:34.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:34 smithi064 ceph-mon[186706]: pgmap v1630: 89 pgs: 1 active+remapped+backfilling, 10 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 127 B/s rd, 8.0 MiB/s wr, 656 op/s; 69639/309192 objects misplaced (22.523%); 8.1 MiB/s, 129 objects/s recovering 2023-06-10T21:47:34.683 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:34.763 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:34.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:34.868 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:34.940 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:35.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:47:35.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: failed to encode map e445 with expected crc 2023-06-10T21:47:35.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: osdmap e445: 8 total, 8 up, 5 in 2023-06-10T21:47:35.341 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: 14.2 continuing backfill to osd.0 from (425'2200,442'7057] MIN to 442'7057 2023-06-10T21:47:35.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: 14.d starting backfill to osd.0 from (0'0,0'0] MAX to 442'7109 2023-06-10T21:47:35.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: 14.1 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7209 2023-06-10T21:47:35.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: 14.4 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7021 2023-06-10T21:47:35.342 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[180297]: 14.e continuing backfill to osd.0 from (424'1500,442'7258] MIN to 442'7258 2023-06-10T21:47:35.342 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: failed to encode map e445 with expected crc 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: osdmap e445: 8 total, 8 up, 5 in 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: 14.2 continuing backfill to osd.0 from (425'2200,442'7057] MIN to 442'7057 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: 14.d starting backfill to osd.0 from (0'0,0'0] MAX to 442'7109 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: 14.1 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7209 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: 14.4 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7021 2023-06-10T21:47:35.343 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:35 smithi152 ceph-mon[156819]: 14.e continuing backfill to osd.0 from (424'1500,442'7258] MIN to 442'7258 2023-06-10T21:47:35.438 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-06-10T21:47:35.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight-by-utilization 1000 3.0 2023-06-10T21:47:35.490 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:35.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:35.534 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:47:35.601 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:35.636 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: failed to encode map e445 with expected crc 2023-06-10T21:47:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: osdmap e445: 8 total, 8 up, 5 in 2023-06-10T21:47:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: 14.2 continuing backfill to osd.0 from (425'2200,442'7057] MIN to 442'7057 2023-06-10T21:47:35.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: 14.d starting backfill to osd.0 from (0'0,0'0] MAX to 442'7109 2023-06-10T21:47:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: 14.1 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7209 2023-06-10T21:47:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: 14.4 starting backfill to osd.0 from (0'0,0'0] MAX to 442'7021 2023-06-10T21:47:35.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:35 smithi064 ceph-mon[186706]: 14.e continuing backfill to osd.0 from (424'1500,442'7258] MIN to 442'7258 2023-06-10T21:47:35.792 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:35.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:35.900 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:35.984 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:36.082 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:36.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:36.107 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:47:36.151 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:47:36.190 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:47:36.195 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:36.229 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:47:36.255 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:36.259 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:47:36.262 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:36.313 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:36.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:36.418 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:36.456 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:36.462 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:47:36.503 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:47:36.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3394175275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:36.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:36 smithi064 ceph-mon[186706]: pgmap v1632: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 10 active+remapped+backfill_wait, 76 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 75050/306420 objects misplaced (24.493%); 12 MiB/s, 198 objects/s recovering 2023-06-10T21:47:36.533 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3394175275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:36.534 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:36 smithi064 ceph-mon[180297]: pgmap v1632: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 10 active+remapped+backfill_wait, 76 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 75050/306420 objects misplaced (24.493%); 12 MiB/s, 198 objects/s recovering 2023-06-10T21:47:36.540 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:47:36.573 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.574 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.575 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:47:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3394175275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:36 smithi152 ceph-mon[156819]: pgmap v1632: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 10 active+remapped+backfill_wait, 76 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 646 op/s; 75050/306420 objects misplaced (24.493%); 12 MiB/s, 198 objects/s recovering 2023-06-10T21:47:36.739 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:36.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:36.846 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:36.885 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:37.044 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:37.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:37.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:37.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:37.199 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:37.270 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:47:37.270 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:47:37.278 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:37.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:37.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:37.386 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:37.465 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:37.494 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:47:38.113 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:47:38.351 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:38 smithi152 ceph-mon[156819]: pgmap v1633: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 92609/303942 objects misplaced (30.469%); 13 MiB/s, 216 objects/s recovering 2023-06-10T21:47:38.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:38 smithi064 ceph-mon[180297]: pgmap v1633: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 92609/303942 objects misplaced (30.469%); 13 MiB/s, 216 objects/s recovering 2023-06-10T21:47:38.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:38 smithi064 ceph-mon[186706]: pgmap v1633: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 23 GiB used, 424 GiB / 447 GiB avail; 0 B/s wr, 291 op/s; 92609/303942 objects misplaced (30.469%); 13 MiB/s, 216 objects/s recovering 2023-06-10T21:47:38.605 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:38.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:38.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:38.765 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:38.829 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:38.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:38.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:38.936 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:38.973 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:39.040 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:39.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:39.152 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:39.182 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:39.232 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:39.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:39.251 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:39.307 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:39.337 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:39.369 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:39.450 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:39.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:39.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:39.585 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:39.649 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:39.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:39.760 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:39.837 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:40.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:40.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:40.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:40.237 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 267 (0%) 2023-06-10T21:47:40.237 INFO:teuthology.orchestra.run.smithi064.stdout:avg 53.4 2023-06-10T21:47:40.237 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 7.46994 -> 7.46994 (expected baseline 6.53605) 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 44 -> 44 pgs (0.82397 -> 0.82397 * mean) 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.0 with 66 -> 66 pgs (1.23595 -> 1.23595 * mean) 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-06-10T21:47:40.238 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0505 2023-06-10T21:47:40.239 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.5049 2023-06-10T21:47:40.262 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-06-10T21:47:40.282 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:40.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:40 smithi064 ceph-mon[186706]: from='client.57058 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:40.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:40 smithi064 ceph-mon[186706]: pgmap v1634: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 98746/302805 objects misplaced (32.610%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:47:40.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:40 smithi064 ceph-mon[180297]: from='client.57058 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:40.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:40 smithi064 ceph-mon[180297]: pgmap v1634: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 98746/302805 objects misplaced (32.610%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:47:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:40 smithi152 ceph-mon[156819]: from='client.57058 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:40 smithi152 ceph-mon[156819]: pgmap v1634: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 268 op/s; 98746/302805 objects misplaced (32.610%); 10 MiB/s, 163 objects/s recovering 2023-06-10T21:47:41.069 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:47:42.599 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:42 smithi064 ceph-mon[180297]: pgmap v1635: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 95428/294441 objects misplaced (32.410%); 12 MiB/s, 192 objects/s recovering 2023-06-10T21:47:42.600 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:42 smithi064 ceph-mon[186706]: pgmap v1635: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 95428/294441 objects misplaced (32.410%); 12 MiB/s, 192 objects/s recovering 2023-06-10T21:47:42.642 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-06-10T21:47:42.642 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd test-reweight-by-utilization 110 0.05 2023-06-10T21:47:42.660 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:42.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:42 smithi152 ceph-mon[156819]: pgmap v1635: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 549 op/s; 95428/294441 objects misplaced (32.410%); 12 MiB/s, 192 objects/s recovering 2023-06-10T21:47:42.773 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:42.862 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:43.159 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:42] "GET /metrics HTTP/1.1" 200 224247 "" "Prometheus/2.33.4" 2023-06-10T21:47:43.293 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:43.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:43.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:43.500 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:43.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:43.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:43.752 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:43.788 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:43.851 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:43.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:43.944 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:43.963 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:44.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:44.106 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:47:44.106 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:47:44.729 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:44.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:44.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:44.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:44.933 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:45.091 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:45 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:45.092 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:45 smithi064 ceph-mon[180297]: pgmap v1636: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 95428/294441 objects misplaced (32.410%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:47:45.406 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:45 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:45.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:45 smithi064 ceph-mon[186706]: pgmap v1636: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 95428/294441 objects misplaced (32.410%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:47:45.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:45 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:47:45.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:45 smithi152 ceph-mon[156819]: pgmap v1636: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 491 op/s; 95428/294441 objects misplaced (32.410%); 11 MiB/s, 171 objects/s recovering 2023-06-10T21:47:45.936 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:45.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:46.043 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:46.117 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:46.128 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:47:46.143 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:46.159 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:47:46.231 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:47:46.232 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:47:46.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:47:47.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:47.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:47:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: osdmap e446: 8 total, 8 up, 5 in 2023-06-10T21:47:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[186706]: pgmap v1638: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.9 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 93854/289575 objects misplaced (32.411%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:47:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:47:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: osdmap e446: 8 total, 8 up, 5 in 2023-06-10T21:47:47.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:47 smithi064 ceph-mon[180297]: pgmap v1638: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.9 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 93854/289575 objects misplaced (32.411%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:47:47.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:47:47.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2833905154' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: osdmap e446: 8 total, 8 up, 5 in 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: failed to encode map e446 with expected crc 2023-06-10T21:47:47.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:47 smithi152 ceph-mon[156819]: pgmap v1638: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.9 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 0 B/s wr, 561 op/s; 93854/289575 objects misplaced (32.411%); 9.4 MiB/s, 150 objects/s recovering 2023-06-10T21:47:47.691 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:47.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:47.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:47.800 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:47.839 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:47.917 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:47.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:48.022 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:48.055 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:48.122 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:48.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:48.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:48.235 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:48.374 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:48.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:48.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:48.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:47:48.880 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:48.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:48.985 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:49.044 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:47:49.088 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:49.099 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:47:49.132 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:47:49.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:47:49.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:47:49.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:49.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:49.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:49.277 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:47:49.378 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:49.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:49.529 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:47:49.529 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.530 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.531 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:47:49.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:49 smithi064 ceph-mon[180297]: pgmap v1639: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 641 op/s; 91892/284691 objects misplaced (32.278%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:49.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2764051213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:49 smithi064 ceph-mon[186706]: pgmap v1639: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 641 op/s; 91892/284691 objects misplaced (32.278%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2764051213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:49.682 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:49.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:49 smithi152 ceph-mon[156819]: pgmap v1639: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 22 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 641 op/s; 91892/284691 objects misplaced (32.278%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2764051213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:47:49.792 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:49.827 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:49.901 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:49.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:50.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:50.041 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:50.109 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:50.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:50.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:50.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:50.290 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:47:50.310 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:50.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:50.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:50.416 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:50.442 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:50.483 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:50.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:50.591 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:47:50.629 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:50.697 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:47:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:50.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:47:50.804 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:50.836 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:50.886 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:50.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:50.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:50.998 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:51.033 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:51.059 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:51.104 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:51.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:51.212 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:51.305 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:51.346 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:51 smithi064 ceph-mon[180297]: pgmap v1640: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 644 op/s; 91674/283467 objects misplaced (32.340%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:51.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:51 smithi064 ceph-mon[186706]: pgmap v1640: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 644 op/s; 91674/283467 objects misplaced (32.340%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:51.531 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:51.649 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:51.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:51.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:51 smithi152 ceph-mon[156819]: pgmap v1640: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 644 op/s; 91674/283467 objects misplaced (32.340%); 5.1 MiB/s, 81 objects/s recovering 2023-06-10T21:47:51.765 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:51.798 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:51.854 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:51.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:51.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:51.993 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:52.051 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:52.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:52.110 INFO:teuthology.orchestra.run.smithi064.stdout:moved 12 / 267 (4.49438%) 2023-06-10T21:47:52.111 INFO:teuthology.orchestra.run.smithi064.stdout:avg 53.4 2023-06-10T21:47:52.111 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 7.46994 -> 9.50999 (expected baseline 6.53605) 2023-06-10T21:47:52.111 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 44 -> 41 pgs (0.82397 -> 0.76779 * mean) 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.0 with 66 -> 69 pgs (1.23595 -> 1.29213 * mean) 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-06-10T21:47:52.112 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0475 2023-06-10T21:47:52.113 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0523 2023-06-10T21:47:52.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.8560 -> 0.8060 2023-06-10T21:47:52.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 0.8053 -> 0.7554 2023-06-10T21:47:52.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2 weight 0.9007 -> 0.8507 2023-06-10T21:47:52.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.7307 -> 0.6808 2023-06-10T21:47:52.138 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-06-10T21:47:52.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:52.193 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:52.244 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:52.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:52.327 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:52 smithi064 conmon[171866]: 2023-06-10T21:47:52.097+0000 7f6cfc057700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-06-10T21:47:52.353 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:52.444 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:52.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:52 smithi152 ceph-mon[156819]: from='client.57073 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:52.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:52 smithi152 ceph-mon[156819]: pgmap v1641: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:52 smithi064 ceph-mon[186706]: from='client.57073 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:52.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:52 smithi064 ceph-mon[186706]: pgmap v1641: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:52.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:52 smithi064 ceph-mon[180297]: from='client.57073 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:47:52.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:52 smithi064 ceph-mon[180297]: pgmap v1641: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:53.099 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:47:53.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:47:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:47:52] "GET /metrics HTTP/1.1" 200 224256 "" "Prometheus/2.33.4" 2023-06-10T21:47:54.507 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:54.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:54.619 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:54.651 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:54 smithi152 ceph-mon[156819]: pgmap v1642: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:54.654 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:54.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:54 smithi064 ceph-mon[180297]: pgmap v1642: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:54.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:54 smithi064 ceph-mon[186706]: pgmap v1642: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 627 op/s; 88514/275613 objects misplaced (32.115%); 5.4 MiB/s, 85 objects/s recovering 2023-06-10T21:47:54.724 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:54.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:54.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:54.951 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:55.141 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:55.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:55.175 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-06-10T21:47:55.175 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd test-reweight-by-utilization 110 0.05 2023-06-10T21:47:55.249 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:55.281 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:55.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:55.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:55.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:55.596 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:55.803 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:55.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:55.827 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:55.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:47:55.926 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:47:55.927 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:47:55.927 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:47:55.927 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:47:55.927 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:47:55.945 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:55.995 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:47:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:56.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:47:56.101 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:56.191 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:56.327 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:56.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:56.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:56.433 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:56.497 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:56.583 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[180297]: pgmap v1643: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 609 op/s; 88346/274659 objects misplaced (32.166%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:47:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/839155655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.584 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.584 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[186706]: pgmap v1643: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 609 op/s; 88346/274659 objects misplaced (32.166%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:47:56.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/839155655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.585 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:56 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:56.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:56.687 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:47:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:56 smithi152 ceph-mon[156819]: pgmap v1643: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 0 B/s wr, 609 op/s; 88346/274659 objects misplaced (32.166%); 5.3 MiB/s, 84 objects/s recovering 2023-06-10T21:47:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/839155655' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:56 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:47:56.738 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:56.853 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:47:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:56.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:47:56.966 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:47:57.012 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:57.076 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:47:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:57.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:47:57.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:47:57.299 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:57.371 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:47:57.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:57.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:47:57.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:57.547 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[180297]: osdmap e447: 8 total, 8 up, 5 in 2023-06-10T21:47:57.548 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:57.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:47:57.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:57.549 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:57 smithi064 ceph-mon[186706]: osdmap e447: 8 total, 8 up, 5 in 2023-06-10T21:47:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:57 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:47:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:57 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:47:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:57 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:47:57.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:57 smithi152 ceph-mon[156819]: osdmap e447: 8 total, 8 up, 5 in 2023-06-10T21:47:57.855 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:47:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:57.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:47:57.967 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:58.045 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:58.435 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:58.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:58.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:58.458 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:47:58.505 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:47:58.544 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:58.585 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:58.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[180297]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[180297]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[180297]: pgmap v1645: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 85334/266829 objects misplaced (31.981%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:47:58.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[186706]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[186706]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:47:58 smithi064 ceph-mon[186706]: pgmap v1645: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 85334/266829 objects misplaced (31.981%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:47:58.644 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:58.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:58.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:58 smithi152 ceph-mon[156819]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:58 smithi152 ceph-mon[156819]: failed to encode map e447 with expected crc 2023-06-10T21:47:58.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:47:58 smithi152 ceph-mon[156819]: pgmap v1645: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 595 op/s; 85334/266829 objects misplaced (31.981%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:47:58.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:47:58.878 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:59.050 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:47:59.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:59.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:47:59.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:47:59.184 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:59.658 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:47:59.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:59.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:47:59.769 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:47:59.806 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:47:59.863 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:47:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:47:59.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:47:59.969 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:00.080 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:00.290 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:00.361 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:00 smithi064 ceph-mon[180297]: pgmap v1646: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 85170/265698 objects misplaced (32.055%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:48:00.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:00 smithi064 ceph-mon[186706]: pgmap v1646: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 85170/265698 objects misplaced (32.055%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:48:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:00 smithi152 ceph-mon[156819]: pgmap v1646: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 592 op/s; 85170/265698 objects misplaced (32.055%); 4.8 MiB/s, 77 objects/s recovering 2023-06-10T21:48:00.766 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:00.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:00.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:00.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:00.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:48:00.906 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:00.975 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:00.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:00.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:01.082 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:01.112 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:01.172 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:01.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:01.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:01.283 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:01.406 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:01.443 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:01.485 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:01.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:01.554 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:01.602 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:01.652 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:01.686 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:01.723 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:01.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.737 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.738 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:48:01.893 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:01.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:02.001 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:02.089 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:02.328 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:02 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3600491161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:02 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3600491161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:02.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:02 smithi064 ceph-mon[186706]: pgmap v1647: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:02.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:02 smithi064 ceph-mon[180297]: pgmap v1647: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:02 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3600491161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:02.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:02 smithi152 ceph-mon[156819]: pgmap v1647: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:03.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:02] "GET /metrics HTTP/1.1" 200 224256 "" "Prometheus/2.33.4" 2023-06-10T21:48:03.319 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:03.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:03.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:03.468 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:03.560 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:03.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:03.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:03.665 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:03.691 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:03.731 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:03.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:03.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:03.836 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:03.867 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:04.272 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:04.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:04.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:04.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:04.404 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:04.463 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:04.466 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:04.466 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:04.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:04.535 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:04.568 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:04.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:04.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:04.601 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:04.601 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:04.615 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:04.798 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:04.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:04.907 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:04.938 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:04.997 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:05.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:05.103 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:05.141 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:05 smithi064 conmon[171866]: 2023-06-10T21:48:05.135+0000 7f6cfc057700 -1 mgr.server reply reply (3) No such process no change 2023-06-10T21:48:05.143 INFO:teuthology.orchestra.run.smithi064.stdout:moved 9 / 267 (3.37079%) 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout:avg 53.4 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 7.46994 -> 9.10604 (expected baseline 6.53605) 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 44 -> 41 pgs (0.82397 -> 0.76779 * mean) 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.0 with 66 -> 69 pgs (1.23595 -> 1.29213 * mean) 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:48:05.144 INFO:teuthology.orchestra.run.smithi064.stdout:oload 110 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 0.05 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0446 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.0491 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7 weight 0.8560 -> 0.8060 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3 weight 0.8053 -> 0.7554 2023-06-10T21:48:05.145 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5 weight 0.7307 -> 0.6808 2023-06-10T21:48:05.172 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-06-10T21:48:05.174 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:05.282 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:05.283 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:05.327 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:48:05.410 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:05 smithi064 ceph-mon[180297]: pgmap v1648: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:05.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:05 smithi064 ceph-mon[186706]: pgmap v1648: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:05.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:05 smithi152 ceph-mon[156819]: pgmap v1648: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 605 op/s; 81929/257454 objects misplaced (31.823%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:05.979 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:05.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:05.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:06.090 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:06.123 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:06.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:06 smithi064 ceph-mon[186706]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:06 smithi064 ceph-mon[180297]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:06.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:06 smithi152 ceph-mon[156819]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:07.384 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:07.389 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:07 smithi064 ceph-mon[186706]: pgmap v1649: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 81751/256299 objects misplaced (31.897%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:07.390 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:07 smithi064 ceph-mon[180297]: pgmap v1649: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 81751/256299 objects misplaced (31.897%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:07.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:07.491 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:07.526 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:07 smithi152 ceph-mon[156819]: pgmap v1649: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 0 B/s wr, 611 op/s; 81751/256299 objects misplaced (31.897%); 4.7 MiB/s, 74 objects/s recovering 2023-06-10T21:48:07.528 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:07.584 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:07.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:07.691 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:07.758 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:07.823 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-06-10T21:48:07.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd test-reweight-by-utilization 1000 3.0 2023-06-10T21:48:08.289 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:48:08.454 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:08.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:08.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:08.549 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:08 smithi064 ceph-mon[180297]: pgmap v1650: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 78551/248265 objects misplaced (31.640%); 5.2 MiB/s, 82 objects/s recovering 2023-06-10T21:48:08.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:08 smithi064 ceph-mon[186706]: pgmap v1650: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 78551/248265 objects misplaced (31.640%); 5.2 MiB/s, 82 objects/s recovering 2023-06-10T21:48:08.562 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:08.595 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:08.645 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:08.648 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:08 smithi152 ceph-mon[156819]: pgmap v1650: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 78551/248265 objects misplaced (31.640%); 5.2 MiB/s, 82 objects/s recovering 2023-06-10T21:48:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:08.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:08.724 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:08.751 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:08.757 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:08.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:09.035 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:09.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:09.158 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:09.190 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:09.251 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:09.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:09.358 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:09.393 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:09.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:09.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:09.556 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:09.584 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:09.638 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:09.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:09.744 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:09.775 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:09.826 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:09.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:09.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:09.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:09 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2598918471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:09.939 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:09 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:09.962 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:48:10.014 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:10.195 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:09 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2598918471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:10.196 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:09 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:10.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:09 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2598918471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:10.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:09 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:10.613 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:10.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:10.721 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:10.802 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:11.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:11.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.281 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:11.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: osdmap e448: 8 total, 8 up, 5 in 2023-06-10T21:48:11.282 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[186706]: pgmap v1652: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 78366/247089 objects misplaced (31.716%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: osdmap e448: 8 total, 8 up, 5 in 2023-06-10T21:48:11.284 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:10 smithi064 ceph-mon[180297]: pgmap v1652: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 78366/247089 objects misplaced (31.716%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:48:11.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:11.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:11.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: failed to encode map e448 with expected crc 2023-06-10T21:48:11.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: osdmap e448: 8 total, 8 up, 5 in 2023-06-10T21:48:11.446 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:10 smithi152 ceph-mon[156819]: pgmap v1652: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 78366/247089 objects misplaced (31.716%); 5.6 MiB/s, 90 objects/s recovering 2023-06-10T21:48:12.329 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:12.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:12.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:12 smithi064 ceph-mon[186706]: pgmap v1653: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:12.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:12 smithi064 ceph-mon[180297]: pgmap v1653: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:12.437 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:12.465 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:12.542 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:12.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:12.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:12.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:12.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:12 smithi152 ceph-mon[156819]: pgmap v1653: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:12.693 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:12.753 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:12.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:12.826 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:12.858 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:12.889 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:12.950 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:12.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:12.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:13.056 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:13.089 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:13.147 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:13.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:12] "GET /metrics HTTP/1.1" 200 224230 "" "Prometheus/2.33.4" 2023-06-10T21:48:13.254 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:13.334 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:13.584 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:13.592 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2329721006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:13.622 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:13.627 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:13.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:13.684 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:13.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:13.740 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:13.768 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:13.786 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:13.822 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:13.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:13.832 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:13.874 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2329721006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:13.877 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:13.914 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:13.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:13.937 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.938 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.939 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:48:13.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2329721006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:13.965 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:14.022 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:14.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:14.131 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:14.161 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:14.220 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:14.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:14.304 INFO:teuthology.orchestra.run.smithi064.stdout:moved 0 / 267 (0%) 2023-06-10T21:48:14.305 INFO:teuthology.orchestra.run.smithi064.stdout:avg 53.4 2023-06-10T21:48:14.305 INFO:teuthology.orchestra.run.smithi064.stdout:stddev 7.46994 -> 7.46994 (expected baseline 6.53605) 2023-06-10T21:48:14.305 INFO:teuthology.orchestra.run.smithi064.stdout:min osd.5 with 44 -> 44 pgs (0.82397 -> 0.82397 * mean) 2023-06-10T21:48:14.305 INFO:teuthology.orchestra.run.smithi064.stdout:max osd.0 with 66 -> 66 pgs (1.23595 -> 1.23595 * mean) 2023-06-10T21:48:14.305 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:48:14.306 INFO:teuthology.orchestra.run.smithi064.stdout:oload 1000 2023-06-10T21:48:14.306 INFO:teuthology.orchestra.run.smithi064.stdout:max_change 3 2023-06-10T21:48:14.306 INFO:teuthology.orchestra.run.smithi064.stdout:max_change_osds 4 2023-06-10T21:48:14.306 INFO:teuthology.orchestra.run.smithi064.stdout:average_utilization 0.0423 2023-06-10T21:48:14.306 INFO:teuthology.orchestra.run.smithi064.stdout:overload_utilization 0.4232 2023-06-10T21:48:14.313 INFO:teuthology.orchestra.run.smithi064.stderr:no change 2023-06-10T21:48:14.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:14.371 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:14.430 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:14.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:14.536 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:14.571 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:14.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:14 smithi064 ceph-mon[180297]: pgmap v1654: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:14.614 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:14 smithi064 ceph-mon[180297]: from='client.57097 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:14.628 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:14.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:48:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:14.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:14.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:14.769 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:14.825 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:14.830 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:14 smithi152 ceph-mon[156819]: pgmap v1654: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:14.831 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:14 smithi152 ceph-mon[156819]: from='client.57097 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:14.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:14.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:14 smithi064 ceph-mon[186706]: pgmap v1654: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 75153/238788 objects misplaced (31.473%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:14.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:14 smithi064 ceph-mon[186706]: from='client.57097 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-06-10T21:48:14.933 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:15.006 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:16.074 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:16.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:16.183 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:16.216 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:16.256 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:16.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:16.361 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:16.374 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:16 smithi064 ceph-mon[180297]: pgmap v1655: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 74950/237603 objects misplaced (31.544%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:16.395 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:16.463 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:16.468 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:16 smithi152 ceph-mon[156819]: pgmap v1655: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 74950/237603 objects misplaced (31.544%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:16.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:16.570 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:16.612 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:16 smithi064 ceph-mon[186706]: pgmap v1655: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 622 op/s; 74950/237603 objects misplaced (31.544%); 5.5 MiB/s, 87 objects/s recovering 2023-06-10T21:48:16.666 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:16.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:16.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:16.775 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:16.849 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:16.964 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:16.976 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:16.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:16.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:17.091 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:17.180 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:17.447 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:17.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:17.607 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:17.685 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:17.695 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:48:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:48:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.695 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140055 14.d 14:b5a5b314:::benchmark_data_smithi064_559916_object32199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662588+0000", 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016190541999999999, 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016222370999999999, 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.696 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140055 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662588+0000", 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662588+0000", 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662586+0000", 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662593+0000", 2023-06-10T21:48:17.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9879999999999998e-06 2023-06-10T21:48:17.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662610+0000", 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7221999999999999e-05 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662620+0000", 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3209999999999994e-06 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662720+0000", 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010037100000000001 2023-06-10T21:48:17.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662856+0000", 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013584400000000001 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:48:17.703 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.663001+0000", 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014549800000000001 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.669809+0000", 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.006807446 2023-06-10T21:48:17.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:48:17.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676827+0000", 2023-06-10T21:48:17.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007018018 2023-06-10T21:48:17.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:48:17.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676862+0000", 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5460999999999998e-05 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140058 14.0 14:0d072c98:::benchmark_data_smithi064_559916_object32202:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662633+0000", 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016145623000000001, 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016282652000000002, 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.708 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.709 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.709 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140058 2023-06-10T21:48:17.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.709 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662633+0000", 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662633+0000", 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662632+0000", 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662636+0000", 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6509999999999999e-06 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662644+0000", 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1499999999999999e-06 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662650+0000", 2023-06-10T21:48:17.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2920000000000001e-06 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.671277+0000", 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086270860000000008 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.671421+0000", 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014366800000000001 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.671557+0000", 2023-06-10T21:48:17.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000135718 2023-06-10T21:48:17.716 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.716 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.716 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140059 14.c 14:3823a44b:::benchmark_data_smithi064_559916_object32203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662669+0000", 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016109058999999998, 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016323671000000001, 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.717 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140059 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662669+0000", 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662669+0000", 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662669+0000", 2023-06-10T21:48:17.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662673+0000", 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8630000000000001e-06 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662685+0000", 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2258e-05 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662692+0000", 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5610000000000001e-06 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.666725+0000", 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0040328370000000001 2023-06-10T21:48:17.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.666860+0000", 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013457700000000001 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.666991+0000", 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131688 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140060 14.8 14:1d1b1d63:::benchmark_data_smithi064_559916_object32204:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662705+0000", 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016073463, 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016364391999999998, 2023-06-10T21:48:17.726 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140060 2023-06-10T21:48:17.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.728 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662705+0000", 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662705+0000", 2023-06-10T21:48:17.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662704+0000", 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662708+0000", 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6679999999999999e-06 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662715+0000", 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4170000000000003e-06 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662721+0000", 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3399999999999997e-06 2023-06-10T21:48:17.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672882+0000", 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010161691 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.673021+0000", 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013835099999999999 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.673152+0000", 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013174399999999999 2023-06-10T21:48:17.734 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140062 14.e 14:7ec09c85:::benchmark_data_smithi064_559916_object32206:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662726+0000", 2023-06-10T21:48:17.735 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016052291, 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016418047000000002, 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.736 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140062 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662726+0000", 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662726+0000", 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662726+0000", 2023-06-10T21:48:17.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662728+0000", 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6000000000000001e-06 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662734+0000", 2023-06-10T21:48:17.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0170000000000002e-06 2023-06-10T21:48:17.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662739+0000", 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2669999999999997e-06 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662822+0000", 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2142000000000001e-05 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662947+0000", 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124893 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.663076+0000", 2023-06-10T21:48:17.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012945900000000001 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.669481+0000", 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064045559999999996 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676978+0000", 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0074978199999999997 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140065 14.0 14.6f4af540 (undecoded) ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.662764+0000", 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.016014008999999999, 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.016462871, 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:48:17.746 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140065 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662764+0000", 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662764+0000", 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662764+0000", 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662767+0000", 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5429999999999998e-06 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662775+0000", 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8809999999999996e-06 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.662781+0000", 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5589999999999996e-06 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:140059 14.c e448/443)", 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.672772+0000", 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.006006768, 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0065091150000000002, 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.752 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672772+0000", 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672772+0000", 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672771+0000", 2023-06-10T21:48:17.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672778+0000", 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5580000000000003e-06 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672789+0000", 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1080000000000001e-05 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.672797+0000", 2023-06-10T21:48:17.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9740000000000002e-06 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:140059 14.c e448/443)", 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.676764+0000", 2023-06-10T21:48:17.757 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.002014685, 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025654900000000001, 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676764+0000", 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676764+0000", 2023-06-10T21:48:17.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676763+0000", 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676769+0000", 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6729999999999997e-06 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676782+0000", 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3098999999999999e-05 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676790+0000", 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8089999999999999e-06 2023-06-10T21:48:17.762 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140067 14.c 14.800c7bc (undecoded) ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.676997+0000", 2023-06-10T21:48:17.763 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0017814370000000001, 2023-06-10T21:48:17.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0023894110000000001, 2023-06-10T21:48:17.764 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.764 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:48:17.764 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.764 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140067 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676997+0000", 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676997+0000", 2023-06-10T21:48:17.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.676995+0000", 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677001+0000", 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2939999999999996e-06 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677013+0000", 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2140999999999999e-05 2023-06-10T21:48:17.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677021+0000", 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3890000000000005e-06 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:140069 14.e 14:7b8b0795:::benchmark_data_smithi064_559916_object32213:head [delete] snapc 0=[] ondisk+write+known_if_redirected e448)", 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.677028+0000", 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0017506869999999999, 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024168760000000001, 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.770 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "waiting for sub ops", 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 140069 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677028+0000", 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677028+0000", 2023-06-10T21:48:17.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677027+0000", 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677030+0000", 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4980000000000001e-06 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677036+0000", 2023-06-10T21:48:17.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1380000000000004e-06 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677041+0000", 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4310000000000002e-06 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677118+0000", 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6809000000000003e-05 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677253+0000", 2023-06-10T21:48:17.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013533999999999999 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677408+0000", 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000154563 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.777 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:140066 14.1 e448/444 14:853a3746:::benchmark_data_smithi064_559916_object32210:head v 448'8918, mlcod=448'8916)", 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.677085+0000", 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.00169327, 2023-06-10T21:48:17.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024327519999999998, 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677085+0000", 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677085+0000", 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677085+0000", 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677090+0000", 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7089999999999999e-06 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677099+0000", 2023-06-10T21:48:17.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7350000000000002e-06 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677105+0000", 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7950000000000004e-06 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677767+0000", 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00066193199999999995 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677792+0000", 2023-06-10T21:48:17.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4926000000000002e-05 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop_reply(client.47150.0:140062 14.e e448/444)", 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:17.677119+0000", 2023-06-10T21:48:17.785 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 0.0016597179999999999, 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024558850000000001, 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "queued for pg", 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:48:17.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677119+0000", 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677119+0000", 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677118+0000", 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:48:17.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677121+0000", 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.999e-06 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677128+0000", 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8310000000000003e-06 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:48:17.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:17.677133+0000", 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1919999999999996e-06 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:48:17.790 INFO:teuthology.orchestra.run.smithi064.stdout: ], 2023-06-10T21:48:17.791 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 12 2023-06-10T21:48:17.791 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:48:18.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:48:19.570 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:19.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:19.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:19.593 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[186706]: pgmap v1656: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 607 op/s; 72106/230034 objects misplaced (31.346%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:48:19.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:19.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:19.594 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:19 smithi064 ceph-mon[180297]: pgmap v1656: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 607 op/s; 72106/230034 objects misplaced (31.346%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:48:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:19.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:19.623 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:48:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:19 smithi152 ceph-mon[156819]: pgmap v1656: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 607 op/s; 72106/230034 objects misplaced (31.346%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:48:19.696 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:19.729 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:19.803 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:19.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:19.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:19.944 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:19.999 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:20.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:20.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:20.138 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:20.195 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:20.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:20.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:20.329 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:20.371 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:20 smithi064 ceph-mon[180297]: pgmap v1657: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 71919/228912 objects misplaced (31.418%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:48:20.379 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:20.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:20.461 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:20.484 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:20.553 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:20.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:20.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:20 smithi064 ceph-mon[186706]: pgmap v1657: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 71919/228912 objects misplaced (31.418%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:48:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:20 smithi152 ceph-mon[156819]: pgmap v1657: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 71919/228912 objects misplaced (31.418%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:48:20.769 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:20.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:20.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:20.917 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:20.982 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:20.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:21.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:21.273 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:21.354 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:21.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:21.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:21.409 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:48:21.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:21.540 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:21 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:48:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:21.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:21 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:48:21.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:21.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:21 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:48:21.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:21.771 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5, 0] out_osds: [6, 4, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:48:21.771 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:48:21.772 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-06-10T21:48:21.772 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:48:21.991 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:22.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:22.103 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:22.135 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:22.249 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:22.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:22.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:22.423 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: osdmap e449: 8 total, 8 up, 5 in 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.424 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:22 smithi152 ceph-mon[156819]: pgmap v1659: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:22.431 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:22.817 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:22.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:22.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: osdmap e449: 8 total, 8 up, 5 in 2023-06-10T21:48:22.818 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.819 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[186706]: pgmap v1659: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:22.820 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4135818559' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: osdmap e449: 8 total, 8 up, 5 in 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: failed to encode map e449 with expected crc 2023-06-10T21:48:22.821 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:22 smithi064 ceph-mon[180297]: pgmap v1659: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:22] "GET /metrics HTTP/1.1" 200 224230 "" "Prometheus/2.33.4" 2023-06-10T21:48:23.352 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:23.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:23.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:23.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:23.491 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:23.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:23.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:23.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:23.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:23.741 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:24.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:24.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:24.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:24.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:24.331 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:24.656 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:24.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:24.767 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:24.804 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:24.847 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:24.859 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:24.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:24.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:24.999 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:25.057 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:25.058 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:48:25.058 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:48:25.058 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:48:25.058 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:48:25.058 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:48:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:25.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:25.073 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:25.121 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:25.161 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:25.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:25.195 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:25.216 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:25.250 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:25.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:25.264 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:25.322 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:25.358 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:25.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.359 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[180297]: pgmap v1660: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:25.360 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/28904757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:25.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.364 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[186706]: pgmap v1660: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:25.365 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/28904757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:25.370 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:25.415 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:25.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.432 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.433 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:48:25.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:25 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:25.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:25 smithi152 ceph-mon[156819]: pgmap v1660: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 586 op/s; 69001/221184 objects misplaced (31.196%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:25.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/28904757' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:25.489 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:25.817 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:25.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:25.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:26.030 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:26.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:48:26.377 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:26 smithi064 ceph-mon[180297]: pgmap v1661: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 68853/220230 objects misplaced (31.264%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:26.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:26 smithi064 ceph-mon[186706]: pgmap v1661: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 68853/220230 objects misplaced (31.264%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:26 smithi152 ceph-mon[156819]: pgmap v1661: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 68853/220230 objects misplaced (31.264%); 4.1 MiB/s, 66 objects/s recovering 2023-06-10T21:48:28.065 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:48:28.194 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:28.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:28.302 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:28.393 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:28.577 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:28.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:28 smithi064 ceph-mon[186706]: pgmap v1662: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 66004/212775 objects misplaced (31.021%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:28.677 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:28 smithi064 ceph-mon[180297]: pgmap v1662: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 66004/212775 objects misplaced (31.021%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:28.690 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:28.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:28 smithi152 ceph-mon[156819]: pgmap v1662: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 575 op/s; 66004/212775 objects misplaced (31.021%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:28.721 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:28.781 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:28.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:28.884 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:28.888 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:28.920 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:28.974 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:28.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:28.985 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:29.080 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:29.171 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:29.289 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:48:29.302 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:48:29.405 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:29 smithi152 ceph-mon[156819]: from='client.57115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:48:29.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:29 smithi064 ceph-mon[180297]: from='client.57115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:48:29.522 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:29.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:29.634 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:29.677 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:29.749 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:29.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:29.860 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:29.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:29 smithi064 ceph-mon[186706]: from='client.57115 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:48:29.969 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:30.436 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:48:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:30 smithi064 ceph-mon[186706]: pgmap v1663: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 570 op/s; 65843/211791 objects misplaced (31.089%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:30 smithi064 ceph-mon[180297]: pgmap v1663: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 570 op/s; 65843/211791 objects misplaced (31.089%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:30.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:30 smithi152 ceph-mon[156819]: pgmap v1663: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 570 op/s; 65843/211791 objects misplaced (31.089%); 4.5 MiB/s, 72 objects/s recovering 2023-06-10T21:48:30.841 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:30.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:30.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:31.017 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:31.874 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:31.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:31.982 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:32.024 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:32.096 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:32.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:32.207 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:32.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:32.472 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1662,"stamp":"2023-06-10T21:48:28.287135+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4647098278,"num_objects":70925,"num_object_clones":0,"num_object_copies":212775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":66004,"num_objects_unfound":0,"num_objects_dirty":70925,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":145068,"num_write_kb":6905039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6524,"num_bytes_recovered":423453126,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93073,"ondisk_log_size":93073,"up":266,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":282,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":18213016,"kb_used_data":14106224,"kb_used_omap":38,"kb_used_meta":4106585,"kb_avail":450512744,"statfs":{"total":479975178240,"available":461325049856,"internally_reserved":0,"allocated":14444773376,"data_stored":14438902803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39282,"internal_metadata":4205143694},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,4,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-377028608,"num_objects":-5753,"num_object_clones":0,"num_object_copies":-17259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-6102,"num_objects_unfound":0,"num_objects_dirty":-5753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5753,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":725,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003032"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":498,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.945984+0000","last_change":"2023-06-10T21:47:18.342858+0000","last_active":"2023-06-10T21:48:09.945984+0000","last_peered":"2023-06-10T21:48:09.945984+0000","last_clean":"2023-06-10T21:48:09.945984+0000","last_became_active":"2023-06-10T21:47:18.342378+0000","last_became_peered":"2023-06-10T21:47:18.342378+0000","last_unstale":"2023-06-10T21:48:09.945984+0000","last_undegraded":"2023-06-10T21:48:09.945984+0000","last_fullsized":"2023-06-10T21:48:09.945984+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:30.423525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":128,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936956+0000","last_change":"2023-06-10T21:47:33.347537+0000","last_active":"2023-06-10T21:48:09.936956+0000","last_peered":"2023-06-10T21:48:09.936956+0000","last_clean":"2023-06-10T21:48:09.936956+0000","last_became_active":"2023-06-10T21:47:33.345744+0000","last_became_peered":"2023-06-10T21:47:33.345744+0000","last_unstale":"2023-06-10T21:48:09.936956+0000","last_undegraded":"2023-06-10T21:48:09.936956+0000","last_fullsized":"2023-06-10T21:48:09.936956+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:34:25.130686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":391,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:21.344117+0000","last_change":"2023-06-10T21:47:33.370243+0000","last_active":"2023-06-10T21:48:21.344117+0000","last_peered":"2023-06-10T21:48:21.344117+0000","last_clean":"2023-06-10T21:48:21.344117+0000","last_became_active":"2023-06-10T21:47:33.369778+0000","last_became_peered":"2023-06-10T21:47:33.369778+0000","last_unstale":"2023-06-10T21:48:21.344117+0000","last_undegraded":"2023-06-10T21:48:21.344117+0000","last_fullsized":"2023-06-10T21:48:21.344117+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:08:04.445031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956195+0000","last_change":"2023-06-10T21:47:33.366424+0000","last_active":"2023-06-10T21:48:09.956195+0000","last_peered":"2023-06-10T21:48:09.956195+0000","last_clean":"2023-06-10T21:48:09.956195+0000","last_became_active":"2023-06-10T21:47:33.365958+0000","last_became_peered":"2023-06-10T21:47:33.365958+0000","last_unstale":"2023-06-10T21:48:09.956195+0000","last_undegraded":"2023-06-10T21:48:09.956195+0000","last_fullsized":"2023-06-10T21:48:09.956195+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:14:34.339878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":113,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935604+0000","last_change":"2023-06-10T21:47:18.339120+0000","last_active":"2023-06-10T21:48:09.935604+0000","last_peered":"2023-06-10T21:48:09.935604+0000","last_clean":"2023-06-10T21:48:09.935604+0000","last_became_active":"2023-06-10T21:47:18.338758+0000","last_became_peered":"2023-06-10T21:47:18.338758+0000","last_unstale":"2023-06-10T21:48:09.935604+0000","last_undegraded":"2023-06-10T21:48:09.935604+0000","last_fullsized":"2023-06-10T21:48:09.935604+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:10:24.375802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954548+0000","last_change":"2023-06-10T21:47:33.366261+0000","last_active":"2023-06-10T21:48:09.954548+0000","last_peered":"2023-06-10T21:48:09.954548+0000","last_clean":"2023-06-10T21:48:09.954548+0000","last_became_active":"2023-06-10T21:47:33.365960+0000","last_became_peered":"2023-06-10T21:47:33.365960+0000","last_unstale":"2023-06-10T21:48:09.954548+0000","last_undegraded":"2023-06-10T21:48:09.954548+0000","last_fullsized":"2023-06-10T21:48:09.954548+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:04:13.924182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954694+0000","last_change":"2023-06-10T21:47:18.325364+0000","last_active":"2023-06-10T21:48:09.954694+0000","last_peered":"2023-06-10T21:48:09.954694+0000","last_clean":"2023-06-10T21:48:09.954694+0000","last_became_active":"2023-06-10T21:47:18.324927+0000","last_became_peered":"2023-06-10T21:47:18.324927+0000","last_unstale":"2023-06-10T21:48:09.954694+0000","last_undegraded":"2023-06-10T21:48:09.954694+0000","last_fullsized":"2023-06-10T21:48:09.954694+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:17:56.499465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":128,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935689+0000","last_change":"2023-06-10T21:47:33.342823+0000","last_active":"2023-06-10T21:48:09.935689+0000","last_peered":"2023-06-10T21:48:09.935689+0000","last_clean":"2023-06-10T21:48:09.935689+0000","last_became_active":"2023-06-10T21:47:33.342424+0000","last_became_peered":"2023-06-10T21:47:33.342424+0000","last_unstale":"2023-06-10T21:48:09.935689+0000","last_undegraded":"2023-06-10T21:48:09.935689+0000","last_fullsized":"2023-06-10T21:48:09.935689+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:05:27.714545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":118,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936851+0000","last_change":"2023-06-10T21:47:33.365384+0000","last_active":"2023-06-10T21:48:09.936851+0000","last_peered":"2023-06-10T21:48:09.936851+0000","last_clean":"2023-06-10T21:48:09.936851+0000","last_became_active":"2023-06-10T21:47:33.365005+0000","last_became_peered":"2023-06-10T21:47:33.365005+0000","last_unstale":"2023-06-10T21:48:09.936851+0000","last_undegraded":"2023-06-10T21:48:09.936851+0000","last_fullsized":"2023-06-10T21:48:09.936851+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.940267+0000","last_change":"2023-06-10T21:43:01.210488+0000","last_active":"2023-06-10T21:48:09.940267+0000","last_peered":"2023-06-10T21:48:09.940267+0000","last_clean":"2023-06-10T21:48:09.940267+0000","last_became_active":"2023-06-10T21:43:01.210161+0000","last_became_peered":"2023-06-10T21:43:01.210161+0000","last_unstale":"2023-06-10T21:48:09.940267+0000","last_undegraded":"2023-06-10T21:48:09.940267+0000","last_fullsized":"2023-06-10T21:48:09.940267+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:19:11.296560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":221,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936852+0000","last_change":"2023-06-10T21:47:18.431486+0000","last_active":"2023-06-10T21:48:09.936852+0000","last_peered":"2023-06-10T21:48:09.936852+0000","last_clean":"2023-06-10T21:48:09.936852+0000","last_became_active":"2023-06-10T21:47:18.326725+0000","last_became_peered":"2023-06-10T21:47:18.326725+0000","last_unstale":"2023-06-10T21:48:09.936852+0000","last_undegraded":"2023-06-10T21:48:09.936852+0000","last_fullsized":"2023-06-10T21:48:09.936852+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":459,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.946080+0000","last_change":"2023-06-10T21:47:18.333737+0000","last_active":"2023-06-10T21:48:09.946080+0000","last_peered":"2023-06-10T21:48:09.946080+0000","last_clean":"2023-06-10T21:48:09.946080+0000","last_became_active":"2023-06-10T21:47:18.333439+0000","last_became_peered":"2023-06-10T21:47:18.333439+0000","last_unstale":"2023-06-10T21:48:09.946080+0000","last_undegraded":"2023-06-10T21:48:09.946080+0000","last_fullsized":"2023-06-10T21:48:09.946080+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:22:19.410820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":391,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.938071+0000","last_change":"2023-06-10T21:47:18.716040+0000","last_active":"2023-06-10T21:48:09.938071+0000","last_peered":"2023-06-10T21:48:09.938071+0000","last_clean":"2023-06-10T21:48:09.938071+0000","last_became_active":"2023-06-10T21:47:18.334031+0000","last_became_peered":"2023-06-10T21:47:18.334031+0000","last_unstale":"2023-06-10T21:48:09.938071+0000","last_undegraded":"2023-06-10T21:48:09.938071+0000","last_fullsized":"2023-06-10T21:48:09.938071+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":18864,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":141,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.946345+0000","last_change":"2023-06-10T21:46:14.281135+0000","last_active":"2023-06-10T21:48:09.946345+0000","last_peered":"2023-06-10T21:48:09.946345+0000","last_clean":"2023-06-10T21:48:09.946345+0000","last_became_active":"2023-06-10T21:46:14.280752+0000","last_became_peered":"2023-06-10T21:46:14.280752+0000","last_unstale":"2023-06-10T21:48:09.946345+0000","last_undegraded":"2023-06-10T21:48:09.946345+0000","last_fullsized":"2023-06-10T21:48:09.946345+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:30:54.587316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":523,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936795+0000","last_change":"2023-06-10T21:47:18.534400+0000","last_active":"2023-06-10T21:48:09.936795+0000","last_peered":"2023-06-10T21:48:09.936795+0000","last_clean":"2023-06-10T21:48:09.936795+0000","last_became_active":"2023-06-10T21:47:18.333706+0000","last_became_peered":"2023-06-10T21:47:18.333706+0000","last_unstale":"2023-06-10T21:48:09.936795+0000","last_undegraded":"2023-06-10T21:48:09.936795+0000","last_fullsized":"2023-06-10T21:48:09.936795+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":398,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935663+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:48:09.935663+0000","last_peered":"2023-06-10T21:48:09.935663+0000","last_clean":"2023-06-10T21:48:09.935663+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:48:09.935663+0000","last_undegraded":"2023-06-10T21:48:09.935663+0000","last_fullsized":"2023-06-10T21:48:09.935663+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.955410+0000","last_change":"2023-06-10T21:47:18.322156+0000","last_active":"2023-06-10T21:48:09.955410+0000","last_peered":"2023-06-10T21:48:09.955410+0000","last_clean":"2023-06-10T21:48:09.955410+0000","last_became_active":"2023-06-10T21:47:18.320854+0000","last_became_peered":"2023-06-10T21:47:18.320854+0000","last_unstale":"2023-06-10T21:48:09.955410+0000","last_undegraded":"2023-06-10T21:48:09.955410+0000","last_fullsized":"2023-06-10T21:48:09.955410+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:07:26.646183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":325,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.937114+0000","last_change":"2023-06-10T21:47:33.362328+0000","last_active":"2023-06-10T21:48:09.937114+0000","last_peered":"2023-06-10T21:48:09.937114+0000","last_clean":"2023-06-10T21:48:09.937114+0000","last_became_active":"2023-06-10T21:47:33.362007+0000","last_became_peered":"2023-06-10T21:47:33.362007+0000","last_unstale":"2023-06-10T21:48:09.937114+0000","last_undegraded":"2023-06-10T21:48:09.937114+0000","last_fullsized":"2023-06-10T21:48:09.937114+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":470,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936604+0000","last_change":"2023-06-10T21:47:18.327786+0000","last_active":"2023-06-10T21:48:09.936604+0000","last_peered":"2023-06-10T21:48:09.936604+0000","last_clean":"2023-06-10T21:48:09.936604+0000","last_became_active":"2023-06-10T21:47:18.327469+0000","last_became_peered":"2023-06-10T21:47:18.327469+0000","last_unstale":"2023-06-10T21:48:09.936604+0000","last_undegraded":"2023-06-10T21:48:09.936604+0000","last_fullsized":"2023-06-10T21:48:09.936604+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.955955+0000","last_change":"2023-06-10T21:47:33.372564+0000","last_active":"2023-06-10T21:48:09.955955+0000","last_peered":"2023-06-10T21:48:09.955955+0000","last_clean":"2023-06-10T21:48:09.955955+0000","last_became_active":"2023-06-10T21:47:33.371667+0000","last_became_peered":"2023-06-10T21:47:33.371667+0000","last_unstale":"2023-06-10T21:48:09.955955+0000","last_undegraded":"2023-06-10T21:48:09.955955+0000","last_fullsized":"2023-06-10T21:48:09.955955+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:02.002487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":50,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.947627+0000","last_change":"2023-06-10T21:47:33.426219+0000","last_active":"2023-06-10T21:48:09.947627+0000","last_peered":"2023-06-10T21:48:09.947627+0000","last_clean":"2023-06-10T21:48:09.947627+0000","last_became_active":"2023-06-10T21:47:33.370278+0000","last_became_peered":"2023-06-10T21:47:33.370278+0000","last_unstale":"2023-06-10T21:48:09.947627+0000","last_undegraded":"2023-06-10T21:48:09.947627+0000","last_fullsized":"2023-06-10T21:48:09.947627+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:50:33.467628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":282,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.951513+0000","last_change":"2023-06-10T21:47:33.368498+0000","last_active":"2023-06-10T21:48:09.951513+0000","last_peered":"2023-06-10T21:48:09.951513+0000","last_clean":"2023-06-10T21:48:09.951513+0000","last_became_active":"2023-06-10T21:47:33.368058+0000","last_became_peered":"2023-06-10T21:47:33.368058+0000","last_unstale":"2023-06-10T21:48:09.951513+0000","last_undegraded":"2023-06-10T21:48:09.951513+0000","last_fullsized":"2023-06-10T21:48:09.951513+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:44:15.397639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.2","version":"449'9011","reported_seq":9074,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.287710+0000","last_change":"2023-06-10T21:47:34.372361+0000","last_active":"2023-06-10T21:48:26.287710+0000","last_peered":"2023-06-10T21:48:26.287710+0000","last_clean":"2023-06-10T21:47:17.271926+0000","last_became_active":"2023-06-10T21:47:34.348340+0000","last_became_peered":"2023-06-10T21:47:34.348340+0000","last_unstale":"2023-06-10T21:48:26.287710+0000","last_undegraded":"2023-06-10T21:48:26.287710+0000","last_fullsized":"2023-06-10T21:48:26.287710+0000","mapping_epoch":444,"log_start":"430'3500","ondisk_log_start":"430'3500","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:47:10.074291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4457,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9011,"num_write_kb":430976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4457}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":254,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.937355+0000","last_change":"2023-06-10T21:47:18.443129+0000","last_active":"2023-06-10T21:48:09.937355+0000","last_peered":"2023-06-10T21:48:09.937355+0000","last_clean":"2023-06-10T21:48:09.937355+0000","last_became_active":"2023-06-10T21:47:18.326666+0000","last_became_peered":"2023-06-10T21:47:18.326666+0000","last_unstale":"2023-06-10T21:48:09.937355+0000","last_undegraded":"2023-06-10T21:48:09.937355+0000","last_fullsized":"2023-06-10T21:48:09.937355+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936356+0000","last_change":"2023-06-10T21:47:33.341680+0000","last_active":"2023-06-10T21:48:09.936356+0000","last_peered":"2023-06-10T21:48:09.936356+0000","last_clean":"2023-06-10T21:48:09.936356+0000","last_became_active":"2023-06-10T21:47:33.341393+0000","last_became_peered":"2023-06-10T21:47:33.341393+0000","last_unstale":"2023-06-10T21:48:09.936356+0000","last_undegraded":"2023-06-10T21:48:09.936356+0000","last_fullsized":"2023-06-10T21:48:09.936356+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"449'9071","reported_seq":17300,"reported_epoch":449,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:48:26.340651+0000","last_change":"2023-06-10T21:47:34.420652+0000","last_active":"2023-06-10T21:48:26.340651+0000","last_peered":"2023-06-10T21:48:26.340651+0000","last_clean":"2023-06-10T21:47:17.250828+0000","last_became_active":"2023-06-10T21:47:33.346635+0000","last_became_peered":"2023-06-10T21:47:33.346635+0000","last_unstale":"2023-06-10T21:48:26.340651+0000","last_undegraded":"2023-06-10T21:48:26.340651+0000","last_fullsized":"2023-06-10T21:48:26.340651+0000","mapping_epoch":443,"log_start":"430'3600","ondisk_log_start":"430'3600","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5471,"log_dups_size":0,"ondisk_log_size":5471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:30:57.552551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1031,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9071,"num_write_kb":430848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4084,"num_bytes_recovered":267649024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[3,5,7],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4393}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":267,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936937+0000","last_change":"2023-06-10T21:47:33.362906+0000","last_active":"2023-06-10T21:48:09.936937+0000","last_peered":"2023-06-10T21:48:09.936937+0000","last_clean":"2023-06-10T21:48:09.936937+0000","last_became_active":"2023-06-10T21:47:33.349060+0000","last_became_peered":"2023-06-10T21:47:33.349060+0000","last_unstale":"2023-06-10T21:48:09.936937+0000","last_undegraded":"2023-06-10T21:48:09.936937+0000","last_fullsized":"2023-06-10T21:48:09.936937+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.952524+0000","last_change":"2023-06-10T21:47:33.364962+0000","last_active":"2023-06-10T21:48:09.952524+0000","last_peered":"2023-06-10T21:48:09.952524+0000","last_clean":"2023-06-10T21:48:09.952524+0000","last_became_active":"2023-06-10T21:47:33.364669+0000","last_became_peered":"2023-06-10T21:47:33.364669+0000","last_unstale":"2023-06-10T21:48:09.952524+0000","last_undegraded":"2023-06-10T21:48:09.952524+0000","last_fullsized":"2023-06-10T21:48:09.952524+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:23:29.453829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.0","version":"449'8988","reported_seq":9043,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.335683+0000","last_change":"2023-06-10T21:47:19.662715+0000","last_active":"2023-06-10T21:48:26.335683+0000","last_peered":"2023-06-10T21:48:26.335683+0000","last_clean":"2023-06-10T21:47:17.279925+0000","last_became_active":"2023-06-10T21:47:19.608783+0000","last_became_peered":"2023-06-10T21:47:19.608783+0000","last_unstale":"2023-06-10T21:48:26.335683+0000","last_undegraded":"2023-06-10T21:48:26.335683+0000","last_fullsized":"2023-06-10T21:48:26.335683+0000","mapping_epoch":439,"log_start":"428'2800","ondisk_log_start":"428'2800","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6188,"log_dups_size":0,"ondisk_log_size":6188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:23:21.473590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290848768,"num_objects":4438,"num_object_clones":0,"num_object_copies":13314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4438,"num_objects_unfound":0,"num_objects_dirty":4438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8988,"num_write_kb":429632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,5,3],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":4438}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":541,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936960+0000","last_change":"2023-06-10T21:47:33.418326+0000","last_active":"2023-06-10T21:48:09.936960+0000","last_peered":"2023-06-10T21:48:09.936960+0000","last_clean":"2023-06-10T21:48:09.936960+0000","last_became_active":"2023-06-10T21:47:33.365635+0000","last_became_peered":"2023-06-10T21:47:33.365635+0000","last_unstale":"2023-06-10T21:48:09.936960+0000","last_undegraded":"2023-06-10T21:48:09.936960+0000","last_fullsized":"2023-06-10T21:48:09.936960+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":221,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936842+0000","last_change":"2023-06-10T21:47:18.326618+0000","last_active":"2023-06-10T21:48:09.936842+0000","last_peered":"2023-06-10T21:48:09.936842+0000","last_clean":"2023-06-10T21:48:09.936842+0000","last_became_active":"2023-06-10T21:47:18.326043+0000","last_became_peered":"2023-06-10T21:47:18.326043+0000","last_unstale":"2023-06-10T21:48:09.936842+0000","last_undegraded":"2023-06-10T21:48:09.936842+0000","last_fullsized":"2023-06-10T21:48:09.936842+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.1","version":"449'9215","reported_seq":9262,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.340491+0000","last_change":"2023-06-10T21:47:34.449489+0000","last_active":"2023-06-10T21:48:26.340491+0000","last_peered":"2023-06-10T21:48:26.340491+0000","last_clean":"2023-06-10T21:47:32.299728+0000","last_became_active":"2023-06-10T21:47:34.441653+0000","last_became_peered":"2023-06-10T21:47:34.441653+0000","last_unstale":"2023-06-10T21:48:26.340491+0000","last_undegraded":"2023-06-10T21:48:26.340491+0000","last_fullsized":"2023-06-10T21:48:26.340491+0000","mapping_epoch":444,"log_start":"430'3700","ondisk_log_start":"430'3700","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:59:58.966545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026112,"num_objects":4517,"num_object_clones":0,"num_object_copies":13551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4517,"num_objects_unfound":0,"num_objects_dirty":4517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9215,"num_write_kb":439424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":4517}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935516+0000","last_change":"2023-06-10T21:47:18.334056+0000","last_active":"2023-06-10T21:48:09.935516+0000","last_peered":"2023-06-10T21:48:09.935516+0000","last_clean":"2023-06-10T21:48:09.935516+0000","last_became_active":"2023-06-10T21:47:18.333610+0000","last_became_peered":"2023-06-10T21:47:18.333610+0000","last_unstale":"2023-06-10T21:48:09.935516+0000","last_undegraded":"2023-06-10T21:48:09.935516+0000","last_fullsized":"2023-06-10T21:48:09.935516+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:02:04.157991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956320+0000","last_change":"2023-06-10T21:47:18.321572+0000","last_active":"2023-06-10T21:48:09.956320+0000","last_peered":"2023-06-10T21:48:09.956320+0000","last_clean":"2023-06-10T21:48:09.956320+0000","last_became_active":"2023-06-10T21:47:18.321254+0000","last_became_peered":"2023-06-10T21:47:18.321254+0000","last_unstale":"2023-06-10T21:48:09.956320+0000","last_undegraded":"2023-06-10T21:48:09.956320+0000","last_fullsized":"2023-06-10T21:48:09.956320+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:01:59.761479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"449'8886","reported_seq":8939,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.422442+0000","last_change":"2023-06-10T21:47:19.639384+0000","last_active":"2023-06-10T21:48:26.422442+0000","last_peered":"2023-06-10T21:48:26.422442+0000","last_clean":"2023-06-10T21:47:17.257047+0000","last_became_active":"2023-06-10T21:47:19.603771+0000","last_became_peered":"2023-06-10T21:47:19.603771+0000","last_unstale":"2023-06-10T21:48:26.422442+0000","last_undegraded":"2023-06-10T21:48:26.422442+0000","last_fullsized":"2023-06-10T21:48:26.422442+0000","mapping_epoch":439,"log_start":"428'2700","ondisk_log_start":"428'2700","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6186,"log_dups_size":0,"ondisk_log_size":6186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:38:04.341765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285343744,"num_objects":4354,"num_object_clones":0,"num_object_copies":13062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4354,"num_objects_unfound":0,"num_objects_dirty":4354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8886,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":4354}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":121,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.939663+0000","last_change":"2023-06-10T21:47:18.440477+0000","last_active":"2023-06-10T21:48:09.939663+0000","last_peered":"2023-06-10T21:48:09.939663+0000","last_clean":"2023-06-10T21:48:09.939663+0000","last_became_active":"2023-06-10T21:47:18.339247+0000","last_became_peered":"2023-06-10T21:47:18.339247+0000","last_unstale":"2023-06-10T21:48:09.939663+0000","last_undegraded":"2023-06-10T21:48:09.939663+0000","last_fullsized":"2023-06-10T21:48:09.939663+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:44:14.648072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":104,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935594+0000","last_change":"2023-06-10T21:40:30.273072+0000","last_active":"2023-06-10T21:48:09.935594+0000","last_peered":"2023-06-10T21:48:09.935594+0000","last_clean":"2023-06-10T21:48:09.935594+0000","last_became_active":"2023-06-10T21:40:30.272733+0000","last_became_peered":"2023-06-10T21:40:30.272733+0000","last_unstale":"2023-06-10T21:48:09.935594+0000","last_undegraded":"2023-06-10T21:48:09.935594+0000","last_fullsized":"2023-06-10T21:48:09.935594+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:34:42.540153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"449'8994","reported_seq":11671,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.349549+0000","last_change":"2023-06-10T21:47:33.561938+0000","last_active":"2023-06-10T21:48:26.349549+0000","last_peered":"2023-06-10T21:48:26.349549+0000","last_clean":"2023-06-10T21:47:17.256756+0000","last_became_active":"2023-06-10T21:47:33.555979+0000","last_became_peered":"2023-06-10T21:47:33.555979+0000","last_unstale":"2023-06-10T21:48:26.349549+0000","last_undegraded":"2023-06-10T21:48:26.349549+0000","last_fullsized":"2023-06-10T21:48:26.349549+0000","mapping_epoch":443,"log_start":"428'2800","ondisk_log_start":"428'2800","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:43.006037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7973,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1307,"num_bytes_recovered":85655552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":4430}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":280,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936165+0000","last_change":"2023-06-10T21:47:33.346678+0000","last_active":"2023-06-10T21:48:09.936165+0000","last_peered":"2023-06-10T21:48:09.936165+0000","last_clean":"2023-06-10T21:48:09.936165+0000","last_became_active":"2023-06-10T21:47:33.346339+0000","last_became_peered":"2023-06-10T21:47:33.346339+0000","last_unstale":"2023-06-10T21:48:09.936165+0000","last_undegraded":"2023-06-10T21:48:09.936165+0000","last_fullsized":"2023-06-10T21:48:09.936165+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:57:33.958041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":497,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.950439+0000","last_change":"2023-06-10T21:47:18.338213+0000","last_active":"2023-06-10T21:48:09.950439+0000","last_peered":"2023-06-10T21:48:09.950439+0000","last_clean":"2023-06-10T21:48:09.950439+0000","last_became_active":"2023-06-10T21:47:18.337970+0000","last_became_peered":"2023-06-10T21:47:18.337970+0000","last_unstale":"2023-06-10T21:48:09.950439+0000","last_undegraded":"2023-06-10T21:48:09.950439+0000","last_fullsized":"2023-06-10T21:48:09.950439+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:23:12.041464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"449'8857","reported_seq":8904,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:24.302885+0000","last_change":"2023-06-10T21:47:34.444993+0000","last_active":"2023-06-10T21:48:24.302885+0000","last_peered":"2023-06-10T21:48:24.302885+0000","last_clean":"2023-06-10T21:47:32.300137+0000","last_became_active":"2023-06-10T21:47:34.433837+0000","last_became_peered":"2023-06-10T21:47:34.433837+0000","last_unstale":"2023-06-10T21:48:24.302885+0000","last_undegraded":"2023-06-10T21:48:24.302885+0000","last_fullsized":"2023-06-10T21:48:24.302885+0000","mapping_epoch":444,"log_start":"430'3700","ondisk_log_start":"430'3700","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5157,"log_dups_size":0,"ondisk_log_size":5157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294715392,"num_objects":4497,"num_object_clones":0,"num_object_copies":13491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4497,"num_objects_unfound":0,"num_objects_dirty":4497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8857,"num_write_kb":427328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,5],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":4497}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954256+0000","last_change":"2023-06-10T21:43:01.253252+0000","last_active":"2023-06-10T21:48:09.954256+0000","last_peered":"2023-06-10T21:48:09.954256+0000","last_clean":"2023-06-10T21:48:09.954256+0000","last_became_active":"2023-06-10T21:43:01.252764+0000","last_became_peered":"2023-06-10T21:43:01.252764+0000","last_unstale":"2023-06-10T21:48:09.954256+0000","last_undegraded":"2023-06-10T21:48:09.954256+0000","last_fullsized":"2023-06-10T21:48:09.954256+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:54:39.712088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936117+0000","last_change":"2023-06-10T21:43:01.210090+0000","last_active":"2023-06-10T21:48:09.936117+0000","last_peered":"2023-06-10T21:48:09.936117+0000","last_clean":"2023-06-10T21:48:09.936117+0000","last_became_active":"2023-06-10T21:43:01.209795+0000","last_became_peered":"2023-06-10T21:43:01.209795+0000","last_unstale":"2023-06-10T21:48:09.936117+0000","last_undegraded":"2023-06-10T21:48:09.936117+0000","last_fullsized":"2023-06-10T21:48:09.936117+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:37.239929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.5","version":"449'9139","reported_seq":9186,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.304235+0000","last_change":"2023-06-10T21:47:33.369686+0000","last_active":"2023-06-10T21:48:26.304235+0000","last_peered":"2023-06-10T21:48:26.304235+0000","last_clean":"2023-06-10T21:47:17.264551+0000","last_became_active":"2023-06-10T21:47:19.619202+0000","last_became_peered":"2023-06-10T21:47:19.619202+0000","last_unstale":"2023-06-10T21:48:26.304235+0000","last_undegraded":"2023-06-10T21:48:26.304235+0000","last_fullsized":"2023-06-10T21:48:26.304235+0000","mapping_epoch":439,"log_start":"430'3600","ondisk_log_start":"430'3600","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:32.972643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291045376,"num_objects":4441,"num_object_clones":0,"num_object_copies":13323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4441,"num_objects_unfound":0,"num_objects_dirty":4441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9139,"num_write_kb":434560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":4441}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":305,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936486+0000","last_change":"2023-06-10T21:47:33.342394+0000","last_active":"2023-06-10T21:48:09.936486+0000","last_peered":"2023-06-10T21:48:09.936486+0000","last_clean":"2023-06-10T21:48:09.936486+0000","last_became_active":"2023-06-10T21:47:33.342086+0000","last_became_peered":"2023-06-10T21:47:33.342086+0000","last_unstale":"2023-06-10T21:48:09.936486+0000","last_undegraded":"2023-06-10T21:48:09.936486+0000","last_fullsized":"2023-06-10T21:48:09.936486+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":398,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936109+0000","last_change":"2023-06-10T21:43:01.212353+0000","last_active":"2023-06-10T21:48:09.936109+0000","last_peered":"2023-06-10T21:48:09.936109+0000","last_clean":"2023-06-10T21:48:09.936109+0000","last_became_active":"2023-06-10T21:43:01.211584+0000","last_became_peered":"2023-06-10T21:43:01.211584+0000","last_unstale":"2023-06-10T21:48:09.936109+0000","last_undegraded":"2023-06-10T21:48:09.936109+0000","last_fullsized":"2023-06-10T21:48:09.936109+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:44:49.153257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"449'9089","reported_seq":10193,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.299425+0000","last_change":"2023-06-10T21:47:34.372053+0000","last_active":"2023-06-10T21:48:26.299425+0000","last_peered":"2023-06-10T21:48:26.299425+0000","last_clean":"2023-06-10T21:47:17.280051+0000","last_became_active":"2023-06-10T21:47:19.616748+0000","last_became_peered":"2023-06-10T21:47:19.616748+0000","last_unstale":"2023-06-10T21:48:26.299425+0000","last_undegraded":"2023-06-10T21:48:26.299425+0000","last_fullsized":"2023-06-10T21:48:26.299425+0000","mapping_epoch":439,"log_start":"428'2900","ondisk_log_start":"428'2900","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:17.444409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291045376,"num_objects":4441,"num_object_clones":0,"num_object_copies":13323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4091,"num_objects_unfound":0,"num_objects_dirty":4441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9089,"num_write_kb":432960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":529,"num_bytes_recovered":34668544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":4441}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.955796+0000","last_change":"2023-06-10T21:47:18.325919+0000","last_active":"2023-06-10T21:48:09.955796+0000","last_peered":"2023-06-10T21:48:09.955796+0000","last_clean":"2023-06-10T21:48:09.955796+0000","last_became_active":"2023-06-10T21:47:18.325473+0000","last_became_peered":"2023-06-10T21:47:18.325473+0000","last_unstale":"2023-06-10T21:48:09.955796+0000","last_undegraded":"2023-06-10T21:48:09.955796+0000","last_fullsized":"2023-06-10T21:48:09.955796+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:29:40.800760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":81,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:21.344344+0000","last_change":"2023-06-10T21:46:14.277043+0000","last_active":"2023-06-10T21:48:21.344344+0000","last_peered":"2023-06-10T21:48:21.344344+0000","last_clean":"2023-06-10T21:48:21.344344+0000","last_became_active":"2023-06-10T21:46:14.276737+0000","last_became_peered":"2023-06-10T21:46:14.276737+0000","last_unstale":"2023-06-10T21:48:21.344344+0000","last_undegraded":"2023-06-10T21:48:21.344344+0000","last_fullsized":"2023-06-10T21:48:21.344344+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:03:03.540332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.955254+0000","last_change":"2023-06-10T21:47:33.366140+0000","last_active":"2023-06-10T21:48:09.955254+0000","last_peered":"2023-06-10T21:48:09.955254+0000","last_clean":"2023-06-10T21:48:09.955254+0000","last_became_active":"2023-06-10T21:47:33.365711+0000","last_became_peered":"2023-06-10T21:47:33.365711+0000","last_unstale":"2023-06-10T21:48:09.955254+0000","last_undegraded":"2023-06-10T21:48:09.955254+0000","last_fullsized":"2023-06-10T21:48:09.955254+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:02:40.888164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.e","version":"449'9224","reported_seq":9285,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.335881+0000","last_change":"2023-06-10T21:47:34.413294+0000","last_active":"2023-06-10T21:48:26.335881+0000","last_peered":"2023-06-10T21:48:26.335881+0000","last_clean":"2023-06-10T21:47:17.288812+0000","last_became_active":"2023-06-10T21:47:34.372854+0000","last_became_peered":"2023-06-10T21:47:34.372854+0000","last_unstale":"2023-06-10T21:48:26.335881+0000","last_undegraded":"2023-06-10T21:48:26.335881+0000","last_fullsized":"2023-06-10T21:48:26.335881+0000","mapping_epoch":444,"log_start":"428'3100","ondisk_log_start":"428'3100","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6124,"log_dups_size":0,"ondisk_log_size":6124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:16:13.424452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301989888,"num_objects":4608,"num_object_clones":0,"num_object_copies":13824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4608,"num_objects_unfound":0,"num_objects_dirty":4608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9224,"num_write_kb":442624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":4608}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.955330+0000","last_change":"2023-06-10T21:47:33.345775+0000","last_active":"2023-06-10T21:48:09.955330+0000","last_peered":"2023-06-10T21:48:09.955330+0000","last_clean":"2023-06-10T21:48:09.955330+0000","last_became_active":"2023-06-10T21:47:33.345396+0000","last_became_peered":"2023-06-10T21:47:33.345396+0000","last_unstale":"2023-06-10T21:48:09.955330+0000","last_undegraded":"2023-06-10T21:48:09.955330+0000","last_fullsized":"2023-06-10T21:48:09.955330+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:02:01.664092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956607+0000","last_change":"2023-06-10T21:47:18.326316+0000","last_active":"2023-06-10T21:48:09.956607+0000","last_peered":"2023-06-10T21:48:09.956607+0000","last_clean":"2023-06-10T21:48:09.956607+0000","last_became_active":"2023-06-10T21:47:18.326038+0000","last_became_peered":"2023-06-10T21:47:18.326038+0000","last_unstale":"2023-06-10T21:48:09.956607+0000","last_undegraded":"2023-06-10T21:48:09.956607+0000","last_fullsized":"2023-06-10T21:48:09.956607+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:23:31.873946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":315,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.946591+0000","last_change":"2023-06-10T21:40:30.238521+0000","last_active":"2023-06-10T21:48:09.946591+0000","last_peered":"2023-06-10T21:48:09.946591+0000","last_clean":"2023-06-10T21:48:09.946591+0000","last_became_active":"2023-06-10T21:40:30.238147+0000","last_became_peered":"2023-06-10T21:40:30.238147+0000","last_unstale":"2023-06-10T21:48:09.946591+0000","last_undegraded":"2023-06-10T21:48:09.946591+0000","last_fullsized":"2023-06-10T21:48:09.946591+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:35:34.873141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"449'9101","reported_seq":9156,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.409485+0000","last_change":"2023-06-10T21:47:34.473707+0000","last_active":"2023-06-10T21:48:26.409485+0000","last_peered":"2023-06-10T21:48:26.409485+0000","last_clean":"2023-06-10T21:47:32.290559+0000","last_became_active":"2023-06-10T21:47:34.462123+0000","last_became_peered":"2023-06-10T21:47:34.462123+0000","last_unstale":"2023-06-10T21:48:26.409485+0000","last_undegraded":"2023-06-10T21:48:26.409485+0000","last_fullsized":"2023-06-10T21:48:26.409485+0000","mapping_epoch":444,"log_start":"428'2900","ondisk_log_start":"428'2900","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6201,"log_dups_size":0,"ondisk_log_size":6201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:41:32.162964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288227352,"num_objects":4399,"num_object_clones":0,"num_object_copies":13197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4399,"num_objects_unfound":0,"num_objects_dirty":4399,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9101,"num_write_kb":431937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":4399}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":344,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954174+0000","last_change":"2023-06-10T21:47:18.337074+0000","last_active":"2023-06-10T21:48:09.954174+0000","last_peered":"2023-06-10T21:48:09.954174+0000","last_clean":"2023-06-10T21:48:09.954174+0000","last_became_active":"2023-06-10T21:47:18.336698+0000","last_became_peered":"2023-06-10T21:47:18.336698+0000","last_unstale":"2023-06-10T21:48:09.954174+0000","last_undegraded":"2023-06-10T21:48:09.954174+0000","last_fullsized":"2023-06-10T21:48:09.954174+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:58.371008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":349,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.946735+0000","last_change":"2023-06-10T21:47:18.337618+0000","last_active":"2023-06-10T21:48:09.946735+0000","last_peered":"2023-06-10T21:48:09.946735+0000","last_clean":"2023-06-10T21:48:09.946735+0000","last_became_active":"2023-06-10T21:47:18.337335+0000","last_became_peered":"2023-06-10T21:47:18.337335+0000","last_unstale":"2023-06-10T21:48:09.946735+0000","last_undegraded":"2023-06-10T21:48:09.946735+0000","last_fullsized":"2023-06-10T21:48:09.946735+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:45:38.349933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954619+0000","last_change":"2023-06-10T21:47:18.312334+0000","last_active":"2023-06-10T21:48:09.954619+0000","last_peered":"2023-06-10T21:48:09.954619+0000","last_clean":"2023-06-10T21:48:09.954619+0000","last_became_active":"2023-06-10T21:47:18.311967+0000","last_became_peered":"2023-06-10T21:47:18.311967+0000","last_unstale":"2023-06-10T21:48:09.954619+0000","last_undegraded":"2023-06-10T21:48:09.954619+0000","last_fullsized":"2023-06-10T21:48:09.954619+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:52:32.880695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.f","version":"449'9046","reported_seq":9083,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-06-10T21:48:24.306131+0000","last_change":"2023-06-10T21:45:44.480803+0000","last_active":"2023-06-10T21:48:24.306131+0000","last_peered":"2023-06-10T21:48:24.306131+0000","last_clean":"2023-06-10T21:48:24.306131+0000","last_became_active":"2023-06-10T21:45:44.480472+0000","last_became_peered":"2023-06-10T21:45:44.480472+0000","last_unstale":"2023-06-10T21:48:24.306131+0000","last_undegraded":"2023-06-10T21:48:24.306131+0000","last_fullsized":"2023-06-10T21:48:24.306131+0000","mapping_epoch":420,"log_start":"431'3929","ondisk_log_start":"431'3929","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289669120,"num_objects":4420,"num_object_clones":0,"num_object_copies":13260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9046,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":406,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-06-10T21:48:21.409428+0000","last_change":"2023-06-10T21:46:14.280999+0000","last_active":"2023-06-10T21:48:21.409428+0000","last_peered":"2023-06-10T21:48:21.409428+0000","last_clean":"2023-06-10T21:48:21.409428+0000","last_became_active":"2023-06-10T21:46:14.280409+0000","last_became_peered":"2023-06-10T21:46:14.280409+0000","last_unstale":"2023-06-10T21:48:21.409428+0000","last_undegraded":"2023-06-10T21:48:21.409428+0000","last_fullsized":"2023-06-10T21:48:21.409428+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:24:46.683725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":672,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936393+0000","last_change":"2023-06-10T21:47:21.365719+0000","last_active":"2023-06-10T21:48:09.936393+0000","last_peered":"2023-06-10T21:48:09.936393+0000","last_clean":"2023-06-10T21:48:09.936393+0000","last_became_active":"2023-06-10T21:47:21.365357+0000","last_became_peered":"2023-06-10T21:47:21.365357+0000","last_unstale":"2023-06-10T21:48:09.936393+0000","last_undegraded":"2023-06-10T21:48:09.936393+0000","last_fullsized":"2023-06-10T21:48:09.936393+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936078+0000","last_change":"2023-06-10T21:43:01.221538+0000","last_active":"2023-06-10T21:48:09.936078+0000","last_peered":"2023-06-10T21:48:09.936078+0000","last_clean":"2023-06-10T21:48:09.936078+0000","last_became_active":"2023-06-10T21:43:01.221154+0000","last_became_peered":"2023-06-10T21:43:01.221154+0000","last_unstale":"2023-06-10T21:48:09.936078+0000","last_undegraded":"2023-06-10T21:48:09.936078+0000","last_fullsized":"2023-06-10T21:48:09.936078+0000","mapping_epoch":390,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:05:46.747480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":350,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.937070+0000","last_change":"2023-06-10T21:47:33.366143+0000","last_active":"2023-06-10T21:48:09.937070+0000","last_peered":"2023-06-10T21:48:09.937070+0000","last_clean":"2023-06-10T21:48:09.937070+0000","last_became_active":"2023-06-10T21:47:33.365660+0000","last_became_peered":"2023-06-10T21:47:33.365660+0000","last_unstale":"2023-06-10T21:48:09.937070+0000","last_undegraded":"2023-06-10T21:48:09.937070+0000","last_fullsized":"2023-06-10T21:48:09.937070+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.c","version":"449'9015","reported_seq":9075,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.409879+0000","last_change":"2023-06-10T21:47:33.419657+0000","last_active":"2023-06-10T21:48:26.409879+0000","last_peered":"2023-06-10T21:48:26.409879+0000","last_clean":"2023-06-10T21:47:17.279622+0000","last_became_active":"2023-06-10T21:47:33.367457+0000","last_became_peered":"2023-06-10T21:47:33.367457+0000","last_unstale":"2023-06-10T21:48:26.409879+0000","last_undegraded":"2023-06-10T21:48:26.409879+0000","last_fullsized":"2023-06-10T21:48:26.409879+0000","mapping_epoch":443,"log_start":"428'2800","ondisk_log_start":"428'2800","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6215,"log_dups_size":0,"ondisk_log_size":6215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:12:30.562028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288423936,"num_objects":4401,"num_object_clones":0,"num_object_copies":13203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8802,"num_objects_unfound":0,"num_objects_dirty":4401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9015,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":4401}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":510,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936436+0000","last_change":"2023-06-10T21:47:18.326869+0000","last_active":"2023-06-10T21:48:09.936436+0000","last_peered":"2023-06-10T21:48:09.936436+0000","last_clean":"2023-06-10T21:48:09.936436+0000","last_became_active":"2023-06-10T21:47:18.326544+0000","last_became_peered":"2023-06-10T21:47:18.326544+0000","last_unstale":"2023-06-10T21:48:09.936436+0000","last_undegraded":"2023-06-10T21:48:09.936436+0000","last_fullsized":"2023-06-10T21:48:09.936436+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":30,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.952713+0000","last_change":"2023-06-10T21:47:33.364421+0000","last_active":"2023-06-10T21:48:09.952713+0000","last_peered":"2023-06-10T21:48:09.952713+0000","last_clean":"2023-06-10T21:48:09.952713+0000","last_became_active":"2023-06-10T21:47:33.364107+0000","last_became_peered":"2023-06-10T21:47:33.364107+0000","last_unstale":"2023-06-10T21:48:09.952713+0000","last_undegraded":"2023-06-10T21:48:09.952713+0000","last_fullsized":"2023-06-10T21:48:09.952713+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:00:55.559846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956263+0000","last_change":"2023-06-10T21:47:18.334732+0000","last_active":"2023-06-10T21:48:09.956263+0000","last_peered":"2023-06-10T21:48:09.956263+0000","last_clean":"2023-06-10T21:48:09.956263+0000","last_became_active":"2023-06-10T21:47:18.334458+0000","last_became_peered":"2023-06-10T21:47:18.334458+0000","last_unstale":"2023-06-10T21:48:09.956263+0000","last_undegraded":"2023-06-10T21:48:09.956263+0000","last_fullsized":"2023-06-10T21:48:09.956263+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:31.688908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"449'9132","reported_seq":10178,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.324295+0000","last_change":"2023-06-10T21:47:34.440526+0000","last_active":"2023-06-10T21:48:26.324295+0000","last_peered":"2023-06-10T21:48:26.324295+0000","last_clean":"2023-06-10T21:47:17.293084+0000","last_became_active":"2023-06-10T21:47:19.547131+0000","last_became_peered":"2023-06-10T21:47:19.547131+0000","last_unstale":"2023-06-10T21:48:26.324295+0000","last_undegraded":"2023-06-10T21:48:26.324295+0000","last_fullsized":"2023-06-10T21:48:26.324295+0000","mapping_epoch":439,"log_start":"430'3600","ondisk_log_start":"430'3600","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5532,"log_dups_size":0,"ondisk_log_size":5532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T09:16:25.568267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289669120,"num_objects":4420,"num_object_clones":0,"num_object_copies":13260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4088,"num_objects_unfound":0,"num_objects_dirty":4420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9132,"num_write_kb":433664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":492,"num_bytes_recovered":32243712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,2],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":4420}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":498,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.937066+0000","last_change":"2023-06-10T21:47:33.366361+0000","last_active":"2023-06-10T21:48:09.937066+0000","last_peered":"2023-06-10T21:48:09.937066+0000","last_clean":"2023-06-10T21:48:09.937066+0000","last_became_active":"2023-06-10T21:47:33.366066+0000","last_became_peered":"2023-06-10T21:47:33.366066+0000","last_unstale":"2023-06-10T21:48:09.937066+0000","last_undegraded":"2023-06-10T21:48:09.937066+0000","last_fullsized":"2023-06-10T21:48:09.937066+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":339,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.946669+0000","last_change":"2023-06-10T21:47:18.343163+0000","last_active":"2023-06-10T21:48:09.946669+0000","last_peered":"2023-06-10T21:48:09.946669+0000","last_clean":"2023-06-10T21:48:09.946669+0000","last_became_active":"2023-06-10T21:47:18.341580+0000","last_became_peered":"2023-06-10T21:47:18.341580+0000","last_unstale":"2023-06-10T21:48:09.946669+0000","last_undegraded":"2023-06-10T21:48:09.946669+0000","last_fullsized":"2023-06-10T21:48:09.946669+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:49:42.047017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956513+0000","last_change":"2023-06-10T21:47:18.312934+0000","last_active":"2023-06-10T21:48:09.956513+0000","last_peered":"2023-06-10T21:48:09.956513+0000","last_clean":"2023-06-10T21:48:09.956513+0000","last_became_active":"2023-06-10T21:47:18.312671+0000","last_became_peered":"2023-06-10T21:47:18.312671+0000","last_unstale":"2023-06-10T21:48:09.956513+0000","last_undegraded":"2023-06-10T21:48:09.956513+0000","last_fullsized":"2023-06-10T21:48:09.956513+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:39:15.202689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"449'9045","reported_seq":9091,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-06-10T21:48:26.265379+0000","last_change":"2023-06-10T21:46:14.285180+0000","last_active":"2023-06-10T21:48:26.265379+0000","last_peered":"2023-06-10T21:48:26.265379+0000","last_clean":"2023-06-10T21:48:26.265379+0000","last_became_active":"2023-06-10T21:46:14.279416+0000","last_became_peered":"2023-06-10T21:46:14.279416+0000","last_unstale":"2023-06-10T21:48:26.265379+0000","last_undegraded":"2023-06-10T21:48:26.265379+0000","last_fullsized":"2023-06-10T21:48:26.265379+0000","mapping_epoch":426,"log_start":"430'3500","ondisk_log_start":"430'3500","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:05:25.130576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286851072,"num_objects":4377,"num_object_clones":0,"num_object_copies":13131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9045,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":474,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-06-10T21:48:21.414282+0000","last_change":"2023-06-10T21:47:18.433743+0000","last_active":"2023-06-10T21:48:21.414282+0000","last_peered":"2023-06-10T21:48:21.414282+0000","last_clean":"2023-06-10T21:48:21.414282+0000","last_became_active":"2023-06-10T21:47:18.338606+0000","last_became_peered":"2023-06-10T21:47:18.338606+0000","last_unstale":"2023-06-10T21:48:21.414282+0000","last_undegraded":"2023-06-10T21:48:21.414282+0000","last_fullsized":"2023-06-10T21:48:21.414282+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:13:02.602599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":281,"reported_epoch":449,"state":"active+clean","last_fresh":"2023-06-10T21:48:21.415686+0000","last_change":"2023-06-10T21:47:18.339369+0000","last_active":"2023-06-10T21:48:21.415686+0000","last_peered":"2023-06-10T21:48:21.415686+0000","last_clean":"2023-06-10T21:48:21.415686+0000","last_became_active":"2023-06-10T21:47:18.339023+0000","last_became_peered":"2023-06-10T21:47:18.339023+0000","last_unstale":"2023-06-10T21:48:21.415686+0000","last_undegraded":"2023-06-10T21:48:21.415686+0000","last_fullsized":"2023-06-10T21:48:21.415686+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:46:02.440868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":309,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936305+0000","last_change":"2023-06-10T21:47:33.345545+0000","last_active":"2023-06-10T21:48:09.936305+0000","last_peered":"2023-06-10T21:48:09.936305+0000","last_clean":"2023-06-10T21:48:09.936305+0000","last_became_active":"2023-06-10T21:47:33.345220+0000","last_became_peered":"2023-06-10T21:47:33.345220+0000","last_unstale":"2023-06-10T21:48:09.936305+0000","last_undegraded":"2023-06-10T21:48:09.936305+0000","last_fullsized":"2023-06-10T21:48:09.936305+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"449'8994","reported_seq":9045,"reported_epoch":449,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:48:26.384844+0000","last_change":"2023-06-10T21:47:19.676197+0000","last_active":"2023-06-10T21:48:26.384844+0000","last_peered":"2023-06-10T21:48:26.384844+0000","last_clean":"2023-06-10T21:47:17.285378+0000","last_became_active":"2023-06-10T21:47:19.620122+0000","last_became_peered":"2023-06-10T21:47:19.620122+0000","last_unstale":"2023-06-10T21:48:26.384844+0000","last_undegraded":"2023-06-10T21:48:26.384844+0000","last_fullsized":"2023-06-10T21:48:26.384844+0000","mapping_epoch":439,"log_start":"427'2800","ondisk_log_start":"427'2800","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6194,"log_dups_size":0,"ondisk_log_size":6194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:36.961451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282329088,"num_objects":4308,"num_object_clones":0,"num_object_copies":12924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4308,"num_objects_unfound":0,"num_objects_dirty":4308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8994,"num_write_kb":425664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":4308}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":274,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.951437+0000","last_change":"2023-06-10T21:47:33.442304+0000","last_active":"2023-06-10T21:48:09.951437+0000","last_peered":"2023-06-10T21:48:09.951437+0000","last_clean":"2023-06-10T21:48:09.951437+0000","last_became_active":"2023-06-10T21:47:33.441907+0000","last_became_peered":"2023-06-10T21:47:33.441907+0000","last_unstale":"2023-06-10T21:48:09.951437+0000","last_undegraded":"2023-06-10T21:48:09.951437+0000","last_fullsized":"2023-06-10T21:48:09.951437+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:25:27.531977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":39,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956113+0000","last_change":"2023-06-10T21:47:33.433623+0000","last_active":"2023-06-10T21:48:09.956113+0000","last_peered":"2023-06-10T21:48:09.956113+0000","last_clean":"2023-06-10T21:48:09.956113+0000","last_became_active":"2023-06-10T21:47:33.372099+0000","last_became_peered":"2023-06-10T21:47:33.372099+0000","last_unstale":"2023-06-10T21:48:09.956113+0000","last_undegraded":"2023-06-10T21:48:09.956113+0000","last_fullsized":"2023-06-10T21:48:09.956113+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:33:58.316182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.952781+0000","last_change":"2023-06-10T21:47:18.316764+0000","last_active":"2023-06-10T21:48:09.952781+0000","last_peered":"2023-06-10T21:48:09.952781+0000","last_clean":"2023-06-10T21:48:09.952781+0000","last_became_active":"2023-06-10T21:47:18.316333+0000","last_became_peered":"2023-06-10T21:47:18.316333+0000","last_unstale":"2023-06-10T21:48:09.952781+0000","last_undegraded":"2023-06-10T21:48:09.952781+0000","last_fullsized":"2023-06-10T21:48:09.952781+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:22:30.996515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":21,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.956425+0000","last_change":"2023-06-10T21:47:18.324336+0000","last_active":"2023-06-10T21:48:09.956425+0000","last_peered":"2023-06-10T21:48:09.956425+0000","last_clean":"2023-06-10T21:48:09.956425+0000","last_became_active":"2023-06-10T21:47:18.321834+0000","last_became_peered":"2023-06-10T21:47:18.321834+0000","last_unstale":"2023-06-10T21:48:09.956425+0000","last_undegraded":"2023-06-10T21:48:09.956425+0000","last_fullsized":"2023-06-10T21:48:09.956425+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:40.514667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":353,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.935739+0000","last_change":"2023-06-10T21:47:33.347302+0000","last_active":"2023-06-10T21:48:09.935739+0000","last_peered":"2023-06-10T21:48:09.935739+0000","last_clean":"2023-06-10T21:48:09.935739+0000","last_became_active":"2023-06-10T21:47:33.346982+0000","last_became_peered":"2023-06-10T21:47:33.346982+0000","last_unstale":"2023-06-10T21:48:09.935739+0000","last_undegraded":"2023-06-10T21:48:09.935739+0000","last_fullsized":"2023-06-10T21:48:09.935739+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:47:16.801321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":117,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.938135+0000","last_change":"2023-06-10T21:46:14.283333+0000","last_active":"2023-06-10T21:48:09.938135+0000","last_peered":"2023-06-10T21:48:09.938135+0000","last_clean":"2023-06-10T21:48:09.938135+0000","last_became_active":"2023-06-10T21:46:14.282987+0000","last_became_peered":"2023-06-10T21:46:14.282987+0000","last_unstale":"2023-06-10T21:48:09.938135+0000","last_undegraded":"2023-06-10T21:48:09.938135+0000","last_fullsized":"2023-06-10T21:48:09.938135+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":354,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.939798+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:48:09.939798+0000","last_peered":"2023-06-10T21:48:09.939798+0000","last_clean":"2023-06-10T21:48:09.939798+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:48:09.939798+0000","last_undegraded":"2023-06-10T21:48:09.939798+0000","last_fullsized":"2023-06-10T21:48:09.939798+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936875+0000","last_change":"2023-06-10T21:46:14.279723+0000","last_active":"2023-06-10T21:48:09.936875+0000","last_peered":"2023-06-10T21:48:09.936875+0000","last_clean":"2023-06-10T21:48:09.936875+0000","last_became_active":"2023-06-10T21:46:14.279371+0000","last_became_peered":"2023-06-10T21:46:14.279371+0000","last_unstale":"2023-06-10T21:48:09.936875+0000","last_undegraded":"2023-06-10T21:48:09.936875+0000","last_fullsized":"2023-06-10T21:48:09.936875+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:11:43.249586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":434,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936237+0000","last_change":"2023-06-10T21:47:18.335340+0000","last_active":"2023-06-10T21:48:09.936237+0000","last_peered":"2023-06-10T21:48:09.936237+0000","last_clean":"2023-06-10T21:48:09.936237+0000","last_became_active":"2023-06-10T21:47:18.335048+0000","last_became_peered":"2023-06-10T21:47:18.335048+0000","last_unstale":"2023-06-10T21:48:09.936237+0000","last_undegraded":"2023-06-10T21:48:09.936237+0000","last_fullsized":"2023-06-10T21:48:09.936237+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:11:27.198035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":104,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936269+0000","last_change":"2023-06-10T21:40:30.351974+0000","last_active":"2023-06-10T21:48:09.936269+0000","last_peered":"2023-06-10T21:48:09.936269+0000","last_clean":"2023-06-10T21:48:09.936269+0000","last_became_active":"2023-06-10T21:40:30.350847+0000","last_became_peered":"2023-06-10T21:40:30.350847+0000","last_unstale":"2023-06-10T21:48:09.936269+0000","last_undegraded":"2023-06-10T21:48:09.936269+0000","last_fullsized":"2023-06-10T21:48:09.936269+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:36:30.843833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":128,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.940229+0000","last_change":"2023-06-10T21:47:18.344279+0000","last_active":"2023-06-10T21:48:09.940229+0000","last_peered":"2023-06-10T21:48:09.940229+0000","last_clean":"2023-06-10T21:48:09.940229+0000","last_became_active":"2023-06-10T21:47:18.343646+0000","last_became_peered":"2023-06-10T21:47:18.343646+0000","last_unstale":"2023-06-10T21:48:09.940229+0000","last_undegraded":"2023-06-10T21:48:09.940229+0000","last_fullsized":"2023-06-10T21:48:09.940229+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:43.284066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":42,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.954461+0000","last_change":"2023-06-10T21:47:33.356600+0000","last_active":"2023-06-10T21:48:09.954461+0000","last_peered":"2023-06-10T21:48:09.954461+0000","last_clean":"2023-06-10T21:48:09.954461+0000","last_became_active":"2023-06-10T21:47:33.346441+0000","last_became_peered":"2023-06-10T21:47:33.346441+0000","last_unstale":"2023-06-10T21:48:09.954461+0000","last_undegraded":"2023-06-10T21:48:09.954461+0000","last_fullsized":"2023-06-10T21:48:09.954461+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:18:13.801397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":128,"reported_epoch":448,"state":"active+clean","last_fresh":"2023-06-10T21:48:09.936012+0000","last_change":"2023-06-10T21:47:18.338284+0000","last_active":"2023-06-10T21:48:09.936012+0000","last_peered":"2023-06-10T21:48:09.936012+0000","last_clean":"2023-06-10T21:48:09.936012+0000","last_became_active":"2023-06-10T21:47:18.337963+0000","last_became_peered":"2023-06-10T21:47:18.337963+0000","last_unstale":"2023-06-10T21:48:09.936012+0000","last_undegraded":"2023-06-10T21:48:09.936012+0000","last_fullsized":"2023-06-10T21:48:09.936012+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:00:05.745786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":4646502424,"num_objects":70901,"num_object_clones":0,"num_object_copies":212703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":66004,"num_objects_unfound":0,"num_objects_dirty":70901,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144807,"num_write_kb":6902593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6419,"num_bytes_recovered":420478976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14342500352,"data_stored":14342488136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92878,"ondisk_log_size":92878,"up":47,"acting":48,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":22310,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42309,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430816,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4268096,"kb_used_data":3171896,"kb_used_omap":0,"kb_used_meta":1096192,"kb_avail":89477056,"statfs":{"total":95995035648,"available":91624505344,"internally_reserved":0,"allocated":3248021504,"data_stored":3247294108,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1122500608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:48:04 2023","interfaces":[{"interface":"back","average":{"1min":8.9800000000000004,"5min":2.6019999999999999,"15min":2.2160000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":170.05199999999999,"5min":170.05199999999999,"15min":170.05199999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":10.705,"5min":3.3319999999999999,"15min":2.7629999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":170.126,"5min":170.126,"15min":170.126},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:47:34 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.3089999999999999,"15min":2.3180000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25600000000000001},"max":{"1min":3.609,"5min":8.1500000000000004,"15min":60.863},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.9119999999999999,"5min":1.653,"15min":2.7050000000000001},"min":{"1min":0.38,"5min":0.29999999999999999,"15min":0.27300000000000002},"max":{"1min":8.8680000000000003,"5min":10.484,"15min":61.148000000000003},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:47:45 2023","interfaces":[{"interface":"back","average":{"1min":6.8010000000000002,"5min":9.5020000000000007,"15min":6.8650000000000002},"min":{"1min":0.41499999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":96.338999999999999,"5min":218.90600000000001,"15min":218.90600000000001},"last":30.734000000000002},{"interface":"front","average":{"1min":6.6890000000000001,"5min":9.7040000000000006,"15min":6.8970000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.30499999999999999},"max":{"1min":96.159000000000006,"5min":218.851,"15min":218.851},"last":30.452999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:47:34 2023","interfaces":[{"interface":"back","average":{"1min":6.5060000000000002,"5min":7.0670000000000002,"15min":5.7599999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":77.433999999999997,"5min":218.35300000000001,"15min":218.35300000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":6.7110000000000003,"5min":7.0419999999999998,"15min":5.625},"min":{"1min":0.43099999999999999,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":77.582999999999998,"5min":218.74600000000001,"15min":218.74600000000001},"last":0.442}]},{"osd":4,"last update":"Sat Jun 10 21:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.355,"15min":0.79600000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":3.2669999999999999,"5min":13.429,"15min":13.429},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.3380000000000001,"15min":0.94099999999999995},"min":{"1min":0.30199999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.6680000000000001,"5min":12.279999999999999,"15min":22.77},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:48:14 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":3.5680000000000001,"15min":9.3079999999999998},"min":{"1min":0.317,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.302,"5min":131.619,"15min":131.619},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":3.5499999999999998,"15min":9.1820000000000004},"min":{"1min":0.30099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.823,"5min":134.928,"15min":134.928},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Jun 10 21:48:18 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.083,"15min":0.79800000000000004},"min":{"1min":0.317,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":4.5359999999999996,"5min":7.9660000000000002,"15min":10.890000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.4490000000000001,"15min":0.89400000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.30599999999999999},"max":{"1min":3.3290000000000002,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.58099999999999996}]}]},{"osd":6,"up_from":0,"seq":180388627044,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":450971566487,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":244,"seq":1047972020463,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":868352,"kb_used_data":315640,"kb_used_omap":10,"kb_used_meta":552693,"kb_avail":92876800,"statfs":{"total":95995035648,"available":95105843200,"internally_reserved":0,"allocated":323215360,"data_stored":321731290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":565957684},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.2529999999999999,"15min":2.0779999999999998},"min":{"1min":0.35999999999999999,"5min":0.23699999999999999,"15min":0.189},"max":{"1min":6.2290000000000001,"5min":6.4100000000000001,"15min":27.759},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.456,"5min":1.5249999999999999,"15min":2.7509999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":9.4800000000000004,"5min":9.4800000000000004,"15min":31.419},"last":8.3170000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1619999999999999,"5min":5.4740000000000002,"15min":7.2030000000000003},"min":{"1min":0.45500000000000002,"5min":0.26000000000000001,"15min":0.253},"max":{"1min":7.21,"5min":126.429,"15min":134.47499999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.3170000000000002,"5min":5.1210000000000004,"15min":6.6660000000000004},"min":{"1min":0.36499999999999999,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":7.4779999999999998,"5min":125.831,"15min":132.578},"last":3.605}]},{"osd":3,"last update":"Sat Jun 10 21:47:38 2023","interfaces":[{"interface":"back","average":{"1min":6.8390000000000004,"5min":4.1390000000000002,"15min":5.4160000000000004},"min":{"1min":0.27900000000000003,"5min":0.251,"15min":0.23999999999999999},"max":{"1min":96.906999999999996,"5min":96.906999999999996,"15min":211.864},"last":14.147},{"interface":"front","average":{"1min":6.6429999999999998,"5min":3.9350000000000001,"15min":5.3179999999999996},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":95.140000000000001,"5min":95.140000000000001,"15min":207.517},"last":3.2949999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.381,"15min":2.4049999999999998},"min":{"1min":0.33200000000000002,"5min":0.312,"15min":0.312},"max":{"1min":4.9550000000000001,"5min":7.3710000000000004,"15min":87.054000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.2889999999999999,"15min":2.1669999999999998},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.308},"max":{"1min":6.5179999999999998,"5min":6.6139999999999999,"15min":93.649000000000001},"last":3.206}]},{"osd":5,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3450000000000002,"5min":3.109,"15min":4.8140000000000001},"min":{"1min":0.54000000000000004,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.0369999999999999,"5min":92.149000000000001,"15min":92.149000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":2.0329999999999999,"5min":3.4870000000000001,"15min":5.5220000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":6.5819999999999999,"5min":93.292000000000002,"15min":93.292000000000002},"last":3.3220000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.667,"15min":2.4980000000000002},"min":{"1min":0.435,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.1440000000000001,"5min":7.9059999999999997,"15min":87.019999999999996},"last":0.503},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.6910000000000001,"15min":2.2999999999999998},"min":{"1min":0.443,"5min":0.42999999999999999,"15min":0.373},"max":{"1min":6.6159999999999997,"5min":10.382,"15min":93.614999999999995},"last":3.262}]},{"osd":7,"last update":"Sat Jun 10 21:48:12 2023","interfaces":[{"interface":"back","average":{"1min":11.268000000000001,"5min":3.8250000000000002,"15min":3.766},"min":{"1min":0.53500000000000003,"5min":0.38100000000000001,"15min":0.34100000000000003},"max":{"1min":191.566,"5min":191.566,"15min":191.566},"last":0.73199999999999998},{"interface":"front","average":{"1min":11.032,"5min":4.1849999999999996,"15min":3.9670000000000001},"min":{"1min":0.44,"5min":0.39400000000000002,"15min":0.35399999999999998},"max":{"1min":191.58000000000001,"5min":191.58000000000001,"15min":191.58000000000001},"last":0.53700000000000003}]}]},{"osd":2,"up_from":120,"seq":515396075875,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4005560,"kb_used_data":2866736,"kb_used_omap":7,"kb_used_meta":1138808,"kb_avail":89739592,"statfs":{"total":95995035648,"available":91893342208,"internally_reserved":0,"allocated":2935537664,"data_stored":2934122140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1166139473},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:48:05 2023","interfaces":[{"interface":"back","average":{"1min":7.0149999999999997,"5min":2.629,"15min":4.0910000000000002},"min":{"1min":0.215,"5min":0.20899999999999999,"15min":0.19},"max":{"1min":103.19,"5min":103.19,"15min":171.33699999999999},"last":1.5129999999999999},{"interface":"front","average":{"1min":6.2080000000000002,"5min":2.2200000000000002,"15min":3.6949999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":103.125,"5min":103.125,"15min":171.56200000000001},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:47:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":1.615,"15min":4.4619999999999997},"min":{"1min":0.32200000000000001,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":8.7889999999999997,"5min":11.398999999999999,"15min":171.286},"last":5.593},{"interface":"front","average":{"1min":2.399,"5min":1.7609999999999999,"15min":3.1179999999999999},"min":{"1min":0.30199999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":9.4190000000000005,"5min":9.4190000000000005,"15min":85.501999999999995},"last":1.0029999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:47:27 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":6.8659999999999997,"15min":6.8449999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.5620000000000003,"5min":145.07599999999999,"15min":171.78299999999999},"last":1.3580000000000001},{"interface":"front","average":{"1min":1.9650000000000001,"5min":6.5389999999999997,"15min":6.6050000000000004},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.24399999999999999},"max":{"1min":10.727,"5min":144.99000000000001,"15min":171.65100000000001},"last":1.337}]},{"osd":4,"last update":"Sat Jun 10 21:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.4930000000000001,"15min":2.2210000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":9.577,"5min":9.577,"15min":171.48699999999999},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.5489999999999999,"5min":1.7290000000000001,"15min":3.0870000000000002},"min":{"1min":0.51000000000000001,"5min":0.33600000000000002,"15min":0.29999999999999999},"max":{"1min":8.5190000000000001,"5min":8.5190000000000001,"15min":171.18100000000001},"last":5.4089999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:48:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2309999999999999,"5min":5.8780000000000001,"15min":6.6139999999999999},"min":{"1min":0.47899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":8.9480000000000004,"5min":157.756,"15min":157.756},"last":0.85899999999999999},{"interface":"front","average":{"1min":2.0019999999999998,"5min":5.5289999999999999,"15min":6.0709999999999997},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.5190000000000001,"5min":157.59899999999999,"15min":157.59899999999999},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:47:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.7909999999999999,"15min":2.3730000000000002},"min":{"1min":0.43099999999999999,"5min":0.41999999999999998,"15min":0.307},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":171.441},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.8580000000000001,"15min":3.2000000000000002},"min":{"1min":0.51600000000000001,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":8.4789999999999992,"5min":8.4789999999999992,"15min":171.24600000000001},"last":5.3739999999999997}]},{"osd":7,"last update":"Sat Jun 10 21:47:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4399999999999999,"5min":3.7799999999999998,"15min":4.7199999999999998},"min":{"1min":0.52200000000000002,"5min":0.46500000000000002,"15min":0.34200000000000003},"max":{"1min":9.1050000000000004,"5min":124.071,"15min":219.131},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.806,"5min":3.5419999999999998,"15min":4.524},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.32200000000000001},"max":{"1min":9.6020000000000003,"5min":125.203,"15min":219.19},"last":0.51600000000000001}]}]},{"osd":3,"up_from":250,"seq":1073741824235,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4342080,"kb_used_data":3428472,"kb_used_omap":7,"kb_used_meta":913464,"kb_avail":89403072,"statfs":{"total":95995035648,"available":91548745728,"internally_reserved":0,"allocated":3510755328,"data_stored":3509212142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":935387155},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":1.141,"5min":0.871,"15min":2.5960000000000001},"min":{"1min":0.217,"5min":0.216,"15min":0.19800000000000001},"max":{"1min":6.1669999999999998,"5min":6.46,"15min":105.09},"last":11.358000000000001},{"interface":"front","average":{"1min":2.2160000000000002,"5min":1.698,"15min":2.8769999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":11.638,"5min":11.638,"15min":99.852999999999994},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":1.863,"5min":1.405,"15min":2.9740000000000002},"min":{"1min":0.28199999999999997,"5min":0.247,"15min":0.20499999999999999},"max":{"1min":6.6349999999999998,"5min":6.6459999999999999,"15min":94.766000000000005},"last":4.5919999999999996},{"interface":"front","average":{"1min":1.2,"5min":0.98099999999999998,"15min":3.0230000000000001},"min":{"1min":0.30199999999999999,"5min":0.24399999999999999,"15min":0.23000000000000001},"max":{"1min":6.2640000000000002,"5min":6.2640000000000002,"15min":94.691000000000003},"last":0.309}]},{"osd":2,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6529999999999996,"5min":3.8860000000000001,"15min":4.1550000000000002},"min":{"1min":0.27700000000000002,"5min":0.251,"15min":0.23300000000000001},"max":{"1min":20.829999999999998,"5min":126.913,"15min":231.07900000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.597,"5min":3.8319999999999999,"15min":4.9980000000000002},"min":{"1min":0.33800000000000002,"5min":0.30199999999999999,"15min":0.22800000000000001},"max":{"1min":23.609000000000002,"5min":128.52199999999999,"15min":231.58600000000001},"last":5.4800000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.659,"15min":1.486},"min":{"1min":0.377,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":11.071,"5min":11.071,"15min":13.417},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.369,"5min":1.631,"15min":2.0459999999999998},"min":{"1min":0.39400000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":5.0149999999999997,"5min":8.8379999999999992,"15min":19.244},"last":1.113}]},{"osd":5,"last update":"Sat Jun 10 21:48:09 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.1859999999999999,"15min":1.1220000000000001},"min":{"1min":0.34100000000000003,"5min":0.317,"15min":0.317},"max":{"1min":9.6440000000000001,"5min":9.6440000000000001,"15min":9.6440000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.4319999999999999,"5min":2.415,"15min":2.6070000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":11.032999999999999,"5min":11.032999999999999,"15min":11.032999999999999},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":1.742,"15min":2.1160000000000001},"min":{"1min":0.40699999999999997,"5min":0.34200000000000003,"15min":0.32300000000000001},"max":{"1min":5.0510000000000002,"5min":9.0250000000000004,"15min":19.164999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.238,"15min":1.5389999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.6219999999999999,"5min":6.0750000000000002,"15min":17.327000000000002},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:47:39 2023","interfaces":[{"interface":"back","average":{"1min":1.8169999999999999,"5min":2.7269999999999999,"15min":2.6970000000000001},"min":{"1min":0.46400000000000002,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":5.5490000000000004,"5min":56.317999999999998,"15min":79.984999999999999},"last":1.151},{"interface":"front","average":{"1min":1.1890000000000001,"5min":2.0830000000000002,"15min":2.113},"min":{"1min":0.374,"5min":0.316,"15min":0.316},"max":{"1min":4.593,"5min":56.731999999999999,"15min":80.272999999999996},"last":4.4800000000000004}]}]},{"osd":4,"up_from":0,"seq":128849019500,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":390,"seq":1675037245506,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4728928,"kb_used_data":4323480,"kb_used_omap":11,"kb_used_meta":405428,"kb_avail":89016224,"statfs":{"total":95995035648,"available":91152613376,"internally_reserved":0,"allocated":4427243520,"data_stored":4426543123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":415158774},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.746,"15min":1.968},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":36.256999999999998,"5min":36.256999999999998,"15min":36.256999999999998},"last":2.8820000000000001},{"interface":"front","average":{"1min":3.5350000000000001,"5min":1.677,"15min":1.4399999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":37.081000000000003,"5min":37.081000000000003,"15min":37.081000000000003},"last":2.0489999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.7350000000000001,"15min":2.351},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.472,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":2.758},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.5329999999999999,"15min":2.032},"min":{"1min":0.34399999999999997,"5min":0.308,"15min":0.308},"max":{"1min":3.4449999999999998,"5min":9.7989999999999995,"15min":9.7989999999999995},"last":2.0190000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0979999999999999,"5min":6.8680000000000003,"15min":3.5299999999999998},"min":{"1min":0.309,"5min":0.254,"15min":0.254},"max":{"1min":65.908000000000001,"5min":219.84700000000001,"15min":219.84700000000001},"last":2.069},{"interface":"front","average":{"1min":4.867,"5min":6.7190000000000003,"15min":3.7770000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":65.890000000000001,"5min":216.589,"15min":216.589},"last":1.8859999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":4.8129999999999997,"5min":4.6319999999999997,"15min":7.032},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":66.188999999999993,"5min":133.07499999999999,"15min":133.07499999999999},"last":2.9199999999999999},{"interface":"front","average":{"1min":5.5430000000000001,"5min":4.8499999999999996,"15min":6.8799999999999999},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":68.007000000000005,"5min":133.03800000000001,"15min":133.03800000000001},"last":4.8890000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.0049999999999999,"15min":0.94099999999999995},"min":{"1min":0.33500000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.5539999999999998,"5min":5.9130000000000003,"15min":5.9130000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.49,"15min":2.0350000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.319,"5min":7.476,"15min":7.476},"last":2.7970000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":1.179,"15min":1.651},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":2.411,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":2.2290000000000001},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.125,"15min":1.2709999999999999},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.867,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":0.499}]},{"osd":7,"last update":"Sat Jun 10 21:48:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":1.385,"15min":1.536},"min":{"1min":0.37,"5min":0.25,"15min":0.25},"max":{"1min":3.0920000000000001,"5min":8.6210000000000004,"15min":8.6210000000000004},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":2.1930000000000001,"15min":2.6579999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.2810000000000001,"5min":9.5370000000000008,"15min":9.5370000000000008},"last":2.948}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":320143360,"data_stored":320143360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2903052288,"data_stored":2903048216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3487301632,"data_stored":3487301632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4397731840,"data_stored":4397727768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3234271232,"data_stored":3234267160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:48:32.478 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:48:32.478 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set device_health_metrics pg_num 31 2023-06-10T21:48:32.561 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:32 smithi064 ceph-mon[180297]: pgmap v1664: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 63433/205290 objects misplaced (30.899%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:48:32.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:32 smithi064 ceph-mon[186706]: pgmap v1664: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 63433/205290 objects misplaced (30.899%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:48:32.677 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:32.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:32 smithi152 ceph-mon[156819]: pgmap v1664: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 63433/205290 objects misplaced (30.899%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:48:32.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:32.822 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:32.874 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:32.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:32.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:32.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:33.070 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:33.104 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:32] "GET /metrics HTTP/1.1" 200 224230 "" "Prometheus/2.33.4" 2023-06-10T21:48:33.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:33.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:33.719 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:33.752 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:33.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:33.867 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:33.867 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:33.871 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:33.898 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:33.959 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:33.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:33.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:34.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:34.102 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:34.163 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:34.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:34.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:34.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:34.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:34.394 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:48:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3312424471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[180297]: pgmap v1665: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 63433/205290 objects misplaced (30.899%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:48:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3312424471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:34 smithi064 ceph-mon[186706]: pgmap v1665: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 63433/205290 objects misplaced (30.899%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:48:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3312424471' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:48:34.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:34 smithi152 ceph-mon[156819]: pgmap v1665: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 441 op/s; 63433/205290 objects misplaced (30.899%); 3.6 MiB/s, 57 objects/s recovering 2023-06-10T21:48:34.455 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:35.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:35.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:35.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:35.620 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:35.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:35.637 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:48:35.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: osdmap e450: 8 total, 8 up, 5 in 2023-06-10T21:48:35.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[186706]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:35.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:48:35.647 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: osdmap e450: 8 total, 8 up, 5 in 2023-06-10T21:48:35.648 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:35 smithi064 ceph-mon[180297]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:48:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: osdmap e450: 8 total, 8 up, 5 in 2023-06-10T21:48:35.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:35 smithi152 ceph-mon[156819]: failed to encode map e450 with expected crc 2023-06-10T21:48:35.727 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:36.260 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:48:36.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:36.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:36.500 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:36.566 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:36 smithi064 ceph-mon[186706]: pgmap v1667: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 63233/204132 objects misplaced (30.977%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:48:36.566 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:36 smithi064 ceph-mon[180297]: pgmap v1667: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 63233/204132 objects misplaced (30.977%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:48:36.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:36 smithi152 ceph-mon[156819]: pgmap v1667: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 536 op/s; 63233/204132 objects misplaced (30.977%); 4.3 MiB/s, 68 objects/s recovering 2023-06-10T21:48:36.778 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:36.801 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:36.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:36.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:37.013 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:38.160 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:38.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:38.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:38.273 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:38.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:38.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:38 smithi152 ceph-mon[156819]: pgmap v1668: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 539 op/s; 60453/196575 objects misplaced (30.753%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:48:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:38 smithi064 ceph-mon[186706]: pgmap v1668: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 539 op/s; 60453/196575 objects misplaced (30.753%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:48:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:38 smithi064 ceph-mon[180297]: pgmap v1668: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 539 op/s; 60453/196575 objects misplaced (30.753%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:48:38.689 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:38.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:38.794 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:38.833 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:48:38.875 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:39.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:39.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:39.228 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:39.262 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:39.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:39.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:39.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:39.521 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:39.531 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:39.575 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:39.644 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:39.646 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:39.646 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4058009676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:39.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4058009676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:39.670 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4058009676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:39.729 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:39.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:39.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:39.784 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:39.828 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:39.866 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:39.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:39.892 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:39.911 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.912 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:48:39.913 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:39.931 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:39.956 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:39.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:40.056 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-10T21:48:40.061 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:40.140 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:40.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:48:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: osdmap e451: 8 total, 8 up, 5 in 2023-06-10T21:48:40.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/250421133' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:40 smithi152 ceph-mon[156819]: pgmap v1670: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 60279/195498 objects misplaced (30.834%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:48:40.781 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:40.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: osdmap e451: 8 total, 8 up, 5 in 2023-06-10T21:48:40.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/250421133' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[186706]: pgmap v1670: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 60279/195498 objects misplaced (30.834%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: osdmap e451: 8 total, 8 up, 5 in 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:40.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: failed to encode map e451 with expected crc 2023-06-10T21:48:40.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/250421133' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-06-10T21:48:40.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:40 smithi064 ceph-mon[180297]: pgmap v1670: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 407 op/s; 60279/195498 objects misplaced (30.834%); 2.5 MiB/s, 39 objects/s recovering 2023-06-10T21:48:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: osdmap e452: 8 total, 8 up, 5 in 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:48:41.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: osdmap e452: 8 total, 8 up, 5 in 2023-06-10T21:48:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:48:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:48:41.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: osdmap e452: 8 total, 8 up, 5 in 2023-06-10T21:48:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: failed to encode map e452 with expected crc 2023-06-10T21:48:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:48:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:48:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:42.045 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:42.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:42.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:42.150 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:42.184 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:42.248 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-10T21:48:42.260 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:42.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:42.366 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:42.456 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:48:42.457 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:48:42.457 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:48:42.457 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:48:42.457 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:48:42.457 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:42.607 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:42.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:42 smithi064 ceph-mon[180297]: failed to encode map e452 with expected crc 2023-06-10T21:48:42.611 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:42 smithi064 ceph-mon[180297]: pgmap v1672: 89 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 615 op/s; 57706/189528 objects misplaced (30.447%); 6.3 MiB/s, 100 objects/s recovering 2023-06-10T21:48:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:42.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:42.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:42 smithi064 ceph-mon[186706]: failed to encode map e452 with expected crc 2023-06-10T21:48:42.613 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:42 smithi064 ceph-mon[186706]: pgmap v1672: 89 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 615 op/s; 57706/189528 objects misplaced (30.447%); 6.3 MiB/s, 100 objects/s recovering 2023-06-10T21:48:42.714 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:42.747 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:42.885 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:42.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:42.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:42 smithi152 ceph-mon[156819]: failed to encode map e452 with expected crc 2023-06-10T21:48:42.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:42 smithi152 ceph-mon[156819]: pgmap v1672: 89 pgs: 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 615 op/s; 57706/189528 objects misplaced (30.447%); 6.3 MiB/s, 100 objects/s recovering 2023-06-10T21:48:42.991 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:43.032 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:42] "GET /metrics HTTP/1.1" 200 224233 "" "Prometheus/2.33.4" 2023-06-10T21:48:43.255 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:48:43.267 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:43.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:43.373 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:43.399 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:43.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:43.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:43.558 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:43.636 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:43.709 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:43.845 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:43.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:43.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:44.030 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:44.191 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:48:44.255 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:44.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:44.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:44.401 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:44.447 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:44.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:44.551 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:44.632 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:44 smithi064 ceph-mon[180297]: pgmap v1673: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 57342/188802 objects misplaced (30.372%); 9.0 MiB/s, 143 objects/s recovering 2023-06-10T21:48:44.633 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:44 smithi064 ceph-mon[186706]: pgmap v1673: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 57342/188802 objects misplaced (30.372%); 9.0 MiB/s, 143 objects/s recovering 2023-06-10T21:48:44.655 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:44 smithi152 ceph-mon[156819]: pgmap v1673: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 57342/188802 objects misplaced (30.372%); 9.0 MiB/s, 143 objects/s recovering 2023-06-10T21:48:44.780 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:44.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:44.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:44.889 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:44.929 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:44.937 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:45.073 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:45.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:45.179 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:45.265 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:45.390 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:48:45.391 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:48:45.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:48:45.391 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [] 2023-06-10T21:48:45.391 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:48:45.425 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:48:45.516 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:45.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:45.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:45.524 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:45.625 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:45.688 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:48:46.119 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:46.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:46.230 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:46.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:48:46.277 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:46.342 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:46.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:46.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:48:46.398 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:48:46.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: failed to encode map e453 with expected crc 2023-06-10T21:48:46.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:46.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: osdmap e453: 8 total, 8 up, 5 in 2023-06-10T21:48:46.399 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[180297]: pgmap v1675: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 57207/187986 objects misplaced (30.432%); 7.7 MiB/s, 123 objects/s recovering 2023-06-10T21:48:46.399 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:48:46.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:48:46.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: failed to encode map e453 with expected crc 2023-06-10T21:48:46.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:46.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: osdmap e453: 8 total, 8 up, 5 in 2023-06-10T21:48:46.400 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:46 smithi064 ceph-mon[186706]: pgmap v1675: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 57207/187986 objects misplaced (30.432%); 7.7 MiB/s, 123 objects/s recovering 2023-06-10T21:48:46.448 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:46.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:48:46.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:48:46.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: failed to encode map e453 with expected crc 2023-06-10T21:48:46.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2154419981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:48:46.504 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: osdmap e453: 8 total, 8 up, 5 in 2023-06-10T21:48:46.505 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:46 smithi152 ceph-mon[156819]: pgmap v1675: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 371 op/s; 57207/187986 objects misplaced (30.432%); 7.7 MiB/s, 123 objects/s recovering 2023-06-10T21:48:46.505 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:46.591 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:46.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:46.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:46.774 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:46.800 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:46.851 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:46.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:46.957 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:47.044 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:47.249 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5, 0] out_osds: [6, 4, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:48:47.250 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:48:47.250 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-06-10T21:48:47.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 6 2023-06-10T21:48:47.636 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:47 smithi064 ceph-mon[180297]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:47 smithi064 ceph-mon[180297]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:47 smithi064 ceph-mon[186706]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.639 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:47 smithi064 ceph-mon[186706]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:47.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:47.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:47 smithi152 ceph-mon[156819]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:47 smithi152 ceph-mon[156819]: failed to encode map e453 with expected crc 2023-06-10T21:48:47.740 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:47.815 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:47.908 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:47.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:47.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:48.018 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:48.028 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:48.108 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:48.249 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:48.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:48.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:48.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:48.418 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:48.588 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:48.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:48.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:48.667 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:48.677 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:48:48.677 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:48:48.678 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:48:48.678 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:48:48.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:48.704 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:48.729 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:48.750 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:48.783 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:48.823 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:48.855 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:48.887 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:48.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.899 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.900 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:48:48.914 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:48.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:49.019 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:49.057 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:49.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:49.419 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:49.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[180297]: pgmap v1676: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 54254/181113 objects misplaced (29.956%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:48:49.420 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2094259546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:49.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:49.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:49.420 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[186706]: pgmap v1676: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 54254/181113 objects misplaced (29.956%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:48:49.421 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:49 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2094259546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:49.481 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:48:49.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:48:49.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:48:49.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:49 smithi152 ceph-mon[156819]: pgmap v1676: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 598 op/s; 54254/181113 objects misplaced (29.956%); 11 MiB/s, 173 objects/s recovering 2023-06-10T21:48:49.482 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:49 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2094259546' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:49.483 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:48:49.805 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:49.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:49.909 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:49.910 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:49.937 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:49.985 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:49.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:50.090 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:50.130 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:50.188 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:50.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:50.292 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:50.362 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:50.445 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:50 smithi152 ceph-mon[156819]: pgmap v1677: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 54093/179481 objects misplaced (30.139%); 8.9 MiB/s, 143 objects/s recovering 2023-06-10T21:48:50.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:50 smithi064 ceph-mon[180297]: pgmap v1677: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 54093/179481 objects misplaced (30.139%); 8.9 MiB/s, 143 objects/s recovering 2023-06-10T21:48:50.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:50 smithi064 ceph-mon[186706]: pgmap v1677: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 548 op/s; 54093/179481 objects misplaced (30.139%); 8.9 MiB/s, 143 objects/s recovering 2023-06-10T21:48:50.838 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:50.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:50.950 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:50.981 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:51.069 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:51.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:51.178 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:51.209 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:51.332 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:51.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:51.422 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:51.438 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:51.517 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:51.545 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:51.546 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:51.732 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:51.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:51.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:51.849 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:51.926 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:52.043 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:51 smithi064 conmon[180272]: 2023-06-10T21:48:51.935+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e453 definitely_dead 0 2023-06-10T21:48:52.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:52.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:52.187 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:48:52.229 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:52.344 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:52 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3990539761' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.345 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:52 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.346 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:52 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3990539761' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.347 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:52 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.400 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.6. 2023-06-10T21:48:52.445 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:48:52.448 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:48:52.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:52.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:48:52.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:52 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3990539761' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.473 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:52 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-10T21:48:52.557 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:52.584 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:52.628 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:52.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:52.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:52.766 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:52.801 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:52.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:48:52.819 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:52.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:52.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:52.923 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:52.954 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:48:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:48:52] "GET /metrics HTTP/1.1" 200 224246 "" "Prometheus/2.33.4" 2023-06-10T21:48:53.568 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:53.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:53.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[180297]: pgmap v1678: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 51275/172515 objects misplaced (29.722%); 8.5 MiB/s, 135 objects/s recovering 2023-06-10T21:48:53.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[180297]: failed to encode map e454 with expected crc 2023-06-10T21:48:53.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-10T21:48:53.641 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[180297]: osdmap e454: 8 total, 8 up, 6 in 2023-06-10T21:48:53.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[186706]: pgmap v1678: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 51275/172515 objects misplaced (29.722%); 8.5 MiB/s, 135 objects/s recovering 2023-06-10T21:48:53.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[186706]: failed to encode map e454 with expected crc 2023-06-10T21:48:53.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-10T21:48:53.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:53 smithi064 ceph-mon[186706]: osdmap e454: 8 total, 8 up, 6 in 2023-06-10T21:48:53.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:53.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:53 smithi152 ceph-mon[156819]: pgmap v1678: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 566 op/s; 51275/172515 objects misplaced (29.722%); 8.5 MiB/s, 135 objects/s recovering 2023-06-10T21:48:53.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:53 smithi152 ceph-mon[156819]: failed to encode map e454 with expected crc 2023-06-10T21:48:53.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-10T21:48:53.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:53 smithi152 ceph-mon[156819]: osdmap e454: 8 total, 8 up, 6 in 2023-06-10T21:48:53.724 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:54.416 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:54.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:54.496 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:54.496 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:54.513 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: osdmap e455: 8 total, 8 up, 6 in 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.2 starting backfill to osd.0 from (433'4400,453'9892] MIN to 453'9893 2023-06-10T21:48:54.514 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.4 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9793 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.b starting backfill to osd.0 from (430'3700,453'9857] MIN to 453'9858 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 453'10099 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.e continuing backfill to osd.0 from (431'4000,453'10147] MIN to 453'10147 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.e starting backfill to osd.6 from (0'0,0'0] MAX to 453'10147 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 453'10009 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.2 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9893 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 453'9858 2023-06-10T21:48:54.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[186706]: pgmap v1681: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 51275/172164 objects misplaced (29.783%); 7.8 MiB/s, 125 objects/s recovering 2023-06-10T21:48:54.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: osdmap e455: 8 total, 8 up, 6 in 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.2 starting backfill to osd.0 from (433'4400,453'9892] MIN to 453'9893 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.4 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9793 2023-06-10T21:48:54.518 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.b starting backfill to osd.0 from (430'3700,453'9857] MIN to 453'9858 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 453'10099 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.e continuing backfill to osd.0 from (431'4000,453'10147] MIN to 453'10147 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.e starting backfill to osd.6 from (0'0,0'0] MAX to 453'10147 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 453'10009 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.2 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9893 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 453'9858 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.519 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:54 smithi064 ceph-mon[180297]: pgmap v1681: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 51275/172164 objects misplaced (29.783%); 7.8 MiB/s, 125 objects/s recovering 2023-06-10T21:48:54.522 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:54.546 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-06-10T21:48:54.625 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: osdmap e455: 8 total, 8 up, 6 in 2023-06-10T21:48:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.2 starting backfill to osd.0 from (433'4400,453'9892] MIN to 453'9893 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.4 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9793 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.b starting backfill to osd.0 from (430'3700,453'9857] MIN to 453'9858 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 453'10099 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.e continuing backfill to osd.0 from (431'4000,453'10147] MIN to 453'10147 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.e starting backfill to osd.6 from (0'0,0'0] MAX to 453'10147 2023-06-10T21:48:54.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.d starting backfill to osd.6 from (0'0,0'0] MAX to 453'10009 2023-06-10T21:48:54.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.2 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9893 2023-06-10T21:48:54.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: 14.b starting backfill to osd.6 from (0'0,0'0] MAX to 453'9858 2023-06-10T21:48:54.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: failed to encode map e455 with expected crc 2023-06-10T21:48:54.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:54 smithi152 ceph-mon[156819]: pgmap v1681: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 51275/172164 objects misplaced (29.783%); 7.8 MiB/s, 125 objects/s recovering 2023-06-10T21:48:54.744 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:54.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:54.855 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:54.888 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:54.924 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:48:54.988 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:54.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:55.094 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:55.160 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:55.289 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:55.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:55.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:55.301 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:48:55.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:55.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.477 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: osdmap e456: 8 total, 8 up, 6 in 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: 14.3 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9977 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: 14.c starting backfill to osd.0 from (430'3700,453'9894] MIN to 453'9895 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: 14.a starting backfill to osd.6 from (0'0,0'0] MAX to 453'9900 2023-06-10T21:48:55.478 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: 14.c starting backfill to osd.6 from (0'0,0'0] MAX to 453'9895 2023-06-10T21:48:55.479 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[186706]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: osdmap e456: 8 total, 8 up, 6 in 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: 14.3 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9977 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: 14.c starting backfill to osd.0 from (430'3700,453'9894] MIN to 453'9895 2023-06-10T21:48:55.480 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: 14.a starting backfill to osd.6 from (0'0,0'0] MAX to 453'9900 2023-06-10T21:48:55.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: 14.c starting backfill to osd.6 from (0'0,0'0] MAX to 453'9895 2023-06-10T21:48:55.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:55 smithi064 ceph-mon[180297]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.481 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: osdmap e456: 8 total, 8 up, 6 in 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: 14.3 starting backfill to osd.6 from (0'0,0'0] MAX to 453'9977 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: 14.c starting backfill to osd.0 from (430'3700,453'9894] MIN to 453'9895 2023-06-10T21:48:55.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: 14.a starting backfill to osd.6 from (0'0,0'0] MAX to 453'9900 2023-06-10T21:48:55.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: 14.c starting backfill to osd.6 from (0'0,0'0] MAX to 453'9895 2023-06-10T21:48:55.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:55 smithi152 ceph-mon[156819]: failed to encode map e456 with expected crc 2023-06-10T21:48:55.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:55.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:55.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:55.901 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:55.946 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:56.019 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:56.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:56.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:56.237 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:48:56.338 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:56.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: osdmap e457: 8 total, 8 up, 6 in 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:56.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:56 smithi152 ceph-mon[156819]: pgmap v1684: 89 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 51167/171495 objects misplaced (29.836%); 1.3 KiB/s, 2 keys/s, 3 objects/s recovering 2023-06-10T21:48:56.455 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:48:56.486 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:48:56.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:56.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:48:56.597 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:48:56.650 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:56.722 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:48:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:56.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:48:56.826 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:48:56.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: osdmap e457: 8 total, 8 up, 6 in 2023-06-10T21:48:56.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:56.855 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[186706]: pgmap v1684: 89 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 51167/171495 objects misplaced (29.836%); 1.3 KiB/s, 2 keys/s, 3 objects/s recovering 2023-06-10T21:48:56.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: osdmap e457: 8 total, 8 up, 6 in 2023-06-10T21:48:56.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: failed to encode map e457 with expected crc 2023-06-10T21:48:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:48:56.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:56 smithi064 ceph-mon[180297]: pgmap v1684: 89 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 51167/171495 objects misplaced (29.836%); 1.3 KiB/s, 2 keys/s, 3 objects/s recovering 2023-06-10T21:48:56.857 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:56.910 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:48:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:56.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:48:57.014 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:57.087 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:57.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:57.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:57.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:57.544 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:57.624 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: failed to encode map e457 with expected crc 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: osdmap e458: 8 total, 8 up, 6 in 2023-06-10T21:48:57.625 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:57 smithi152 ceph-mon[156819]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: failed to encode map e457 with expected crc 2023-06-10T21:48:57.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:57.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: osdmap e458: 8 total, 8 up, 6 in 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[186706]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: failed to encode map e457 with expected crc 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:48:57.627 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1418968471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:48:57.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: osdmap e458: 8 total, 8 up, 6 in 2023-06-10T21:48:57.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:57 smithi064 ceph-mon[180297]: failed to encode map e458 with expected crc 2023-06-10T21:48:57.630 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:57.890 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:48:57.891 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:48:58.231 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:58.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:58.336 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:48:58.371 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:58.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:58 smithi064 ceph-mon[180297]: pgmap v1686: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 13 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 62466/166911 objects misplaced (37.425%); 14 MiB/s, 2 keys/s, 226 objects/s recovering 2023-06-10T21:48:58.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:58 smithi064 ceph-mon[186706]: pgmap v1686: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 13 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 62466/166911 objects misplaced (37.425%); 14 MiB/s, 2 keys/s, 226 objects/s recovering 2023-06-10T21:48:58.576 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:48:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:58.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:48:58.689 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:48:58.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:58 smithi152 ceph-mon[156819]: pgmap v1686: 89 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 13 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 377 op/s; 62466/166911 objects misplaced (37.425%); 14 MiB/s, 2 keys/s, 226 objects/s recovering 2023-06-10T21:48:58.738 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:48:58.771 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:58.932 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:48:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:58.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:48:58.947 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:48:58.985 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:48:59.027 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:48:59.044 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:59.080 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:48:59.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:48:59.117 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:59.151 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:48:59.183 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:48:59.219 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:48:59.228 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.231 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.232 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:48:59.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:59.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:48:59.347 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:48:59.374 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:48:59.414 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:59.542 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:48:59.547 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5, 0, 6] out_osds: [4, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:48:59.547 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:48:59.547 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-06-10T21:48:59.547 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:48:59.547 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:48:59.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-06-10T21:48:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:59.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:48:59.654 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:48:59.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:48:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3135274341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:59.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:48:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3135274341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:48:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3135274341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:48:59.750 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:48:59.887 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:48:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:48:59.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:00.000 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:00.050 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:00.266 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:00.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:00.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:00.372 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:49:00.404 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:00.447 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:00.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:00.551 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:00.607 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:00.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:00 smithi152 ceph-mon[156819]: pgmap v1687: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 65166/165417 objects misplaced (39.395%); 16 MiB/s, 1 keys/s, 264 objects/s recovering 2023-06-10T21:49:00.835 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:00.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:00.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:00 smithi064 ceph-mon[180297]: pgmap v1687: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 65166/165417 objects misplaced (39.395%); 16 MiB/s, 1 keys/s, 264 objects/s recovering 2023-06-10T21:49:00.846 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:00 smithi064 ceph-mon[186706]: pgmap v1687: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 374 op/s; 65166/165417 objects misplaced (39.395%); 16 MiB/s, 1 keys/s, 264 objects/s recovering 2023-06-10T21:49:00.943 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:00.966 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:49:01.015 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:01.131 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:01.154 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:01.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:01.264 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:01.343 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:01.449 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:01.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:01.524 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:49:01.524 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:49:01.525 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:49:01.525 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:49:01.525 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:49:01.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:01.605 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:02.093 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:02.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:02.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:02.197 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:02.251 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:02.380 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:02 smithi152 ceph-mon[156819]: pgmap v1688: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 61105/158226 objects misplaced (38.619%); 20 MiB/s, 317 objects/s recovering 2023-06-10T21:49:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:02 smithi064 ceph-mon[186706]: pgmap v1688: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 61105/158226 objects misplaced (38.619%); 20 MiB/s, 317 objects/s recovering 2023-06-10T21:49:02.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:02 smithi064 ceph-mon[180297]: pgmap v1688: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 61105/158226 objects misplaced (38.619%); 20 MiB/s, 317 objects/s recovering 2023-06-10T21:49:02.873 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:49:02.913 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:02.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:03.020 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:03.112 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:03.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:49:03.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:02] "GET /metrics HTTP/1.1" 200 224246 "" "Prometheus/2.33.4" 2023-06-10T21:49:03.363 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:03.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:03.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:03.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:03.709 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:03.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:03.730 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:49:03.730 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "" 2023-06-10T21:49:03.730 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:49:03.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:03.938 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:04.030 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:04.073 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:49:04.110 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:04.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:04.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:04.296 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:04.507 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:04.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:04.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:04.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:04.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:04.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:05.035 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:05.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:05.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:05.231 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:05.267 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:49:05.378 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:05.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:05.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:05.413 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:05 smithi152 ceph-mon[156819]: pgmap v1689: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 60383/157695 objects misplaced (38.291%); 22 MiB/s, 347 objects/s recovering 2023-06-10T21:49:05.414 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:05.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:05.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:05 smithi064 ceph-mon[180297]: pgmap v1689: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 60383/157695 objects misplaced (38.291%); 22 MiB/s, 347 objects/s recovering 2023-06-10T21:49:05.508 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:05.508 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:05 smithi064 ceph-mon[186706]: pgmap v1689: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 516 op/s; 60383/157695 objects misplaced (38.291%); 22 MiB/s, 347 objects/s recovering 2023-06-10T21:49:05.509 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:05.511 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:05.555 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:05.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:05.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:05.660 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:05.689 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:05.943 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:05.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:05.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:06.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:06.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:06.242 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:06.243 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.243 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:06.243 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: osdmap e459: 8 total, 8 up, 6 in 2023-06-10T21:49:06.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.244 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:06 smithi152 ceph-mon[156819]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: osdmap e459: 8 total, 8 up, 6 in 2023-06-10T21:49:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[180297]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4179265461' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: osdmap e459: 8 total, 8 up, 6 in 2023-06-10T21:49:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.412 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:06 smithi064 ceph-mon[186706]: failed to encode map e459 with expected crc 2023-06-10T21:49:06.604 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136396 12.12 12:4926b373:::benchmark_data_smithi064_442287_object48346:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:49:06.605 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.187222+0000", 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.41277692200003, 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4038103829999999, 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.606 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136396 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187222+0000", 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.607 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187222+0000", 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.608 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187221+0000", 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3589999999999999e-06 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187226+0000", 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0907000000000001e-05 2023-06-10T21:49:06.609 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187237+0000", 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9930000000000003e-06 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.610 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.187242+0000", 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029205070000000001 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.190162+0000", 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00055756899999999995 2023-06-10T21:49:06.611 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.190720+0000", 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.036573343000000001 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.612 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227293+0000", 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3305000000000001e-05 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227307+0000", 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.236829841 2023-06-10T21:49:06.613 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.464136+0000", 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012223600000000001 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.614 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.464259+0000", 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000356177 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3,5", 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.464615+0000", 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028791379999999998 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.615 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.467494+0000", 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00077371500000000001 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.468268+0000", 2023-06-10T21:49:06.616 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7542000000000002e-05 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.468295+0000", 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.093600524000000004 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.617 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.561896+0000", 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.027794448999999999 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.589690+0000", 2023-06-10T21:49:06.618 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6280999999999998e-05 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.589757+0000", 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012757440000000001 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.619 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.591032+0000", 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4038103829999999 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.620 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136402 12.11 12:8f0d796b:::benchmark_data_smithi064_442287_object48352:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.205611+0000", 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.39438829100004, 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1281836329999999, 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.621 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136402 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205611+0000", 2023-06-10T21:49:06.622 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205611+0000", 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.623 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205609+0000", 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8580000000000002e-06 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205616+0000", 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.6021000000000001e-05 2023-06-10T21:49:06.624 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205632+0000", 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2780000000000001e-06 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.625 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205637+0000", 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7203000000000002e-05 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205684+0000", 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000153957 2023-06-10T21:49:06.626 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205838+0000", 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000312696 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,5", 2023-06-10T21:49:06.627 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.206151+0000", 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001012923 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.207164+0000", 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00056762999999999996 2023-06-10T21:49:06.628 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.207732+0000", 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013159327 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.629 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.220891+0000", 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3427999999999999e-05 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.220904+0000", 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0029355019999999 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.630 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.223840+0000", 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3489999999999998e-06 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.631 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.223849+0000", 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.109867937 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.333717+0000", 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6979999999999998e-05 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.632 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.333794+0000", 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1281836329999999 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.633 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136403 12.17 12:ef27d9ec:::benchmark_data_smithi064_442287_object48353:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.205646+0000", 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.39435292500002, 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1393606059999999, 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:49:06.634 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136403 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.635 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205646+0000", 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205646+0000", 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.636 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205645+0000", 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4800000000000001e-06 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.637 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205649+0000", 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2450000000000004e-06 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205658+0000", 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2909999999999999e-06 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.638 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.205661+0000", 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00037101000000000001 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.206032+0000", 2023-06-10T21:49:06.639 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012425000000000001 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.206157+0000", 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00031622699999999998 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.640 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2", 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.206473+0000", 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001608501 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.208081+0000", 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.018858704 2023-06-10T21:49:06.641 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.226940+0000", 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7880000000000001e-06 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.642 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.226950+0000", 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0026066709999999 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.229557+0000", 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2295e-05 2023-06-10T21:49:06.643 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.229569+0000", 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1153557009999999 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.644 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.344925+0000", 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1959000000000006e-05 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.345006+0000", 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1393606059999999 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.645 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136407 12.d 12:b05de182:::benchmark_data_smithi064_442287_object48357:head [delete] snapc 0=[] ondisk+write+known_if_redirected e351)", 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.212237+0000", 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.38776176299996, 2023-06-10T21:49:06.646 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2657842690000001, 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136407 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.647 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212237+0000", 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.648 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212237+0000", 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212236+0000", 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0480000000000001e-06 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.649 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212243+0000", 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1519999999999998e-06 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.650 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212252+0000", 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3829999999999997e-06 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.212258+0000", 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.017800278999999999 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.651 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.230059+0000", 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0744000000000001e-05 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.230069+0000", 2023-06-10T21:49:06.652 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99371636399999996 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.223786+0000", 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7860000000000002e-06 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.653 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.223794+0000", 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.130319514 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.354114+0000", 2023-06-10T21:49:06.654 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000176532 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.354290+0000", 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000485794 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.655 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3", 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.354776+0000", 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0092394060000000004 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.364016+0000", 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001112647 2023-06-10T21:49:06.656 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.365128+0000", 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012308550000000001 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.657 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.366359+0000", 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.11146637199999999 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.477826+0000", 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5109999999999999e-05 2023-06-10T21:49:06.658 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.477881+0000", 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014067600000000001 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.659 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.478021+0000", 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2657842690000001 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.660 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136408 12.d 12:b32fe748:::benchmark_data_smithi064_442287_object48358:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.223323+0000", 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.37667566100004, 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2549523740000001, 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.661 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136408 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223323+0000", 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.662 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223323+0000", 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.663 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223321+0000", 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6690000000000006e-06 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223329+0000", 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1951e-05 2023-06-10T21:49:06.664 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223341+0000", 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6819999999999997e-06 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.665 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.223347+0000", 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067472670000000004 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.230095+0000", 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5739999999999998e-06 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.666 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.230102+0000", 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99501173399999998 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.225114+0000", 2023-06-10T21:49:06.667 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6339999999999996e-06 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.225124+0000", 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1366017669999999 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.668 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.361725+0000", 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105858 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.361831+0000", 2023-06-10T21:49:06.669 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010748599999999999 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3", 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.361939+0000", 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.003207578 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.670 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.365146+0000", 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069453600000000005 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.365841+0000", 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00054942000000000001 2023-06-10T21:49:06.671 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.366390+0000", 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.111684618 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.672 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.478075+0000", 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3175000000000004e-05 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.478138+0000", 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013749300000000001 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.673 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.478276+0000", 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2549523740000001 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.674 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136412 12.1 12:8779924e:::benchmark_data_smithi064_442287_object48362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.227514+0000", 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.372485181, 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2376345870000001, 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.675 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136412 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.676 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227514+0000", 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227514+0000", 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:06.677 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227512+0000", 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7110000000000001e-06 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.678 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227520+0000", 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5362999999999998e-05 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227536+0000", 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2840000000000001e-06 2023-06-10T21:49:06.679 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.227543+0000", 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023983400000000001 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.680 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.229941+0000", 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0770000000000008e-06 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.229949+0000", 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99180943499999996 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.681 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.221759+0000", 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.449e-05 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.682 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.221773+0000", 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1018561739999999 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.323630+0000", 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000142354 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.683 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.323772+0000", 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00040750200000000002 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,5", 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.324179+0000", 2023-06-10T21:49:06.684 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011451747999999999 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.335631+0000", 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021298860000000001 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.685 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.337761+0000", 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0082003330000000006 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.345961+0000", 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.118968015 2023-06-10T21:49:06.686 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.464929+0000", 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5469999999999995e-05 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.687 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.464995+0000", 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015336599999999999 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:51.465148+0000", 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2376345870000001 2023-06-10T21:49:06.688 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136414 12.2 12:448b0a7f:::benchmark_data_smithi064_442287_object48364:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.266201+0000", 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.33379829199998, 2023-06-10T21:49:06.689 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.506168202, 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136414 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.690 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266201+0000", 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.691 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266201+0000", 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266199+0000", 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5310000000000001e-06 2023-06-10T21:49:06.692 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266204+0000", 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0387999999999999e-05 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.693 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266215+0000", 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5529999999999998e-06 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266221+0000", 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000112558 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.694 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266334+0000", 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8429999999999998e-06 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.266340+0000", 2023-06-10T21:49:06.695 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1950090950000001 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.461349+0000", 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00069405699999999999 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.696 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.462043+0000", 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.12862865200000001 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.590672+0000", 2023-06-10T21:49:06.697 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104738 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.590776+0000", 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00036304600000000001 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.698 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3,5", 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.591139+0000", 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057237499999999997 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.596863+0000", 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9895999999999997e-05 2023-06-10T21:49:06.699 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.596923+0000", 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2514000000000001e-05 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.700 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.596955+0000", 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1223999999999994e-05 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.597027+0000", 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.174354764 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.701 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.771381+0000", 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3834000000000001e-05 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.702 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.771425+0000", 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00094339300000000005 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.772369+0000", 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.506168202 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.703 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:136416 12.2 12:447b9997:::benchmark_data_smithi064_442287_object48366:head [delete] snapc 0=[] ondisk+write+known_if_redirected e352)", 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:39:49.283177+0000", 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 557.31682168999998, 2023-06-10T21:49:06.704 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.489846376, 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 136416 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.705 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283177+0000", 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.706 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283177+0000", 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283175+0000", 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0609999999999996e-06 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.707 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283183+0000", 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3706999999999999e-05 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283197+0000", 2023-06-10T21:49:06.708 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2520000000000002e-06 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.283204+0000", 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0013655290000000001 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.709 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.284570+0000", 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1239999999999994e-06 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:49.284579+0000", 2023-06-10T21:49:06.710 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1774893989999999 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.462068+0000", 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00066717299999999996 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.711 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.462736+0000", 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.13172667900000001 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.594462+0000", 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000103701 2023-06-10T21:49:06.712 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.594566+0000", 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035647599999999999 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 1,2,3,5", 2023-06-10T21:49:06.713 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.594922+0000", 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020444880000000001 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.596967+0000", 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011513700000000001 2023-06-10T21:49:06.714 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.597082+0000", 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9505000000000001e-05 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.715 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.597132+0000", 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9624999999999998e-05 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.597161+0000", 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.175277869 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.772439+0000", 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3751000000000002e-05 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.772483+0000", 2023-06-10T21:49:06.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00054073700000000001 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:39:50.773023+0000", 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.489846376 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.718 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46562.0:156300 12.13 12:cbc23d40:::benchmark_data_smithi064_442287_object68250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e357)", 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:40:29.212837+0000", 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 517.38716176000003, 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.98583489, 2023-06-10T21:49:06.719 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46562", 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 156300 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.720 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212837+0000", 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212837+0000", 2023-06-10T21:49:06.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212836+0000", 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0810000000000004e-06 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212841+0000", 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1330000000000001e-06 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212848+0000", 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8720000000000004e-06 2023-06-10T21:49:06.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212852+0000", 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6241e-05 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.212898+0000", 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015222099999999999 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.213050+0000", 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038885299999999999 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,2,3,5", 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.213439+0000", 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001848312 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.726 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.215287+0000", 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029195700000000002 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.215579+0000", 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0012924060000000001 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.216871+0000", 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028547400000000001 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.219726+0000", 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0903e-05 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:29.219737+0000", 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99877735099999998 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:30.218514+0000", 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2059999999999996e-06 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:30.218524+0000", 2023-06-10T21:49:06.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94894513999999996 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:31.167469+0000", 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6203000000000001e-05 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:31.167555+0000", 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.031067298 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:31.198622+0000", 2023-06-10T21:49:06.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9588e-05 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:40:31.198672+0000", 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.98583489 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.733 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:155145 14.4 14:209af74d:::benchmark_data_smithi064_559916_object47289:head [delete] snapc 0=[] ondisk+write+known_if_redirected e453)", 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:48:45.630225+0000", 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.969773832000001, 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20786064100000001, 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.734 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 155145 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630225+0000", 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630225+0000", 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:06.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630224+0000", 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.104e-06 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630228+0000", 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0240000000000003e-06 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630234+0000", 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.608e-06 2023-06-10T21:49:06.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630237+0000", 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9836999999999999e-05 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630267+0000", 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5414000000000005e-05 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630363+0000", 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9486000000000003e-05 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 0,3,5", 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.630462+0000", 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024647100000000002 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.632927+0000", 2023-06-10T21:49:06.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015282499999999999 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633080+0000", 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20476757400000001 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837847+0000", 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5460000000000003e-05 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837883+0000", 2023-06-10T21:49:06.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0191e-05 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837943+0000", 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014276600000000001 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.838086+0000", 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.20786064100000001 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.745 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:155143 14.1 e453/444 14:84b73e8a:::benchmark_data_smithi064_559916_object47287:head v 453'9856, mlcod=453'9853)", 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:48:45.633363+0000", 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.966635512, 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21294689999999999, 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633363+0000", 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633363+0000", 2023-06-10T21:49:06.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633362+0000", 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2649999999999999e-06 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.748 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633370+0000", 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7280000000000007e-06 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633379+0000", 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7310000000000001e-06 2023-06-10T21:49:06.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633384+0000", 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3931999999999999e-05 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633428+0000", 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4388e-05 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633443+0000", 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.212830362 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846273+0000", 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7289000000000002e-05 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846310+0000", 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21294689999999999 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:155144 14.2 e453/444 14:425668c8:::benchmark_data_smithi064_559916_object47288:head v 453'9645, mlcod=453'9642)", 2023-06-10T21:49:06.753 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:48:45.633396+0000", 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.966603107000001, 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21319743599999999, 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633396+0000", 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633396+0000", 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:06.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633395+0000", 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9179999999999996e-06 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633399+0000", 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8270000000000003e-06 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633405+0000", 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0099999999999997e-06 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633409+0000", 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3677000000000003e-05 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633443+0000", 2023-06-10T21:49:06.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4882000000000001e-05 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.633458+0000", 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21310067299999999 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846558+0000", 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4727999999999997e-05 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846593+0000", 2023-06-10T21:49:06.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21319743599999999 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:155148 14.2 e453/444 14:470adc4d:::benchmark_data_smithi064_559916_object47292:head v 453'9646, mlcod=453'9642)", 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:48:45.634771+0000", 2023-06-10T21:49:06.761 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.965228275000001, 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21184420400000001, 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.634771+0000", 2023-06-10T21:49:06.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.634771+0000", 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.763 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.634770+0000", 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2500000000000003e-06 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.634776+0000", 2023-06-10T21:49:06.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.202954102 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837730+0000", 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2509999999999994e-06 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837739+0000", 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0035437749999999999 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.841282+0000", 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4068999999999999e-05 2023-06-10T21:49:06.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.841306+0000", 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052898779999999996 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846596+0000", 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8479999999999999e-05 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.846615+0000", 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.21184420400000001 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.768 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop_reply(client.47150.0:155145 14.4 e453/444 ondisk, result = 0)", 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:48:45.635338+0000", 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 20.964660715000001, 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.202511899, 2023-06-10T21:49:06.769 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.635338+0000", 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.635338+0000", 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.635337+0000", 2023-06-10T21:49:06.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1680000000000003e-06 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.635343+0000", 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.202382535 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837725+0000", 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3857e-05 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837739+0000", 2023-06-10T21:49:06.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6339000000000001e-05 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837825+0000", 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.147e-05 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837837+0000", 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3199e-05 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:48:45.837850+0000", 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.202511899 2023-06-10T21:49:06.775 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:163564 14.f 14:f74f115b:::benchmark_data_smithi064_559916_object55708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:02.006644+0000", 2023-06-10T21:49:06.776 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4.5933548990000004, 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18784975600000001, 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 163564 2023-06-10T21:49:06.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006644+0000", 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006644+0000", 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006642+0000", 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2250000000000003e-06 2023-06-10T21:49:06.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006649+0000", 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3183000000000001e-05 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006663+0000", 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1680000000000003e-06 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006668+0000", 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5506999999999997e-05 2023-06-10T21:49:06.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006703+0000", 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4975000000000005e-05 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006798+0000", 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3288000000000006e-05 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.006882+0000", 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18720973899999999 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194091+0000", 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1986e-05 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194143+0000", 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025794099999999998 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194401+0000", 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9047e-05 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194450+0000", 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3420000000000001e-05 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194494+0000", 2023-06-10T21:49:06.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18784975600000001 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:163568 14.f 14:ff3accef:::benchmark_data_smithi064_559916_object55712:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:02.009260+0000", 2023-06-10T21:49:06.787 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4.5907388989999998, 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18774627699999999, 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 163568 2023-06-10T21:49:06.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009260+0000", 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009260+0000", 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009259+0000", 2023-06-10T21:49:06.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2219999999999999e-06 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009265+0000", 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2887999999999999e-05 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009278+0000", 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6919999999999996e-06 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009285+0000", 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1921000000000001e-05 2023-06-10T21:49:06.792 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009326+0000", 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7645999999999994e-05 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009414+0000", 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011968000000000001 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009534+0000", 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18456955 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194103+0000", 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026334900000000002 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194367+0000", 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002541485 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.196908+0000", 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3384000000000003e-05 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.196962+0000", 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4642999999999999e-05 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.197006+0000", 2023-06-10T21:49:06.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18774627699999999 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:163566 14.b e458/454 14:d106b8d2:::benchmark_data_smithi064_559916_object55710:head v 458'10131, mlcod=458'10129)", 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:02.009811+0000", 2023-06-10T21:49:06.798 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 4.5901880589999999, 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.187508285, 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009811+0000", 2023-06-10T21:49:06.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009811+0000", 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009810+0000", 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3980000000000004e-06 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.009816+0000", 2023-06-10T21:49:06.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.18422465199999999 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194041+0000", 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.169e-05 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194053+0000", 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9845000000000001e-05 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194102+0000", 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8398000000000002e-05 2023-06-10T21:49:06.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.194131+0000", 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0031556029999999999 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.197286+0000", 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2636999999999999e-05 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:02.197319+0000", 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.187508285 2023-06-10T21:49:06.805 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:164800 14.7 e458/443 14:e8cde041:::benchmark_data_smithi064_559916_object56944:head v 458'10217, mlcod=458'10215)", 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:04.478929+0000", 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.121069742, 2023-06-10T21:49:06.806 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16824181499999999, 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478929+0000", 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.807 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478929+0000", 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478928+0000", 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.294e-06 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478936+0000", 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0540000000000001e-05 2023-06-10T21:49:06.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478946+0000", 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5049999999999998e-06 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.478952+0000", 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5933999999999996e-05 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.479018+0000", 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0656999999999999e-05 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.479038+0000", 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16809786700000001 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.647136+0000", 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4801000000000001e-05 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.647171+0000", 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16824181499999999 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.813 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:164806 14.f 14:f40c3b45:::benchmark_data_smithi064_559916_object56950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:04.481743+0000", 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 2.1182556520000002, 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16990365199999999, 2023-06-10T21:49:06.814 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 164806 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.481743+0000", 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.481743+0000", 2023-06-10T21:49:06.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.481741+0000", 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.0440000000000001e-06 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.481750+0000", 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.15707634600000001 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.638827+0000", 2023-06-10T21:49:06.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7268000000000001e-05 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.638844+0000", 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042474059999999996 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.643091+0000", 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00018216099999999999 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.643274+0000", 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012665199999999999 2023-06-10T21:49:06.820 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.643400+0000", 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0038404609999999999 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.821 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.647241+0000", 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0039232399999999997 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.651164+0000", 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038473199999999999 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.822 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.651549+0000", 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4042999999999997e-05 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.651603+0000", 2023-06-10T21:49:06.823 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4121e-05 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:04.651647+0000", 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.16990365199999999 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.824 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:165319 14.f 14:fff65643:::benchmark_data_smithi064_559916_object57463:head [delete] snapc 0=[] ondisk+write+known_if_redirected e459)", 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:05.387937+0000", 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 1.2120613739999999, 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33024788700000002, 2023-06-10T21:49:06.825 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 165319 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:49:06.826 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.387937+0000", 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.387937+0000", 2023-06-10T21:49:06.827 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.387935+0000", 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5429999999999999e-06 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.828 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.387945+0000", 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7918e-05 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.387963+0000", 2023-06-10T21:49:06.829 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0927000000000003e-05 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.388004+0000", 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8724999999999999e-05 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.830 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.388052+0000", 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013486300000000001 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.388187+0000", 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012483000000000001 2023-06-10T21:49:06.831 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,5", 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.388312+0000", 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028570510000000002 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.832 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.391169+0000", 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0419999999999997e-06 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.391176+0000", 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.32690364700000002 2023-06-10T21:49:06.833 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.718080+0000", 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2200999999999996e-05 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:49:06.834 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.718142+0000", 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3402000000000002e-05 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:05.718185+0000", 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33024788700000002 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.835 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.836 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.836 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:49:06.836 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:49:06.836 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:49:06.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:06.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:06.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:06.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:07.019 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:07.095 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:07.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:07.208 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:07.261 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:07.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:49:07.409 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:07.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:07.510 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:07 smithi064 ceph-mon[180297]: pgmap v1691: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 60217/156714 objects misplaced (38.425%); 20 MiB/s, 313 objects/s recovering 2023-06-10T21:49:07.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:07 smithi064 ceph-mon[186706]: pgmap v1691: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 60217/156714 objects misplaced (38.425%); 20 MiB/s, 313 objects/s recovering 2023-06-10T21:49:07.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:07.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:07 smithi152 ceph-mon[156819]: pgmap v1691: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 498 op/s; 60217/156714 objects misplaced (38.425%); 20 MiB/s, 313 objects/s recovering 2023-06-10T21:49:07.554 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:07.638 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:07.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:07.751 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:07.796 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:07.856 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:49:07.876 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:07.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:07.894 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:49:07.934 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:49:07.982 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:07.988 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:49:08.025 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:49:08.059 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:08.068 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:49:08.108 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:49:08.159 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:49:08.181 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.181 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.181 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.182 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:49:08.187 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:08.214 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:08.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:08.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:08.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:08.391 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/605537159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:08.392 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:08 smithi064 ceph-mon[186706]: pgmap v1692: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 56964/150501 objects misplaced (37.850%); 16 MiB/s, 262 objects/s recovering 2023-06-10T21:49:08.395 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:08.471 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:08.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:08.580 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:08.654 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/605537159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:08.655 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:08 smithi064 ceph-mon[180297]: pgmap v1692: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 56964/150501 objects misplaced (37.850%); 16 MiB/s, 262 objects/s recovering 2023-06-10T21:49:08.657 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/605537159' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:08 smithi152 ceph-mon[156819]: pgmap v1692: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 56964/150501 objects misplaced (37.850%); 16 MiB/s, 262 objects/s recovering 2023-06-10T21:49:08.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:08.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:08.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:08.906 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:08.933 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:08.980 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:08.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:08.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:09.084 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:09.114 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:09.164 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:09.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:09.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:09.259 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:49:09.267 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:09.319 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:09.414 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:09.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:09.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:09.523 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:09.616 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:09.810 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:09.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:09.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:09.923 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:09.982 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:10.083 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:10.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:10.131 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 5, 0, 6] out_osds: [4, 1] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:49:10.131 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:49:10.131 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [3, 7, 2, 5, 0, 6] 2023-06-10T21:49:10.131 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd out 5 2023-06-10T21:49:10.190 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:10.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:10.418 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:10 smithi152 ceph-mon[156819]: pgmap v1693: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 543 op/s; 56134/149097 objects misplaced (37.649%); 17 MiB/s, 271 objects/s recovering 2023-06-10T21:49:10.439 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:10.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:10.548 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:10.637 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:10 smithi064 ceph-mon[180297]: pgmap v1693: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 543 op/s; 56134/149097 objects misplaced (37.649%); 17 MiB/s, 271 objects/s recovering 2023-06-10T21:49:10.638 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:10 smithi064 ceph-mon[186706]: pgmap v1693: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 543 op/s; 56134/149097 objects misplaced (37.649%); 17 MiB/s, 271 objects/s recovering 2023-06-10T21:49:10.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:11.362 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:11.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:11.472 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:11.477 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:11.584 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:11.732 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:11.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:49:11.866 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:11.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:11.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:12.089 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:12.441 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:12 smithi064 ceph-mon[180297]: pgmap v1694: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 53076/142590 objects misplaced (37.223%); 13 MiB/s, 209 objects/s recovering 2023-06-10T21:49:12.441 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:12 smithi064 ceph-mon[186706]: pgmap v1694: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 53076/142590 objects misplaced (37.223%); 13 MiB/s, 209 objects/s recovering 2023-06-10T21:49:12.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:12.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:12.618 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:12.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:12 smithi152 ceph-mon[156819]: pgmap v1694: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 521 op/s; 53076/142590 objects misplaced (37.223%); 13 MiB/s, 209 objects/s recovering 2023-06-10T21:49:12.714 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:13.005 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:12] "GET /metrics HTTP/1.1" 200 224296 "" "Prometheus/2.33.4" 2023-06-10T21:49:13.196 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:13.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:13.310 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:13.353 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:13.410 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:13.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:13.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:13.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:13.554 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:13.697 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:49:13.697 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:49:14.414 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:14.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:14.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:14.582 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:14.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:14 smithi152 ceph-mon[156819]: pgmap v1695: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 51999/142101 objects misplaced (36.593%); 16 MiB/s, 254 objects/s recovering 2023-06-10T21:49:14.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:14 smithi064 ceph-mon[186706]: pgmap v1695: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 51999/142101 objects misplaced (36.593%); 16 MiB/s, 254 objects/s recovering 2023-06-10T21:49:14.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:14 smithi064 ceph-mon[180297]: pgmap v1695: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 519 op/s; 51999/142101 objects misplaced (36.593%); 16 MiB/s, 254 objects/s recovering 2023-06-10T21:49:15.876 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:15.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:15.975 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:49:15.988 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:16.032 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:16.100 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:16.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:16.210 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:16.311 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:16.452 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:16.587 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:16.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:16.698 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:16.783 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:16.971 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:16.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:17.099 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:17.182 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:17.183 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:17.280 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:17.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:17.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:17.357 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 conmon[180272]: 2023-06-10T21:49:17.109+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e459 definitely_dead 0 2023-06-10T21:49:17.396 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:17.421 INFO:teuthology.orchestra.run.smithi064.stderr:marked out osd.5. 2023-06-10T21:49:17.422 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:49:17.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:17 smithi152 ceph-mon[156819]: pgmap v1696: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 51814/141042 objects misplaced (36.737%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:49:17.431 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/839655438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:49:17.432 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:17 smithi152 ceph-mon[156819]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:49:17.434 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:17.495 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:17.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:17.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:17.628 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[186706]: pgmap v1696: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 51814/141042 objects misplaced (36.737%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:49:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/839655438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:49:17.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[186706]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:49:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[180297]: pgmap v1696: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 471 op/s; 51814/141042 objects misplaced (36.737%); 14 MiB/s, 229 objects/s recovering 2023-06-10T21:49:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/839655438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:49:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-06-10T21:49:17.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:17 smithi064 ceph-mon[180297]: Client client.admin marked osd.5 out, while it was still marked up 2023-06-10T21:49:17.674 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:17.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:17.779 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:17.814 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:18.276 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:18.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:18.381 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:18.425 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:18.487 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:18.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:18.593 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:18.617 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:18.617 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:49:18.617 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:49:18.618 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: osdmap e460: 8 total, 8 up, 5 in 2023-06-10T21:49:18.618 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: failed to encode map e460 with expected crc 2023-06-10T21:49:18.618 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:18.619 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:49:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:49:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: osdmap e460: 8 total, 8 up, 5 in 2023-06-10T21:49:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: failed to encode map e460 with expected crc 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1820968094' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: osdmap e460: 8 total, 8 up, 5 in 2023-06-10T21:49:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: failed to encode map e460 with expected crc 2023-06-10T21:49:18.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:18.664 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:18.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:18.776 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:18.802 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:18.857 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:18.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:18.961 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:18.996 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:19.157 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:19.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:19.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:19.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:19.291 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:19.335 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:19.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:19.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:19.474 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:19.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: pgmap v1698: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 48561/134475 objects misplaced (36.112%); 15 MiB/s, 242 objects/s recovering 2023-06-10T21:49:19.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: osdmap e461: 8 total, 8 up, 5 in 2023-06-10T21:49:19.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: 14.d starting backfill to osd.0 from (430'3800,453'10008] MIN to 459'10843 2023-06-10T21:49:19.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[180297]: 14.d continuing backfill to osd.6 from (430'3800,459'10843] 14:b09cd29e:::benchmark_data_smithi064_559916_object68721:head to 459'10843 2023-06-10T21:49:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: pgmap v1698: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 48561/134475 objects misplaced (36.112%); 15 MiB/s, 242 objects/s recovering 2023-06-10T21:49:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: osdmap e461: 8 total, 8 up, 5 in 2023-06-10T21:49:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: 14.d starting backfill to osd.0 from (430'3800,453'10008] MIN to 459'10843 2023-06-10T21:49:19.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:19 smithi064 ceph-mon[186706]: 14.d continuing backfill to osd.6 from (430'3800,459'10843] 14:b09cd29e:::benchmark_data_smithi064_559916_object68721:head to 459'10843 2023-06-10T21:49:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: pgmap v1698: 89 pgs: 2 active+remapped+backfilling, 13 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 48561/134475 objects misplaced (36.112%); 15 MiB/s, 242 objects/s recovering 2023-06-10T21:49:19.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: failed to encode map e461 with expected crc 2023-06-10T21:49:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: osdmap e461: 8 total, 8 up, 5 in 2023-06-10T21:49:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: 14.d starting backfill to osd.0 from (430'3800,453'10008] MIN to 459'10843 2023-06-10T21:49:19.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:19 smithi152 ceph-mon[156819]: 14.d continuing backfill to osd.6 from (430'3800,459'10843] 14:b09cd29e:::benchmark_data_smithi064_559916_object68721:head to 459'10843 2023-06-10T21:49:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: osdmap e462: 8 total, 8 up, 5 in 2023-06-10T21:49:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.9 continuing backfill to osd.0 from (435'5000,459'10781] 14:9e8ff083:::benchmark_data_smithi064_559916_object80242:head to 459'10781 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'10767 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.a continuing backfill to osd.6 from (435'4900,459'10687] MIN to 459'10687 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.0 starting backfill to osd.0 from (430'3700,459'10636] MIN to 459'10638 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.9 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10781 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: 14.0 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10638 2023-06-10T21:49:20.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:20 smithi152 ceph-mon[156819]: pgmap v1701: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 474 op/s; 48018/133554 objects misplaced (35.954%); 9.9 MiB/s, 159 objects/s recovering 2023-06-10T21:49:20.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: osdmap e462: 8 total, 8 up, 5 in 2023-06-10T21:49:20.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.9 continuing backfill to osd.0 from (435'5000,459'10781] 14:9e8ff083:::benchmark_data_smithi064_559916_object80242:head to 459'10781 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'10767 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.a continuing backfill to osd.6 from (435'4900,459'10687] MIN to 459'10687 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.0 starting backfill to osd.0 from (430'3700,459'10636] MIN to 459'10638 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.9 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10781 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: 14.0 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10638 2023-06-10T21:49:20.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[180297]: pgmap v1701: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 474 op/s; 48018/133554 objects misplaced (35.954%); 9.9 MiB/s, 159 objects/s recovering 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: osdmap e462: 8 total, 8 up, 5 in 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: failed to encode map e462 with expected crc 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.9 continuing backfill to osd.0 from (435'5000,459'10781] 14:9e8ff083:::benchmark_data_smithi064_559916_object80242:head to 459'10781 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.f starting backfill to osd.6 from (0'0,0'0] MAX to 459'10767 2023-06-10T21:49:20.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.a continuing backfill to osd.6 from (435'4900,459'10687] MIN to 459'10687 2023-06-10T21:49:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.0 starting backfill to osd.0 from (430'3700,459'10636] MIN to 459'10638 2023-06-10T21:49:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.9 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10781 2023-06-10T21:49:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: 14.0 starting backfill to osd.6 from (0'0,0'0] MAX to 459'10638 2023-06-10T21:49:20.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:20 smithi064 ceph-mon[186706]: pgmap v1701: 89 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 12 active+remapped+backfill_wait, 74 active+clean; 2.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 474 op/s; 48018/133554 objects misplaced (35.954%); 9.9 MiB/s, 159 objects/s recovering 2023-06-10T21:49:21.219 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:21.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:21.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:21.385 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:21.487 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:21.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:21.595 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:21.663 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:21.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:21 smithi152 ceph-mon[156819]: failed to encode map e462 with expected crc 2023-06-10T21:49:21.856 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:21.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:21.866 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:21 smithi064 ceph-mon[180297]: failed to encode map e462 with expected crc 2023-06-10T21:49:21.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:21.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:21 smithi064 ceph-mon[186706]: failed to encode map e462 with expected crc 2023-06-10T21:49:21.908 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:49:21.909 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:49:21.967 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:22.032 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:22.136 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:22.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:22.245 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:22.304 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:22.387 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:22.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:22.494 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:49:22.500 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:22.537 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:49:22.560 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:22 smithi152 ceph-mon[156819]: pgmap v1702: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 53575/129996 objects misplaced (41.213%); 21 MiB/s, 334 objects/s recovering 2023-06-10T21:49:22.563 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:22.595 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:49:22.651 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:22.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:22.745 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:22 smithi064 ceph-mon[186706]: pgmap v1702: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 53575/129996 objects misplaced (41.213%); 21 MiB/s, 334 objects/s recovering 2023-06-10T21:49:22.746 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:22 smithi064 ceph-mon[180297]: pgmap v1702: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 613 op/s; 53575/129996 objects misplaced (41.213%); 21 MiB/s, 334 objects/s recovering 2023-06-10T21:49:22.756 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:22.784 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:22.832 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:22.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:22.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:49:22.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:49:22.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:49:22.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:49:22.853 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.854 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.855 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:49:22.937 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:22.979 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:23.105 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:23.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:23.148 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:49:23.164 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:22] "GET /metrics HTTP/1.1" 200 224282 "" "Prometheus/2.33.4" 2023-06-10T21:49:23.218 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:23.286 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:23.423 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:23 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3800703064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:23.461 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:49:23.512 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:23 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3800703064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:23.512 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:23 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3800703064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:23.557 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:23.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:23.664 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:23.691 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:23.739 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:23.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:23.845 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:23.889 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:23.972 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:23.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:23.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:24.079 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:24.135 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:24.241 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:24.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:24.352 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:24.428 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:24.463 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:24.526 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:24.562 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:24.570 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:24 smithi064 ceph-mon[180297]: pgmap v1703: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 53575/129591 objects misplaced (41.342%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:49:24.570 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:24 smithi064 ceph-mon[186706]: pgmap v1703: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 53575/129591 objects misplaced (41.342%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:49:24.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:24.672 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:24 smithi152 ceph-mon[156819]: pgmap v1703: 89 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 234 op/s; 53575/129591 objects misplaced (41.342%); 13 MiB/s, 215 objects/s recovering 2023-06-10T21:49:24.742 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:24.775 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:49:24.775 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:49:24.775 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-06-10T21:49:24.776 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:49:24.776 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:49:24.776 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-06-10T21:49:24.990 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:25.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:25.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:25.102 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:25.160 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:25.240 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:25.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:25.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:25.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:25.421 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:25.523 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:25.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:25.639 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:25.716 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:26.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:49:26.842 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:26.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:26.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:26.953 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:27.065 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:27.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:27.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:27.373 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:27.455 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:27.489 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:27.551 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:27.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:27 smithi064 ceph-mon[180297]: pgmap v1704: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 244 op/s; 54727/128595 objects misplaced (42.558%); 22 MiB/s, 345 objects/s recovering 2023-06-10T21:49:27.632 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152134 14.3 14:cd82057e:::benchmark_data_smithi064_559916_object44278:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e451)", 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:49:27.633 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.048609044999999, 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.031321833, 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.634 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152134 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570116+0000", 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570124+0000", 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5170000000000002e-06 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570137+0000", 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3825000000000001e-05 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570147+0000", 2023-06-10T21:49:27.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2900000000000008e-06 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.572192+0000", 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002045135 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.572201+0000", 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4639999999999995e-06 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601383+0000", 2023-06-10T21:49:27.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0291812229999999 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601440+0000", 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7219000000000002e-05 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152156 14.3 14:cc7f4242:::benchmark_data_smithi064_559916_object44300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.028592918000001, 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.032985703, 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.642 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152156 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:49:27.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590132+0000", 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590142+0000", 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4730000000000006e-06 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590161+0000", 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9055e-05 2023-06-10T21:49:27.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590187+0000", 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6055999999999999e-05 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590276+0000", 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9653000000000004e-05 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590289+0000", 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2323999999999999e-05 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601787+0000", 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0114977540000001 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601859+0000", 2023-06-10T21:49:27.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2046999999999996e-05 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.602296+0000", 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043782199999999998 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617514+0000", 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01521788 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617560+0000", 2023-06-10T21:49:27.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5865999999999998e-05 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623034+0000", 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005473449 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623080+0000", 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6686e-05 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623120+0000", 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9532e-05 2023-06-10T21:49:27.653 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152185 14.3 14:c57ea95c:::benchmark_data_smithi064_559916_object44329:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.656018+0000", 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.962708612, 2023-06-10T21:49:27.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97088494199999997, 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152185 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.655 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656018+0000", 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656018+0000", 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656017+0000", 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656021+0000", 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.245e-06 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656031+0000", 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2809999999999997e-06 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656038+0000", 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.922e-06 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656319+0000", 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00028100299999999999 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.656330+0000", 2023-06-10T21:49:27.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1691e-05 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.610885+0000", 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95455506999999995 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.611001+0000", 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011589799999999999 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.611121+0000", 2023-06-10T21:49:27.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000119868 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617568+0000", 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064468650000000004 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623173+0000", 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056049990000000003 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626810+0000", 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036369150000000001 2023-06-10T21:49:27.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626842+0000", 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2168000000000001e-05 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.665 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626903+0000", 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1321999999999993e-05 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152194 14.3 14:cd160727:::benchmark_data_smithi064_559916_object44338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.666 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.673327+0000", 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.945400235000001, 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96072460599999998, 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.667 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152194 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673327+0000", 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673327+0000", 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673325+0000", 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673332+0000", 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2300000000000002e-06 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673345+0000", 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2413e-05 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673351+0000", 2023-06-10T21:49:27.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.387e-06 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673415+0000", 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3441999999999993e-05 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.673422+0000", 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4080000000000001e-06 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.612725+0000", 2023-06-10T21:49:27.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93930253900000005 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.612836+0000", 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000111438 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.612946+0000", 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010943899999999999 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617573+0000", 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0046275470000000001 2023-06-10T21:49:27.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626955+0000", 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0093814160000000001 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.633939+0000", 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0069845949999999997 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.633987+0000", 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8025999999999997e-05 2023-06-10T21:49:27.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634051+0000", 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4239000000000006e-05 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152199 14.3 14:ca94c5cf:::benchmark_data_smithi064_559916_object44343:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.676218+0000", 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.942509328, 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95799937300000004, 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.679 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152199 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676218+0000", 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676218+0000", 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676216+0000", 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676223+0000", 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.939e-06 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676240+0000", 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7441000000000001e-05 2023-06-10T21:49:27.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676250+0000", 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6670000000000008e-06 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676310+0000", 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9577000000000002e-05 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.676321+0000", 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1613999999999999e-05 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.614600+0000", 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93827877699999995 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.614670+0000", 2023-06-10T21:49:27.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9654999999999996e-05 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.614734+0000", 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4633000000000003e-05 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617578+0000", 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028434559999999998 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.627029+0000", 2023-06-10T21:49:27.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094512309999999992 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634128+0000", 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070987050000000003 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634177+0000", 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9174e-05 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634217+0000", 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0292e-05 2023-06-10T21:49:27.690 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152213 14.3 14:c0cb596f:::benchmark_data_smithi064_559916_object44357:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.703033+0000", 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.915693648000001, 2023-06-10T21:49:27.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93138816000000002, 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152213 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.692 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703033+0000", 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703033+0000", 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703031+0000", 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703040+0000", 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5790000000000004e-06 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703055+0000", 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5203e-05 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703065+0000", 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6239999999999997e-06 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703126+0000", 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0711999999999997e-05 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.703137+0000", 2023-06-10T21:49:27.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1867e-05 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.616013+0000", 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91287597499999995 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.616132+0000", 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011849999999999999 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.616246+0000", 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114214 2023-06-10T21:49:27.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.621600+0000", 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053540519999999998 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.630588+0000", 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0089878479999999997 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634336+0000", 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037477000000000001 2023-06-10T21:49:27.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634386+0000", 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0160000000000001e-05 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634422+0000", 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5614999999999997e-05 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152214 14.3 14:c995371d:::benchmark_data_smithi064_559916_object44358:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.703 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.710501+0000", 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.908226014, 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92419117299999998, 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.704 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152214 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710501+0000", 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710501+0000", 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710499+0000", 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710511+0000", 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1578e-05 2023-06-10T21:49:27.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710529+0000", 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7938e-05 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710537+0000", 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0779999999999996e-06 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710595+0000", 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8204e-05 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.710606+0000", 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.081e-05 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617597+0000", 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90699062900000005 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617668+0000", 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1518000000000002e-05 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617753+0000", 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4442000000000002e-05 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623137+0000", 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053842020000000003 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.630646+0000", 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0075093989999999999 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634616+0000", 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039701370000000003 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634653+0000", 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7129999999999999e-05 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634692+0000", 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8788999999999997e-05 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.717 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152223 14.3 14:cbb8df59:::benchmark_data_smithi064_559916_object44367:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.733082+0000", 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.885644740000004, 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90555055699999998, 2023-06-10T21:49:27.718 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152223 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733082+0000", 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733082+0000", 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733080+0000", 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733089+0000", 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5290000000000008e-06 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733120+0000", 2023-06-10T21:49:27.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0549000000000002e-05 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733133+0000", 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3582e-05 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733518+0000", 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000385342 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.733527+0000", 2023-06-10T21:49:27.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1999999999999994e-06 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.619050+0000", 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88552325099999996 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.619114+0000", 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4517000000000006e-05 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.619179+0000", 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4749e-05 2023-06-10T21:49:27.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623145+0000", 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039662589999999998 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634286+0000", 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011140778 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638545+0000", 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0042590479999999997 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638596+0000", 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0769999999999997e-05 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638633+0000", 2023-06-10T21:49:27.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6789000000000003e-05 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152226 14.3 14:c9d5bf7a:::benchmark_data_smithi064_559916_object44370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.738471+0000", 2023-06-10T21:49:27.730 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.880256025999998, 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90030430500000003, 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.731 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152226 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738471+0000", 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738471+0000", 2023-06-10T21:49:27.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.734 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:27 smithi064 ceph-mon[186706]: pgmap v1704: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 244 op/s; 54727/128595 objects misplaced (42.558%); 22 MiB/s, 345 objects/s recovering 2023-06-10T21:49:27.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738469+0000", 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738477+0000", 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0260000000000007e-06 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738493+0000", 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.562e-05 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738501+0000", 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7120000000000006e-06 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738573+0000", 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1592000000000002e-05 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738581+0000", 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1249999999999993e-06 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.620330+0000", 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.881748482 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.620439+0000", 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010906 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.620502+0000", 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3650000000000002e-05 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.625054+0000", 2023-06-10T21:49:27.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045519649999999998 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634509+0000", 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0094545040000000007 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638704+0000", 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0041952930000000001 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638744+0000", 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9542999999999997e-05 2023-06-10T21:49:27.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638775+0000", 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1628999999999998e-05 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.743 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152228 14.3 14:c5fb0872:::benchmark_data_smithi064_559916_object44372:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.738516+0000", 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.880210812999998, 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.900390248, 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.744 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152228 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738516+0000", 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738516+0000", 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738516+0000", 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738519+0000", 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5779999999999999e-06 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738527+0000", 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2960000000000001e-06 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738532+0000", 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8200000000000002e-06 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738594+0000", 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1960999999999998e-05 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.738598+0000", 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1019999999999999e-06 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.621636+0000", 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88303748299999996 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.621745+0000", 2023-06-10T21:49:27.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108618 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.621878+0000", 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013359099999999999 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.625062+0000", 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0031835919999999998 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638675+0000", 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013612872999999999 2023-06-10T21:49:27.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638837+0000", 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162282 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638876+0000", 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9548000000000002e-05 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638906+0000", 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0052e-05 2023-06-10T21:49:27.755 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152286 14.3 14:c5b006f1:::benchmark_data_smithi064_559916_object44430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.903606+0000", 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.715120861999999, 2023-06-10T21:49:27.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.73549474599999998, 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152286 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.757 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903606+0000", 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903606+0000", 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903604+0000", 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903612+0000", 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4989999999999997e-06 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903632+0000", 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0063000000000001e-05 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903642+0000", 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8949999999999993e-06 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903705+0000", 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2972999999999995e-05 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.903717+0000", 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2707999999999999e-05 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623201+0000", 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.71948369199999995 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623286+0000", 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4509e-05 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623423+0000", 2023-06-10T21:49:27.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000137742 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626962+0000", 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003538958 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638808+0000", 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011845979 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639008+0000", 2023-06-10T21:49:27.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019948200000000001 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639061+0000", 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3245000000000003e-05 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639101+0000", 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9872e-05 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.768 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152305 14.3 14:cac54e0f:::benchmark_data_smithi064_559916_object44449:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:40.057433+0000", 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.561294105999998, 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.58183426199999999, 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.769 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152305 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057433+0000", 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057433+0000", 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057431+0000", 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057439+0000", 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2110000000000001e-06 2023-06-10T21:49:27.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057455+0000", 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6912e-05 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.057464+0000", 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9949999999999994e-06 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.058970+0000", 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001505468 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.058981+0000", 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1593999999999999e-05 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.625100+0000", 2023-06-10T21:49:27.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.56611845199999999 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.625178+0000", 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8139000000000005e-05 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.625250+0000", 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1589999999999997e-05 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.626972+0000", 2023-06-10T21:49:27.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00172186 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.638957+0000", 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011985847000000001 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639183+0000", 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022539399999999999 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639228+0000", 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5649000000000003e-05 2023-06-10T21:49:27.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639267+0000", 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8739000000000003e-05 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152337 14.3 14:c7d84371:::benchmark_data_smithi064_559916_object44481:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:40.126775+0000", 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.491951856999997, 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.51267081599999997, 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.781 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152337 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126775+0000", 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126775+0000", 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126774+0000", 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126780+0000", 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2859999999999996e-06 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126797+0000", 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6469000000000001e-05 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126805+0000", 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4379999999999995e-06 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126858+0000", 2023-06-10T21:49:27.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3214000000000002e-05 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.126866+0000", 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6499999999999996e-06 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.627069+0000", 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.50020295299999995 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.627171+0000", 2023-06-10T21:49:27.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102713 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.627292+0000", 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000120581 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.632456+0000", 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051643690000000003 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639147+0000", 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066909100000000004 2023-06-10T21:49:27.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639346+0000", 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019904800000000001 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639405+0000", 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8798000000000002e-05 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639446+0000", 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.0781999999999997e-05 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.792 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152339 14.3 14:c4352030:::benchmark_data_smithi064_559916_object44483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:40.132468+0000", 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.486259189000002, 2023-06-10T21:49:27.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.507139962, 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152339 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.794 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132468+0000", 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132468+0000", 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132466+0000", 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132473+0000", 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7569999999999998e-06 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132489+0000", 2023-06-10T21:49:27.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5954e-05 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132497+0000", 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5989999999999996e-06 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132560+0000", 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3270999999999998e-05 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.132571+0000", 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1117999999999999e-05 2023-06-10T21:49:27.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.629030+0000", 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.49645857900000001 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.629155+0000", 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012546500000000001 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.629287+0000", 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000131447 2023-06-10T21:49:27.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634072+0000", 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047857569999999999 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639309+0000", 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052365279999999998 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639524+0000", 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00021498599999999999 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639569+0000", 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5126000000000001e-05 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639608+0000", 2023-06-10T21:49:27.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8862000000000002e-05 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152348 14.3 14:ce336f22:::benchmark_data_smithi064_559916_object44492:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:40.167492+0000", 2023-06-10T21:49:27.805 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 47.451235109999999, 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.47223861099999997, 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.806 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152348 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167492+0000", 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167492+0000", 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167490+0000", 2023-06-10T21:49:27.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167497+0000", 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7569999999999998e-06 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167510+0000", 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2532e-05 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167518+0000", 2023-06-10T21:49:27.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8750000000000006e-06 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167574+0000", 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5040000000000002e-05 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.167586+0000", 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2619000000000001e-05 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.630704+0000", 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.46311766300000001 2023-06-10T21:49:27.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.630819+0000", 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011550899999999999 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:27.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.630902+0000", 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2721999999999998e-05 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.634296+0000", 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033940400000000001 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639487+0000", 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051911869999999999 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639647+0000", 2023-06-10T21:49:27.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015929400000000001 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639693+0000", 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6721000000000002e-05 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.639731+0000", 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7237999999999998e-05 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:164803 14.3 14:cd37e68d:::benchmark_data_smithi064_559916_object56947:head [delete] snapc 0=[] ondisk+write+known_if_redirected e458)", 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:04.473781+0000", 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 23.144945595999999, 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.16007911899999999, 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:27.818 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 164803 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473781+0000", 2023-06-10T21:49:27.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473781+0000", 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473780+0000", 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473785+0000", 2023-06-10T21:49:27.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3879999999999999e-06 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473801+0000", 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6072000000000001e-05 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.473809+0000", 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7579999999999992e-06 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.474914+0000", 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011053359999999999 2023-06-10T21:49:27.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.475062+0000", 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014808399999999999 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,6,7", 2023-06-10T21:49:27.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.475209+0000", 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014649800000000001 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.478100+0000", 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028911610000000002 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.478574+0000", 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047415100000000002 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.489942+0000", 2023-06-10T21:49:27.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011367469999999999 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.633697+0000", 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.14375501600000001 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.633740+0000", 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3713e-05 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:04.633861+0000", 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012027399999999999 2023-06-10T21:49:27.828 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:165316 14.6 e459/439 14:6afd87e4:::benchmark_data_smithi064_559916_object57460:head v 459'10110, mlcod=459'10108)", 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:05.384943+0000", 2023-06-10T21:49:27.829 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.233784408999998, 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.43192242199999997, 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384943+0000", 2023-06-10T21:49:27.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384943+0000", 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384942+0000", 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384948+0000", 2023-06-10T21:49:27.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6149999999999996e-06 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384957+0000", 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1910000000000006e-06 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.384964+0000", 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5939999999999999e-06 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.385008+0000", 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4567999999999997e-05 2023-06-10T21:49:27.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.385029+0000", 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0902000000000001e-05 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.816758+0000", 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.43172884 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.816865+0000", 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000107178 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.836 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPushReply(14.3 459/455 [PushReplyOp(14:c5c8d6db:::benchmark_data_smithi064_559916_object64766:head)])", 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:05.388332+0000", 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 22.230394941, 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.428880965, 2023-06-10T21:49:27.837 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388332+0000", 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388332+0000", 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388332+0000", 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967296 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388334+0000", 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5339999999999998e-06 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388339+0000", 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.656e-06 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388344+0000", 2023-06-10T21:49:27.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6940000000000003e-06 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.388400+0000", 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6564000000000003e-05 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:05.817213+0000", 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.42881268099999997 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:165883 14.5 e459/439 14:a080bf8b:::benchmark_data_smithi064_559916_object58027:head v 459'10453, mlcod=459'10450)", 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:06.725682+0000", 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.893045248, 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12796452, 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.725682+0000", 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.725682+0000", 2023-06-10T21:49:27.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.725681+0000", 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.725688+0000", 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0929999999999997e-06 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847522+0000", 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.121833633 2023-06-10T21:49:27.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847534+0000", 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2771e-05 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847581+0000", 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6794e-05 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847607+0000", 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5635000000000001e-05 2023-06-10T21:49:27.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.853607+0000", 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060003729999999998 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.853646+0000", 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9099e-05 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.47150.0:165886 14.6 e459/439 14:6e4b0c80:::benchmark_data_smithi064_559916_object58030:head v 459'10150, mlcod=459'10148)", 2023-06-10T21:49:27.851 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:06.729725+0000", 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.889002185999999, 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.12804489999999999, 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:27.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.729725+0000", 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.729725+0000", 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:27.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.729724+0000", 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:27.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.729735+0000", 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1168999999999999e-05 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847525+0000", 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.117790378 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847536+0000", 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0521e-05 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847598+0000", 2023-06-10T21:49:27.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2138999999999996e-05 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.847623+0000", 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4854999999999998e-05 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.857738+0000", 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010114906999999999 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:06.857770+0000", 2023-06-10T21:49:27.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2100000000000001e-05 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:27.859 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:49:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:27.860 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:27.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:27.861 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:27 smithi152 ceph-mon[156819]: pgmap v1704: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 244 op/s; 54727/128595 objects misplaced (42.558%); 22 MiB/s, 345 objects/s recovering 2023-06-10T21:49:27.962 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:28.040 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:28.277 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:28.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:28.382 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:28.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:28.395 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:28 smithi064 ceph-mon[180297]: pgmap v1705: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 51381/121635 objects misplaced (42.242%); 17 MiB/s, 278 objects/s recovering 2023-06-10T21:49:28.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:28.396 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:28 smithi064 ceph-mon[186706]: pgmap v1705: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 51381/121635 objects misplaced (42.242%); 17 MiB/s, 278 objects/s recovering 2023-06-10T21:49:28.427 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:49:28.485 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:28.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:28.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:28 smithi152 ceph-mon[156819]: pgmap v1705: 89 pgs: 1 active+remapped+backfilling, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.5 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 430 op/s; 51381/121635 objects misplaced (42.242%); 17 MiB/s, 278 objects/s recovering 2023-06-10T21:49:29.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:49:29.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:49:29.550 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:29.551 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[186706]: osdmap e463: 8 total, 8 up, 5 in 2023-06-10T21:49:29.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:49:29.551 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:49:29.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:29.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:29 smithi064 ceph-mon[180297]: osdmap e463: 8 total, 8 up, 5 in 2023-06-10T21:49:29.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:49:29.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:49:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: failed to encode map e463 with expected crc 2023-06-10T21:49:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/137449836' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:29 smithi152 ceph-mon[156819]: osdmap e463: 8 total, 8 up, 5 in 2023-06-10T21:49:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: osdmap e464: 8 total, 8 up, 5 in 2023-06-10T21:49:30.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:30 smithi152 ceph-mon[156819]: pgmap v1708: 89 pgs: 1 active+clean+remapped, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 49844/119919 objects misplaced (41.565%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: osdmap e464: 8 total, 8 up, 5 in 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[180297]: pgmap v1708: 89 pgs: 1 active+clean+remapped, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 49844/119919 objects misplaced (41.565%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: osdmap e464: 8 total, 8 up, 5 in 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: failed to encode map e464 with expected crc 2023-06-10T21:49:30.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:30 smithi064 ceph-mon[186706]: pgmap v1708: 89 pgs: 1 active+clean+remapped, 13 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 419 op/s; 49844/119919 objects misplaced (41.565%); 20 MiB/s, 318 objects/s recovering 2023-06-10T21:49:31.392 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:31.398 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:49:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:31.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:31.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:31.528 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:31.570 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:31.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: osdmap e465: 8 total, 8 up, 5 in 2023-06-10T21:49:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: osdmap e465: 8 total, 8 up, 5 in 2023-06-10T21:49:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:31 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.678 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: osdmap e465: 8 total, 8 up, 5 in 2023-06-10T21:49:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:31 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:31.776 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:31.918 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:31.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:32.037 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:32.074 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:32.156 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:32.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:32.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:32.308 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:32.380 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:32.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:32.487 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:32.527 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:32.558 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:32.597 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:32.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:32.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:32 smithi064 ceph-mon[180297]: failed to encode map e465 with expected crc 2023-06-10T21:49:32.620 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:32 smithi064 ceph-mon[180297]: pgmap v1710: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 850 op/s; 46088/113274 objects misplaced (40.687%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:32.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:32 smithi064 ceph-mon[186706]: failed to encode map e465 with expected crc 2023-06-10T21:49:32.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:32 smithi064 ceph-mon[186706]: pgmap v1710: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 850 op/s; 46088/113274 objects misplaced (40.687%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:32.672 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:49:32.672 INFO:teuthology.orchestra.run.smithi064.stdout: "success": "" 2023-06-10T21:49:32.672 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:49:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:32 smithi152 ceph-mon[156819]: failed to encode map e465 with expected crc 2023-06-10T21:49:32.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:32 smithi152 ceph-mon[156819]: pgmap v1710: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 850 op/s; 46088/113274 objects misplaced (40.687%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:32.700 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:49:32.706 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:32.789 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:32.884 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:32] "GET /metrics HTTP/1.1" 200 224282 "" "Prometheus/2.33.4" 2023-06-10T21:49:33.034 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:49:33.099 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:49:33.135 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:49:33.164 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:33.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:33.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:33.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:49:33.205 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:49:33.271 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:49:33.271 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:33.303 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:49:33.305 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:33.341 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:49:33.347 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.347 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.348 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.349 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:49:33.399 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:33.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:33.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2542506044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:33.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:33.481 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:33.509 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:33.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2542506044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:33.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:33.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:33.556 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:33.664 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:33.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:33.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:33.778 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:33.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2542506044' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:33.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:33.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:33.861 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:34.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:34 smithi064 ceph-mon[180297]: pgmap v1711: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 495 op/s; 46088/112722 objects misplaced (40.886%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:34.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:34 smithi064 ceph-mon[186706]: pgmap v1711: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 495 op/s; 46088/112722 objects misplaced (40.886%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:34 smithi152 ceph-mon[156819]: pgmap v1711: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 12 active+remapped+backfill_wait, 1 active+remapped, 74 active+clean; 2.3 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 495 op/s; 46088/112722 objects misplaced (40.886%); 19 MiB/s, 302 objects/s recovering 2023-06-10T21:49:35.317 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:35.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:35.426 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:35.480 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:35.561 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:35.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:35.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:35.722 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:35.763 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:35.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:35.870 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:35.896 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:35.937 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:35.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:36.043 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:36.103 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:36.210 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:36.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:36.317 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:36.389 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:36.476 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:36.523 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:36.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:36.636 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:36.663 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:36 smithi064 ceph-mon[180297]: pgmap v1712: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.3 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 45915/111720 objects misplaced (41.098%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:49:36.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:36 smithi064 ceph-mon[186706]: pgmap v1712: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.3 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 45915/111720 objects misplaced (41.098%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:49:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:36 smithi152 ceph-mon[156819]: pgmap v1712: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.3 GiB data, 9.7 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 417 op/s; 45915/111720 objects misplaced (41.098%); 14 MiB/s, 228 objects/s recovering 2023-06-10T21:49:36.694 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:36.783 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:36.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:36.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:36.965 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:37.091 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:49:37.091 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:49:37.098 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:37.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:37.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:37.294 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:37.416 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:49:38.646 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:38.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:38.754 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:38.798 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:39.337 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:49:39.418 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:39.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:39.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:39.533 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:39.646 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:39 smithi064 ceph-mon[180297]: pgmap v1713: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 572 op/s; 42046/104643 objects misplaced (40.180%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:49:39.647 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:39 smithi064 ceph-mon[186706]: pgmap v1713: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 572 op/s; 42046/104643 objects misplaced (40.180%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:49:39.666 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:39 smithi152 ceph-mon[156819]: pgmap v1713: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 572 op/s; 42046/104643 objects misplaced (40.180%); 7.4 MiB/s, 118 objects/s recovering 2023-06-10T21:49:39.790 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:39.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:39.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:39.920 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:40.006 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:40.036 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:49:40.036 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:49:40.037 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2023-06-10T21:49:40.037 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:49:40.040 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:40.156 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:40.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:40.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:40.263 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:40.296 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:40.377 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:40.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:40.405 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:40.490 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:40.523 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:40.589 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:40.593 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:40 smithi152 ceph-mon[156819]: pgmap v1714: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 41866/102945 objects misplaced (40.668%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:49:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:40.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:40.695 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:40.749 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:40.814 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:40 smithi064 ceph-mon[180297]: pgmap v1714: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 41866/102945 objects misplaced (40.668%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:49:40.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:40 smithi064 ceph-mon[186706]: pgmap v1714: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 2.1 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 41866/102945 objects misplaced (40.668%); 6.6 MiB/s, 105 objects/s recovering 2023-06-10T21:49:40.824 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:40.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:40.931 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:40.972 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:41.074 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:41.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:41.186 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:41.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:41.344 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:41.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:41.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:41.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:41.509 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1568898646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.510 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:41 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.512 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:41.513 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:49:41.571 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:41.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:41.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:41.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1568898646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:41 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1568898646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:41 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:49:41.683 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:41.770 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:42.856 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: osdmap e466: 8 total, 8 up, 5 in 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:49:42.857 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[180297]: pgmap v1716: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37884/95562 objects misplaced (39.643%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: osdmap e466: 8 total, 8 up, 5 in 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:49:42.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:42 smithi064 ceph-mon[186706]: pgmap v1716: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37884/95562 objects misplaced (39.643%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:42.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:42.950 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: osdmap e466: 8 total, 8 up, 5 in 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: failed to encode map e466 with expected crc 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:49:42.951 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:42 smithi152 ceph-mon[156819]: pgmap v1716: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37884/95562 objects misplaced (39.643%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:43.128 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:43.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:43.141 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:42] "GET /metrics HTTP/1.1" 200 224262 "" "Prometheus/2.33.4" 2023-06-10T21:49:43.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:43.300 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:43.399 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:43.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:43.508 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:43.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:43 smithi152 ceph-mon[156819]: failed to encode map e466 with expected crc 2023-06-10T21:49:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:43 smithi152 ceph-mon[156819]: failed to encode map e466 with expected crc 2023-06-10T21:49:43.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:43 smithi064 ceph-mon[186706]: failed to encode map e466 with expected crc 2023-06-10T21:49:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:43 smithi064 ceph-mon[186706]: failed to encode map e466 with expected crc 2023-06-10T21:49:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:43 smithi064 ceph-mon[180297]: failed to encode map e466 with expected crc 2023-06-10T21:49:43.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:43 smithi064 ceph-mon[180297]: failed to encode map e466 with expected crc 2023-06-10T21:49:44.140 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:44.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:44.194 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:44.250 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:44.359 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:44.601 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:44.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:44.711 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:44.792 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:44 smithi064 ceph-mon[186706]: pgmap v1717: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 37884/95058 objects misplaced (39.854%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:44.794 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:44 smithi064 ceph-mon[180297]: pgmap v1717: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 37884/95058 objects misplaced (39.854%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:44.801 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:44.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:44 smithi152 ceph-mon[156819]: pgmap v1717: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 588 op/s; 37884/95058 objects misplaced (39.854%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:45.015 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:49:45.048 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:45.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:45.088 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:49:45.134 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:49:45.156 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:45.173 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:49:45.213 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:49:45.243 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:45.256 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:49:45.303 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:49:45.362 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:49:45.383 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.384 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.385 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.386 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:49:45.391 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:45.392 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:49:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:49:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:49:45.392 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:49:45.393 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:49:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:45.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:45.498 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:45.592 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:45.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2365759608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:45.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2365759608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:45.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2365759608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:46.186 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:49:47.397 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:47.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:47.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:47.504 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:47.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.554 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: pgmap v1718: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37702/94005 objects misplaced (40.106%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: osdmap e467: 8 total, 8 up, 5 in 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.555 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:47 smithi152 ceph-mon[156819]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.566 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:47.655 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:47.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: pgmap v1718: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37702/94005 objects misplaced (40.106%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:47.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: osdmap e467: 8 total, 8 up, 5 in 2023-06-10T21:49:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[186706]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:47.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: pgmap v1718: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.9 GiB data, 8.3 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 590 op/s; 37702/94005 objects misplaced (40.106%); 6.7 MiB/s, 107 objects/s recovering 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.667 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: osdmap e467: 8 total, 8 up, 5 in 2023-06-10T21:49:47.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:47 smithi064 ceph-mon[180297]: failed to encode map e467 with expected crc 2023-06-10T21:49:47.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:47.791 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:48.541 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:48.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:48.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:48.649 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:48.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: osdmap e468: 8 total, 8 up, 5 in 2023-06-10T21:49:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:48.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[186706]: pgmap v1721: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 770 op/s; 33800/87099 objects misplaced (38.806%); 10 MiB/s, 166 objects/s recovering 2023-06-10T21:49:48.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: osdmap e468: 8 total, 8 up, 5 in 2023-06-10T21:49:48.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.668 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:48.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:48.669 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:48 smithi064 ceph-mon[180297]: pgmap v1721: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 770 op/s; 33800/87099 objects misplaced (38.806%); 10 MiB/s, 166 objects/s recovering 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: osdmap e468: 8 total, 8 up, 5 in 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: failed to encode map e468 with expected crc 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:49:48.679 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:48 smithi152 ceph-mon[156819]: pgmap v1721: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.8 GiB data, 8.1 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 770 op/s; 33800/87099 objects misplaced (38.806%); 10 MiB/s, 166 objects/s recovering 2023-06-10T21:49:48.682 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:48.696 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:49:48.759 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:48.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:48.865 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:48.902 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:48.966 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:48.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:49.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:49.124 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:49.184 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:49.186 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:49.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:49.291 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:49:49.328 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:49.410 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:49:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:49.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:49:49.516 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:49.590 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:49.772 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:49.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:49.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:49.956 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:49.964 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:49:49.966 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:49:50.254 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:50.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:50.359 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:50.437 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:50.521 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:50.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:50.633 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:50.641 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:49:50.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:50 smithi064 ceph-mon[186706]: from='client.57184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:49:50.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:50 smithi064 ceph-mon[186706]: pgmap v1722: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 408 op/s; 33651/85764 objects misplaced (39.237%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:49:50.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:50 smithi064 ceph-mon[180297]: from='client.57184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:49:50.662 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:50 smithi064 ceph-mon[180297]: pgmap v1722: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 408 op/s; 33651/85764 objects misplaced (39.237%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:49:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:50 smithi152 ceph-mon[156819]: from='client.57184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:49:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:50 smithi152 ceph-mon[156819]: pgmap v1722: 89 pgs: 1 peering, 1 active+remapped+backfilling, 11 active+remapped+backfill_wait, 1 active+remapped, 75 active+clean; 1.7 GiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 408 op/s; 33651/85764 objects misplaced (39.237%); 4.6 MiB/s, 74 objects/s recovering 2023-06-10T21:49:50.707 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:51.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:51.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:52.011 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:52.079 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:52.155 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:52.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:52.274 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:52.315 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:52.357 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:52.369 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:52.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:52.383 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1721,"stamp":"2023-06-10T21:49:48.324971+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1901664166,"num_objects":29033,"num_object_clones":0,"num_object_copies":87099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":33800,"num_objects_unfound":0,"num_objects_dirty":29033,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":186960,"num_write_kb":6905039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21381,"num_bytes_recovered":1395620644,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97336,"ondisk_log_size":97336,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8446484,"kb_used_data":4699372,"kb_used_omap":37,"kb_used_meta":3746970,"kb_avail":460279276,"statfs":{"total":479975178240,"available":471325978624,"internally_reserved":0,"allocated":4812156928,"data_stored":4806014443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38699,"internal_metadata":3836897493},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,1,1,3,1,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-346161152,"num_objects":-5282,"num_object_clones":0,"num_object_copies":-15846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-8066,"num_objects_unfound":0,"num_objects_dirty":-5282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5282,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1144,"num_bytes_recovered":74973184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.856968"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":517,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.473062+0000","last_change":"2023-06-10T21:47:18.342858+0000","last_active":"2023-06-10T21:49:41.473062+0000","last_peered":"2023-06-10T21:49:41.473062+0000","last_clean":"2023-06-10T21:49:41.473062+0000","last_became_active":"2023-06-10T21:47:18.342378+0000","last_became_peered":"2023-06-10T21:47:18.342378+0000","last_unstale":"2023-06-10T21:49:41.473062+0000","last_undegraded":"2023-06-10T21:49:41.473062+0000","last_fullsized":"2023-06-10T21:49:41.473062+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:30.423525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":155,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.479805+0000","last_change":"2023-06-10T21:48:53.473755+0000","last_active":"2023-06-10T21:49:41.479805+0000","last_peered":"2023-06-10T21:49:41.479805+0000","last_clean":"2023-06-10T21:49:41.479805+0000","last_became_active":"2023-06-10T21:48:53.473408+0000","last_became_peered":"2023-06-10T21:48:53.473408+0000","last_unstale":"2023-06-10T21:49:41.479805+0000","last_undegraded":"2023-06-10T21:49:41.479805+0000","last_fullsized":"2023-06-10T21:49:41.479805+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:37:52.851231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":418,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.479622+0000","last_change":"2023-06-10T21:48:53.404523+0000","last_active":"2023-06-10T21:49:41.479622+0000","last_peered":"2023-06-10T21:49:41.479622+0000","last_clean":"2023-06-10T21:49:41.479622+0000","last_became_active":"2023-06-10T21:48:53.403684+0000","last_became_peered":"2023-06-10T21:48:53.403684+0000","last_unstale":"2023-06-10T21:49:41.479622+0000","last_undegraded":"2023-06-10T21:49:41.479622+0000","last_fullsized":"2023-06-10T21:49:41.479622+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:06:20.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":66,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.465730+0000","last_change":"2023-06-10T21:49:18.402971+0000","last_active":"2023-06-10T21:49:30.465730+0000","last_peered":"2023-06-10T21:49:30.465730+0000","last_clean":"2023-06-10T21:49:30.465730+0000","last_became_active":"2023-06-10T21:49:18.402645+0000","last_became_peered":"2023-06-10T21:49:18.402645+0000","last_unstale":"2023-06-10T21:49:30.465730+0000","last_undegraded":"2023-06-10T21:49:30.465730+0000","last_fullsized":"2023-06-10T21:49:30.465730+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:57:01.556949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478250+0000","last_change":"2023-06-10T21:47:18.339120+0000","last_active":"2023-06-10T21:49:41.478250+0000","last_peered":"2023-06-10T21:49:41.478250+0000","last_clean":"2023-06-10T21:49:41.478250+0000","last_became_active":"2023-06-10T21:47:18.338758+0000","last_became_peered":"2023-06-10T21:47:18.338758+0000","last_unstale":"2023-06-10T21:49:41.478250+0000","last_undegraded":"2023-06-10T21:49:41.478250+0000","last_fullsized":"2023-06-10T21:49:41.478250+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:10:24.375802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428361+0000","last_change":"2023-06-10T21:49:18.402969+0000","last_active":"2023-06-10T21:49:30.428361+0000","last_peered":"2023-06-10T21:49:30.428361+0000","last_clean":"2023-06-10T21:49:30.428361+0000","last_became_active":"2023-06-10T21:49:18.402624+0000","last_became_peered":"2023-06-10T21:49:18.402624+0000","last_unstale":"2023-06-10T21:49:30.428361+0000","last_undegraded":"2023-06-10T21:49:30.428361+0000","last_fullsized":"2023-06-10T21:49:30.428361+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":48,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.485391+0000","last_change":"2023-06-10T21:49:18.403030+0000","last_active":"2023-06-10T21:49:41.485391+0000","last_peered":"2023-06-10T21:49:41.485391+0000","last_clean":"2023-06-10T21:49:41.485391+0000","last_became_active":"2023-06-10T21:49:18.402590+0000","last_became_peered":"2023-06-10T21:49:18.402590+0000","last_unstale":"2023-06-10T21:49:41.485391+0000","last_undegraded":"2023-06-10T21:49:41.485391+0000","last_fullsized":"2023-06-10T21:49:41.485391+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:55.929958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":164,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478250+0000","last_change":"2023-06-10T21:49:18.398386+0000","last_active":"2023-06-10T21:49:41.478250+0000","last_peered":"2023-06-10T21:49:41.478250+0000","last_clean":"2023-06-10T21:49:41.478250+0000","last_became_active":"2023-06-10T21:49:18.398058+0000","last_became_peered":"2023-06-10T21:49:18.398058+0000","last_unstale":"2023-06-10T21:49:41.478250+0000","last_undegraded":"2023-06-10T21:49:41.478250+0000","last_fullsized":"2023-06-10T21:49:41.478250+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:12:47.857409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":144,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430808+0000","last_change":"2023-06-10T21:49:18.403576+0000","last_active":"2023-06-10T21:49:30.430808+0000","last_peered":"2023-06-10T21:49:30.430808+0000","last_clean":"2023-06-10T21:49:30.430808+0000","last_became_active":"2023-06-10T21:49:18.403071+0000","last_became_peered":"2023-06-10T21:49:18.403071+0000","last_unstale":"2023-06-10T21:49:30.430808+0000","last_undegraded":"2023-06-10T21:49:30.430808+0000","last_fullsized":"2023-06-10T21:49:30.430808+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":229,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.479641+0000","last_change":"2023-06-10T21:48:53.403919+0000","last_active":"2023-06-10T21:49:41.479641+0000","last_peered":"2023-06-10T21:49:41.479641+0000","last_clean":"2023-06-10T21:49:41.479641+0000","last_became_active":"2023-06-10T21:48:53.403593+0000","last_became_peered":"2023-06-10T21:48:53.403593+0000","last_unstale":"2023-06-10T21:49:41.479641+0000","last_undegraded":"2023-06-10T21:49:41.479641+0000","last_fullsized":"2023-06-10T21:49:41.479641+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:59.593873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":238,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430823+0000","last_change":"2023-06-10T21:47:18.431486+0000","last_active":"2023-06-10T21:49:30.430823+0000","last_peered":"2023-06-10T21:49:30.430823+0000","last_clean":"2023-06-10T21:49:30.430823+0000","last_became_active":"2023-06-10T21:47:18.326725+0000","last_became_peered":"2023-06-10T21:47:18.326725+0000","last_unstale":"2023-06-10T21:49:30.430823+0000","last_undegraded":"2023-06-10T21:49:30.430823+0000","last_fullsized":"2023-06-10T21:49:30.430823+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427673+0000","last_change":"2023-06-10T21:48:53.381964+0000","last_active":"2023-06-10T21:49:30.427673+0000","last_peered":"2023-06-10T21:49:30.427673+0000","last_clean":"2023-06-10T21:49:30.427673+0000","last_became_active":"2023-06-10T21:48:53.381436+0000","last_became_peered":"2023-06-10T21:48:53.381436+0000","last_unstale":"2023-06-10T21:49:30.427673+0000","last_undegraded":"2023-06-10T21:49:30.427673+0000","last_fullsized":"2023-06-10T21:49:30.427673+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":427,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430690+0000","last_change":"2023-06-10T21:48:53.849265+0000","last_active":"2023-06-10T21:49:30.430690+0000","last_peered":"2023-06-10T21:49:30.430690+0000","last_clean":"2023-06-10T21:49:30.430690+0000","last_became_active":"2023-06-10T21:48:53.630188+0000","last_became_peered":"2023-06-10T21:48:53.630188+0000","last_unstale":"2023-06-10T21:49:30.430690+0000","last_undegraded":"2023-06-10T21:49:30.430690+0000","last_fullsized":"2023-06-10T21:49:30.430690+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23580,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427634+0000","last_change":"2023-06-10T21:49:18.402957+0000","last_active":"2023-06-10T21:49:30.427634+0000","last_peered":"2023-06-10T21:49:30.427634+0000","last_clean":"2023-06-10T21:49:30.427634+0000","last_became_active":"2023-06-10T21:49:18.402621+0000","last_became_peered":"2023-06-10T21:49:18.402621+0000","last_unstale":"2023-06-10T21:49:30.427634+0000","last_undegraded":"2023-06-10T21:49:30.427634+0000","last_fullsized":"2023-06-10T21:49:30.427634+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":555,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430606+0000","last_change":"2023-06-10T21:48:53.936290+0000","last_active":"2023-06-10T21:49:30.430606+0000","last_peered":"2023-06-10T21:49:30.430606+0000","last_clean":"2023-06-10T21:49:30.430606+0000","last_became_active":"2023-06-10T21:48:53.631956+0000","last_became_peered":"2023-06-10T21:48:53.631956+0000","last_unstale":"2023-06-10T21:49:30.430606+0000","last_undegraded":"2023-06-10T21:49:30.430606+0000","last_fullsized":"2023-06-10T21:49:30.430606+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":416,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.480926+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:49:41.480926+0000","last_peered":"2023-06-10T21:49:41.480926+0000","last_clean":"2023-06-10T21:49:41.480926+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:49:41.480926+0000","last_undegraded":"2023-06-10T21:49:41.480926+0000","last_fullsized":"2023-06-10T21:49:41.480926+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":39,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:39.004858+0000","last_change":"2023-06-10T21:47:18.322156+0000","last_active":"2023-06-10T21:49:39.004858+0000","last_peered":"2023-06-10T21:49:39.004858+0000","last_clean":"2023-06-10T21:49:39.004858+0000","last_became_active":"2023-06-10T21:47:18.320854+0000","last_became_peered":"2023-06-10T21:47:18.320854+0000","last_unstale":"2023-06-10T21:49:39.004858+0000","last_undegraded":"2023-06-10T21:49:39.004858+0000","last_fullsized":"2023-06-10T21:49:39.004858+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:07:26.646183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":351,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430943+0000","last_change":"2023-06-10T21:48:53.385839+0000","last_active":"2023-06-10T21:49:30.430943+0000","last_peered":"2023-06-10T21:49:30.430943+0000","last_clean":"2023-06-10T21:49:30.430943+0000","last_became_active":"2023-06-10T21:48:53.385521+0000","last_became_peered":"2023-06-10T21:48:53.385521+0000","last_unstale":"2023-06-10T21:49:30.430943+0000","last_undegraded":"2023-06-10T21:49:30.430943+0000","last_fullsized":"2023-06-10T21:49:30.430943+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":496,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430472+0000","last_change":"2023-06-10T21:48:53.380288+0000","last_active":"2023-06-10T21:49:30.430472+0000","last_peered":"2023-06-10T21:49:30.430472+0000","last_clean":"2023-06-10T21:49:30.430472+0000","last_became_active":"2023-06-10T21:48:53.376181+0000","last_became_peered":"2023-06-10T21:48:53.376181+0000","last_unstale":"2023-06-10T21:49:30.430472+0000","last_undegraded":"2023-06-10T21:49:30.430472+0000","last_fullsized":"2023-06-10T21:49:30.430472+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":48,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.484802+0000","last_change":"2023-06-10T21:47:33.372564+0000","last_active":"2023-06-10T21:49:41.484802+0000","last_peered":"2023-06-10T21:49:41.484802+0000","last_clean":"2023-06-10T21:49:41.484802+0000","last_became_active":"2023-06-10T21:47:33.371667+0000","last_became_peered":"2023-06-10T21:47:33.371667+0000","last_unstale":"2023-06-10T21:49:41.484802+0000","last_undegraded":"2023-06-10T21:49:41.484802+0000","last_fullsized":"2023-06-10T21:49:41.484802+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:02.002487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":31,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427316+0000","last_change":"2023-06-10T21:48:53.680926+0000","last_active":"2023-06-10T21:49:30.427316+0000","last_peered":"2023-06-10T21:49:30.427316+0000","last_clean":"2023-06-10T21:49:30.427316+0000","last_became_active":"2023-06-10T21:48:53.379343+0000","last_became_peered":"2023-06-10T21:48:53.379343+0000","last_unstale":"2023-06-10T21:49:30.427316+0000","last_undegraded":"2023-06-10T21:49:30.427316+0000","last_fullsized":"2023-06-10T21:49:30.427316+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":309,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.477787+0000","last_change":"2023-06-10T21:48:53.387743+0000","last_active":"2023-06-10T21:49:41.477787+0000","last_peered":"2023-06-10T21:49:41.477787+0000","last_clean":"2023-06-10T21:49:41.477787+0000","last_became_active":"2023-06-10T21:48:53.385099+0000","last_became_peered":"2023-06-10T21:48:53.385099+0000","last_unstale":"2023-06-10T21:49:41.477787+0000","last_undegraded":"2023-06-10T21:49:41.477787+0000","last_fullsized":"2023-06-10T21:49:41.477787+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:50:59.882320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.2","version":"467'11662","reported_seq":11994,"reported_epoch":467,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:49:46.631975+0000","last_change":"2023-06-10T21:49:45.751739+0000","last_active":"2023-06-10T21:49:46.631975+0000","last_peered":"2023-06-10T21:49:46.631975+0000","last_clean":"2023-06-10T21:47:17.271926+0000","last_became_active":"2023-06-10T21:48:53.649930+0000","last_became_peered":"2023-06-10T21:48:53.649930+0000","last_unstale":"2023-06-10T21:49:46.631975+0000","last_undegraded":"2023-06-10T21:49:46.631975+0000","last_fullsized":"2023-06-10T21:49:46.631975+0000","mapping_epoch":454,"log_start":"437'6323","ondisk_log_start":"437'6323","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5339,"log_dups_size":0,"ondisk_log_size":5339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:02:53.749244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118358016,"num_objects":1806,"num_object_clones":0,"num_object_copies":5418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3460,"num_objects_unfound":0,"num_objects_dirty":1806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11662,"num_write_kb":430976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":5111808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1806}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":284,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.431307+0000","last_change":"2023-06-10T21:49:18.419142+0000","last_active":"2023-06-10T21:49:30.431307+0000","last_peered":"2023-06-10T21:49:30.431307+0000","last_clean":"2023-06-10T21:49:30.431307+0000","last_became_active":"2023-06-10T21:49:18.403225+0000","last_became_peered":"2023-06-10T21:49:18.403225+0000","last_unstale":"2023-06-10T21:49:30.431307+0000","last_undegraded":"2023-06-10T21:49:30.431307+0000","last_fullsized":"2023-06-10T21:49:30.431307+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":162,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430044+0000","last_change":"2023-06-10T21:48:53.381781+0000","last_active":"2023-06-10T21:49:30.430044+0000","last_peered":"2023-06-10T21:49:30.430044+0000","last_clean":"2023-06-10T21:49:30.430044+0000","last_became_active":"2023-06-10T21:48:53.381469+0000","last_became_peered":"2023-06-10T21:48:53.381469+0000","last_unstale":"2023-06-10T21:49:30.430044+0000","last_undegraded":"2023-06-10T21:49:30.430044+0000","last_fullsized":"2023-06-10T21:49:30.430044+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"466'11621","reported_seq":27903,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:43.973599+0000","last_change":"2023-06-10T21:49:18.444214+0000","last_active":"2023-06-10T21:49:43.973599+0000","last_peered":"2023-06-10T21:49:43.973599+0000","last_clean":"2023-06-10T21:49:43.973599+0000","last_became_active":"2023-06-10T21:49:18.403027+0000","last_became_peered":"2023-06-10T21:49:18.403027+0000","last_unstale":"2023-06-10T21:49:43.973599+0000","last_undegraded":"2023-06-10T21:49:43.973599+0000","last_fullsized":"2023-06-10T21:49:43.973599+0000","mapping_epoch":460,"log_start":"442'6641","ondisk_log_start":"442'6641","created":420,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4980,"log_dups_size":0,"ondisk_log_size":4980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:02:00.768054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120782848,"num_objects":1843,"num_object_clones":0,"num_object_copies":5529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11621,"num_write_kb":430848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8084,"num_bytes_recovered":529793024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":284,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430837+0000","last_change":"2023-06-10T21:47:33.362906+0000","last_active":"2023-06-10T21:49:30.430837+0000","last_peered":"2023-06-10T21:49:30.430837+0000","last_clean":"2023-06-10T21:49:30.430837+0000","last_became_active":"2023-06-10T21:47:33.349060+0000","last_became_peered":"2023-06-10T21:47:33.349060+0000","last_unstale":"2023-06-10T21:49:30.430837+0000","last_undegraded":"2023-06-10T21:49:30.430837+0000","last_fullsized":"2023-06-10T21:49:30.430837+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427540+0000","last_change":"2023-06-10T21:48:53.379866+0000","last_active":"2023-06-10T21:49:30.427540+0000","last_peered":"2023-06-10T21:49:30.427540+0000","last_clean":"2023-06-10T21:49:30.427540+0000","last_became_active":"2023-06-10T21:48:53.379538+0000","last_became_peered":"2023-06-10T21:48:53.379538+0000","last_unstale":"2023-06-10T21:49:30.427540+0000","last_undegraded":"2023-06-10T21:49:30.427540+0000","last_fullsized":"2023-06-10T21:49:30.427540+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"467'11649","reported_seq":11742,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.554779+0000","last_change":"2023-06-10T21:49:19.876807+0000","last_active":"2023-06-10T21:49:46.554779+0000","last_peered":"2023-06-10T21:49:46.554779+0000","last_clean":"2023-06-10T21:47:17.279925+0000","last_became_active":"2023-06-10T21:49:19.750040+0000","last_became_peered":"2023-06-10T21:49:19.750040+0000","last_unstale":"2023-06-10T21:49:46.554779+0000","last_undegraded":"2023-06-10T21:49:46.554779+0000","last_fullsized":"2023-06-10T21:49:46.554779+0000","mapping_epoch":461,"log_start":"434'4820","ondisk_log_start":"434'4820","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6829,"log_dups_size":0,"ondisk_log_size":6829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:09:25.232898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116457472,"num_objects":1777,"num_object_clones":0,"num_object_copies":5331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3554,"num_objects_unfound":0,"num_objects_dirty":1777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11649,"num_write_kb":429632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1777}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":571,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430863+0000","last_change":"2023-06-10T21:48:53.675024+0000","last_active":"2023-06-10T21:49:30.430863+0000","last_peered":"2023-06-10T21:49:30.430863+0000","last_clean":"2023-06-10T21:49:30.430863+0000","last_became_active":"2023-06-10T21:48:53.385337+0000","last_became_peered":"2023-06-10T21:48:53.385337+0000","last_unstale":"2023-06-10T21:49:30.430863+0000","last_undegraded":"2023-06-10T21:49:30.430863+0000","last_fullsized":"2023-06-10T21:49:30.430863+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427762+0000","last_change":"2023-06-10T21:49:18.401260+0000","last_active":"2023-06-10T21:49:30.427762+0000","last_peered":"2023-06-10T21:49:30.427762+0000","last_clean":"2023-06-10T21:49:30.427762+0000","last_became_active":"2023-06-10T21:49:18.400755+0000","last_became_peered":"2023-06-10T21:49:18.400755+0000","last_unstale":"2023-06-10T21:49:30.427762+0000","last_undegraded":"2023-06-10T21:49:30.427762+0000","last_fullsized":"2023-06-10T21:49:30.427762+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"467'11898","reported_seq":12061,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.627866+0000","last_change":"2023-06-10T21:49:19.881680+0000","last_active":"2023-06-10T21:49:46.627866+0000","last_peered":"2023-06-10T21:49:46.627866+0000","last_clean":"2023-06-10T21:47:32.299728+0000","last_became_active":"2023-06-10T21:48:53.629145+0000","last_became_peered":"2023-06-10T21:48:53.629145+0000","last_unstale":"2023-06-10T21:49:46.627866+0000","last_undegraded":"2023-06-10T21:49:46.627866+0000","last_fullsized":"2023-06-10T21:49:46.627866+0000","mapping_epoch":454,"log_start":"437'6550","ondisk_log_start":"437'6550","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5348,"log_dups_size":0,"ondisk_log_size":5348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:35:51.124004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120193024,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1811,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11898,"num_write_kb":439424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1834}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":145,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.480813+0000","last_change":"2023-06-10T21:49:18.395063+0000","last_active":"2023-06-10T21:49:41.480813+0000","last_peered":"2023-06-10T21:49:41.480813+0000","last_clean":"2023-06-10T21:49:41.480813+0000","last_became_active":"2023-06-10T21:49:18.394738+0000","last_became_peered":"2023-06-10T21:49:18.394738+0000","last_unstale":"2023-06-10T21:49:41.480813+0000","last_undegraded":"2023-06-10T21:49:41.480813+0000","last_fullsized":"2023-06-10T21:49:41.480813+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:47:33.832467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":48,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:34.004232+0000","last_change":"2023-06-10T21:49:18.403580+0000","last_active":"2023-06-10T21:49:34.004232+0000","last_peered":"2023-06-10T21:49:34.004232+0000","last_clean":"2023-06-10T21:49:34.004232+0000","last_became_active":"2023-06-10T21:49:18.403128+0000","last_became_peered":"2023-06-10T21:49:18.403128+0000","last_unstale":"2023-06-10T21:49:34.004232+0000","last_undegraded":"2023-06-10T21:49:34.004232+0000","last_fullsized":"2023-06-10T21:49:34.004232+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:25:18.227144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"467'11465","reported_seq":16539,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.525927+0000","last_change":"2023-06-10T21:49:19.747073+0000","last_active":"2023-06-10T21:49:46.525927+0000","last_peered":"2023-06-10T21:49:46.525927+0000","last_clean":"2023-06-10T21:47:17.257047+0000","last_became_active":"2023-06-10T21:47:19.603771+0000","last_became_peered":"2023-06-10T21:47:19.603771+0000","last_unstale":"2023-06-10T21:49:46.525927+0000","last_undegraded":"2023-06-10T21:49:46.525927+0000","last_fullsized":"2023-06-10T21:49:46.525927+0000","mapping_epoch":439,"log_start":"434'4600","ondisk_log_start":"434'4600","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6865,"log_dups_size":0,"ondisk_log_size":6865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:38:04.341765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":502,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11465,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2502,"num_bytes_recovered":163971072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1775}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":140,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478601+0000","last_change":"2023-06-10T21:47:18.440477+0000","last_active":"2023-06-10T21:49:41.478601+0000","last_peered":"2023-06-10T21:49:41.478601+0000","last_clean":"2023-06-10T21:49:41.478601+0000","last_became_active":"2023-06-10T21:47:18.339247+0000","last_became_peered":"2023-06-10T21:47:18.339247+0000","last_unstale":"2023-06-10T21:49:41.478601+0000","last_undegraded":"2023-06-10T21:49:41.478601+0000","last_fullsized":"2023-06-10T21:49:41.478601+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:44:14.648072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":132,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.480877+0000","last_change":"2023-06-10T21:48:53.389236+0000","last_active":"2023-06-10T21:49:41.480877+0000","last_peered":"2023-06-10T21:49:41.480877+0000","last_clean":"2023-06-10T21:49:41.480877+0000","last_became_active":"2023-06-10T21:48:53.388982+0000","last_became_peered":"2023-06-10T21:48:53.388982+0000","last_unstale":"2023-06-10T21:49:41.480877+0000","last_undegraded":"2023-06-10T21:49:41.480877+0000","last_fullsized":"2023-06-10T21:49:41.480877+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:02:53.667590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"467'11618","reported_seq":14385,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.494353+0000","last_change":"2023-06-10T21:48:54.567679+0000","last_active":"2023-06-10T21:49:46.494353+0000","last_peered":"2023-06-10T21:49:46.494353+0000","last_clean":"2023-06-10T21:47:17.256756+0000","last_became_active":"2023-06-10T21:47:33.555979+0000","last_became_peered":"2023-06-10T21:47:33.555979+0000","last_unstale":"2023-06-10T21:49:46.494353+0000","last_undegraded":"2023-06-10T21:49:46.494353+0000","last_fullsized":"2023-06-10T21:49:46.494353+0000","mapping_epoch":443,"log_start":"434'4726","ondisk_log_start":"434'4726","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:02:43.006037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118358016,"num_objects":1806,"num_object_clones":0,"num_object_copies":5418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3241,"num_objects_unfound":0,"num_objects_dirty":1806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11618,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1341,"num_bytes_recovered":87883776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1806}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427601+0000","last_change":"2023-06-10T21:49:18.394539+0000","last_active":"2023-06-10T21:49:30.427601+0000","last_peered":"2023-06-10T21:49:30.427601+0000","last_clean":"2023-06-10T21:49:30.427601+0000","last_became_active":"2023-06-10T21:49:18.394003+0000","last_became_peered":"2023-06-10T21:49:18.394003+0000","last_unstale":"2023-06-10T21:49:30.427601+0000","last_undegraded":"2023-06-10T21:49:30.427601+0000","last_fullsized":"2023-06-10T21:49:30.427601+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":525,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.477211+0000","last_change":"2023-06-10T21:48:53.472531+0000","last_active":"2023-06-10T21:49:41.477211+0000","last_peered":"2023-06-10T21:49:41.477211+0000","last_clean":"2023-06-10T21:49:41.477211+0000","last_became_active":"2023-06-10T21:48:53.453752+0000","last_became_peered":"2023-06-10T21:48:53.453752+0000","last_unstale":"2023-06-10T21:49:41.477211+0000","last_undegraded":"2023-06-10T21:49:41.477211+0000","last_fullsized":"2023-06-10T21:49:41.477211+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:13.163563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"466'11497","reported_seq":11728,"reported_epoch":466,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:44.382162+0000","last_change":"2023-06-10T21:49:19.769652+0000","last_active":"2023-06-10T21:49:44.382162+0000","last_peered":"2023-06-10T21:49:44.382162+0000","last_clean":"2023-06-10T21:47:32.300137+0000","last_became_active":"2023-06-10T21:48:53.630747+0000","last_became_peered":"2023-06-10T21:48:53.630747+0000","last_unstale":"2023-06-10T21:49:44.382162+0000","last_undegraded":"2023-06-10T21:49:44.382162+0000","last_fullsized":"2023-06-10T21:49:44.382162+0000","mapping_epoch":454,"log_start":"437'6192","ondisk_log_start":"437'6192","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5305,"log_dups_size":0,"ondisk_log_size":5305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121700352,"num_objects":1857,"num_object_clones":0,"num_object_copies":5571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1815,"num_objects_unfound":0,"num_objects_dirty":1857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11497,"num_write_kb":427328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4784128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[7,3,5],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1857}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":493,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.473117+0000","last_change":"2023-06-10T21:49:18.401055+0000","last_active":"2023-06-10T21:49:41.473117+0000","last_peered":"2023-06-10T21:49:41.473117+0000","last_clean":"2023-06-10T21:49:41.473117+0000","last_became_active":"2023-06-10T21:49:18.400215+0000","last_became_peered":"2023-06-10T21:49:18.400215+0000","last_unstale":"2023-06-10T21:49:41.473117+0000","last_undegraded":"2023-06-10T21:49:41.473117+0000","last_fullsized":"2023-06-10T21:49:41.473117+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:57.351053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428466+0000","last_change":"2023-06-10T21:49:18.399132+0000","last_active":"2023-06-10T21:49:30.428466+0000","last_peered":"2023-06-10T21:49:30.428466+0000","last_clean":"2023-06-10T21:49:30.428466+0000","last_became_active":"2023-06-10T21:49:18.398793+0000","last_became_peered":"2023-06-10T21:49:18.398793+0000","last_unstale":"2023-06-10T21:49:30.428466+0000","last_undegraded":"2023-06-10T21:49:30.428466+0000","last_fullsized":"2023-06-10T21:49:30.428466+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"467'11810","reported_seq":12183,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.554462+0000","last_change":"2023-06-10T21:48:54.394407+0000","last_active":"2023-06-10T21:49:46.554462+0000","last_peered":"2023-06-10T21:49:46.554462+0000","last_clean":"2023-06-10T21:47:17.264551+0000","last_became_active":"2023-06-10T21:47:19.619202+0000","last_became_peered":"2023-06-10T21:47:19.619202+0000","last_unstale":"2023-06-10T21:49:46.554462+0000","last_undegraded":"2023-06-10T21:49:46.554462+0000","last_fullsized":"2023-06-10T21:49:46.554462+0000","mapping_epoch":439,"log_start":"437'6463","ondisk_log_start":"437'6463","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5347,"log_dups_size":0,"ondisk_log_size":5347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:50:32.972643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1694,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11810,"num_write_kb":434560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":152,"num_bytes_recovered":9961472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":1770}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":340,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430177+0000","last_change":"2023-06-10T21:49:18.411617+0000","last_active":"2023-06-10T21:49:30.430177+0000","last_peered":"2023-06-10T21:49:30.430177+0000","last_clean":"2023-06-10T21:49:30.430177+0000","last_became_active":"2023-06-10T21:49:18.411293+0000","last_became_peered":"2023-06-10T21:49:18.411293+0000","last_unstale":"2023-06-10T21:49:30.430177+0000","last_undegraded":"2023-06-10T21:49:30.430177+0000","last_fullsized":"2023-06-10T21:49:30.430177+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":426,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.486044+0000","last_change":"2023-06-10T21:49:18.400628+0000","last_active":"2023-06-10T21:49:41.486044+0000","last_peered":"2023-06-10T21:49:41.486044+0000","last_clean":"2023-06-10T21:49:41.486044+0000","last_became_active":"2023-06-10T21:49:18.400273+0000","last_became_peered":"2023-06-10T21:49:18.400273+0000","last_unstale":"2023-06-10T21:49:41.486044+0000","last_undegraded":"2023-06-10T21:49:41.486044+0000","last_fullsized":"2023-06-10T21:49:41.486044+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:43.396235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"467'11704","reported_seq":13834,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.414375+0000","last_change":"2023-06-10T21:49:18.408216+0000","last_active":"2023-06-10T21:49:46.414375+0000","last_peered":"2023-06-10T21:49:46.414375+0000","last_clean":"2023-06-10T21:47:17.280051+0000","last_became_active":"2023-06-10T21:47:19.616748+0000","last_became_peered":"2023-06-10T21:47:19.616748+0000","last_unstale":"2023-06-10T21:49:46.414375+0000","last_undegraded":"2023-06-10T21:49:46.414375+0000","last_fullsized":"2023-06-10T21:49:46.414375+0000","mapping_epoch":439,"log_start":"434'4861","ondisk_log_start":"434'4861","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6843,"log_dups_size":0,"ondisk_log_size":6843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:41:17.444409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119668736,"num_objects":1826,"num_object_clones":0,"num_object_copies":5478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1374,"num_objects_unfound":0,"num_objects_dirty":1826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11704,"num_write_kb":432960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1031,"num_bytes_recovered":67567616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1826}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.466705+0000","last_change":"2023-06-10T21:47:18.325919+0000","last_active":"2023-06-10T21:49:30.466705+0000","last_peered":"2023-06-10T21:49:30.466705+0000","last_clean":"2023-06-10T21:49:30.466705+0000","last_became_active":"2023-06-10T21:47:18.325473+0000","last_became_peered":"2023-06-10T21:47:18.325473+0000","last_unstale":"2023-06-10T21:49:30.466705+0000","last_undegraded":"2023-06-10T21:49:30.466705+0000","last_fullsized":"2023-06-10T21:49:30.466705+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:29:40.800760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.479614+0000","last_change":"2023-06-10T21:49:18.414653+0000","last_active":"2023-06-10T21:49:41.479614+0000","last_peered":"2023-06-10T21:49:41.479614+0000","last_clean":"2023-06-10T21:49:41.479614+0000","last_became_active":"2023-06-10T21:49:18.414276+0000","last_became_peered":"2023-06-10T21:49:18.414276+0000","last_unstale":"2023-06-10T21:49:41.479614+0000","last_undegraded":"2023-06-10T21:49:41.479614+0000","last_fullsized":"2023-06-10T21:49:41.479614+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:25:00.827868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.467312+0000","last_change":"2023-06-10T21:47:33.366140+0000","last_active":"2023-06-10T21:49:30.467312+0000","last_peered":"2023-06-10T21:49:30.467312+0000","last_clean":"2023-06-10T21:49:30.467312+0000","last_became_active":"2023-06-10T21:47:33.365711+0000","last_became_peered":"2023-06-10T21:47:33.365711+0000","last_unstale":"2023-06-10T21:49:30.467312+0000","last_undegraded":"2023-06-10T21:49:30.467312+0000","last_fullsized":"2023-06-10T21:49:30.467312+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:02:40.888164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.e","version":"467'11957","reported_seq":12048,"reported_epoch":467,"state":"active+remapped","last_fresh":"2023-06-10T21:49:46.517131+0000","last_change":"2023-06-10T21:49:19.814091+0000","last_active":"2023-06-10T21:49:46.517131+0000","last_peered":"2023-06-10T21:49:46.517131+0000","last_clean":"2023-06-10T21:47:17.288812+0000","last_became_active":"2023-06-10T21:48:53.526973+0000","last_became_peered":"2023-06-10T21:48:53.526973+0000","last_unstale":"2023-06-10T21:49:46.517131+0000","last_undegraded":"2023-06-10T21:49:46.517131+0000","last_fullsized":"2023-06-10T21:49:46.517131+0000","mapping_epoch":454,"log_start":"435'5052","ondisk_log_start":"435'5052","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:49:28.977308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3750,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11957,"num_write_kb":442624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1875}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427688+0000","last_change":"2023-06-10T21:49:18.401074+0000","last_active":"2023-06-10T21:49:30.427688+0000","last_peered":"2023-06-10T21:49:30.427688+0000","last_clean":"2023-06-10T21:49:30.427688+0000","last_became_active":"2023-06-10T21:49:18.400317+0000","last_became_peered":"2023-06-10T21:49:18.400317+0000","last_unstale":"2023-06-10T21:49:30.427688+0000","last_undegraded":"2023-06-10T21:49:30.427688+0000","last_fullsized":"2023-06-10T21:49:30.427688+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.467201+0000","last_change":"2023-06-10T21:49:18.395157+0000","last_active":"2023-06-10T21:49:30.467201+0000","last_peered":"2023-06-10T21:49:30.467201+0000","last_clean":"2023-06-10T21:49:30.467201+0000","last_became_active":"2023-06-10T21:49:18.394783+0000","last_became_peered":"2023-06-10T21:49:18.394783+0000","last_unstale":"2023-06-10T21:49:30.467201+0000","last_undegraded":"2023-06-10T21:49:30.467201+0000","last_fullsized":"2023-06-10T21:49:30.467201+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:19:02.332466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":343,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.475810+0000","last_change":"2023-06-10T21:48:53.384662+0000","last_active":"2023-06-10T21:49:41.475810+0000","last_peered":"2023-06-10T21:49:41.475810+0000","last_clean":"2023-06-10T21:49:41.475810+0000","last_became_active":"2023-06-10T21:48:53.383318+0000","last_became_peered":"2023-06-10T21:48:53.383318+0000","last_unstale":"2023-06-10T21:49:41.475810+0000","last_undegraded":"2023-06-10T21:49:41.475810+0000","last_fullsized":"2023-06-10T21:49:41.475810+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:01:32.045450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"467'11758","reported_seq":12139,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.470207+0000","last_change":"2023-06-10T21:49:18.564923+0000","last_active":"2023-06-10T21:49:46.470207+0000","last_peered":"2023-06-10T21:49:46.470207+0000","last_clean":"2023-06-10T21:47:32.290559+0000","last_became_active":"2023-06-10T21:49:18.533901+0000","last_became_peered":"2023-06-10T21:49:18.533901+0000","last_unstale":"2023-06-10T21:49:46.470207+0000","last_undegraded":"2023-06-10T21:49:46.470207+0000","last_fullsized":"2023-06-10T21:49:46.470207+0000","mapping_epoch":460,"log_start":"434'4925","ondisk_log_start":"434'4925","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6833,"log_dups_size":0,"ondisk_log_size":6833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:10:01.366032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114098200,"num_objects":1742,"num_object_clones":0,"num_object_copies":5226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3420,"num_objects_unfound":0,"num_objects_dirty":1742,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11758,"num_write_kb":431937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1742}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":363,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.473030+0000","last_change":"2023-06-10T21:47:18.337074+0000","last_active":"2023-06-10T21:49:41.473030+0000","last_peered":"2023-06-10T21:49:41.473030+0000","last_clean":"2023-06-10T21:49:41.473030+0000","last_became_active":"2023-06-10T21:47:18.336698+0000","last_became_peered":"2023-06-10T21:47:18.336698+0000","last_unstale":"2023-06-10T21:49:41.473030+0000","last_undegraded":"2023-06-10T21:49:41.473030+0000","last_fullsized":"2023-06-10T21:49:41.473030+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:58.371008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":377,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.545250+0000","last_change":"2023-06-10T21:48:53.380090+0000","last_active":"2023-06-10T21:49:41.545250+0000","last_peered":"2023-06-10T21:49:41.545250+0000","last_clean":"2023-06-10T21:49:41.545250+0000","last_became_active":"2023-06-10T21:48:53.379772+0000","last_became_peered":"2023-06-10T21:48:53.379772+0000","last_unstale":"2023-06-10T21:49:41.545250+0000","last_undegraded":"2023-06-10T21:49:41.545250+0000","last_fullsized":"2023-06-10T21:49:41.545250+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:14.985441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.465406+0000","last_change":"2023-06-10T21:49:18.398464+0000","last_active":"2023-06-10T21:49:30.465406+0000","last_peered":"2023-06-10T21:49:30.465406+0000","last_clean":"2023-06-10T21:49:30.465406+0000","last_became_active":"2023-06-10T21:49:18.397993+0000","last_became_peered":"2023-06-10T21:49:18.397993+0000","last_unstale":"2023-06-10T21:49:30.465406+0000","last_undegraded":"2023-06-10T21:49:30.465406+0000","last_fullsized":"2023-06-10T21:49:30.465406+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:41.415970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.f","version":"466'11613","reported_seq":16774,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:44.262315+0000","last_change":"2023-06-10T21:49:30.458667+0000","last_active":"2023-06-10T21:49:44.262315+0000","last_peered":"2023-06-10T21:49:44.262315+0000","last_clean":"2023-06-10T21:49:44.262315+0000","last_became_active":"2023-06-10T21:49:30.432070+0000","last_became_peered":"2023-06-10T21:49:30.432070+0000","last_unstale":"2023-06-10T21:49:44.262315+0000","last_undegraded":"2023-06-10T21:49:44.262315+0000","last_fullsized":"2023-06-10T21:49:44.262315+0000","mapping_epoch":464,"log_start":"437'6304","ondisk_log_start":"437'6304","created":420,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121438208,"num_objects":1853,"num_object_clones":0,"num_object_copies":5559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11613,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2542,"num_bytes_recovered":166592512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":433,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.473947+0000","last_change":"2023-06-10T21:49:18.400868+0000","last_active":"2023-06-10T21:49:41.473947+0000","last_peered":"2023-06-10T21:49:41.473947+0000","last_clean":"2023-06-10T21:49:41.473947+0000","last_became_active":"2023-06-10T21:49:18.400214+0000","last_became_peered":"2023-06-10T21:49:18.400214+0000","last_unstale":"2023-06-10T21:49:41.473947+0000","last_undegraded":"2023-06-10T21:49:41.473947+0000","last_fullsized":"2023-06-10T21:49:41.473947+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:12:40.417709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":712,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430069+0000","last_change":"2023-06-10T21:48:56.439594+0000","last_active":"2023-06-10T21:49:30.430069+0000","last_peered":"2023-06-10T21:49:30.430069+0000","last_clean":"2023-06-10T21:49:30.430069+0000","last_became_active":"2023-06-10T21:48:56.439122+0000","last_became_peered":"2023-06-10T21:48:56.439122+0000","last_unstale":"2023-06-10T21:49:30.430069+0000","last_undegraded":"2023-06-10T21:49:30.430069+0000","last_fullsized":"2023-06-10T21:49:30.430069+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":156,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478778+0000","last_change":"2023-06-10T21:49:18.398769+0000","last_active":"2023-06-10T21:49:41.478778+0000","last_peered":"2023-06-10T21:49:41.478778+0000","last_clean":"2023-06-10T21:49:41.478778+0000","last_became_active":"2023-06-10T21:49:18.398564+0000","last_became_peered":"2023-06-10T21:49:18.398564+0000","last_unstale":"2023-06-10T21:49:41.478778+0000","last_undegraded":"2023-06-10T21:49:41.478778+0000","last_fullsized":"2023-06-10T21:49:41.478778+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:25:52.639389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428222+0000","last_change":"2023-06-10T21:48:53.379480+0000","last_active":"2023-06-10T21:49:30.428222+0000","last_peered":"2023-06-10T21:49:30.428222+0000","last_clean":"2023-06-10T21:49:30.428222+0000","last_became_active":"2023-06-10T21:48:53.379153+0000","last_became_peered":"2023-06-10T21:48:53.379153+0000","last_unstale":"2023-06-10T21:49:30.428222+0000","last_undegraded":"2023-06-10T21:49:30.428222+0000","last_fullsized":"2023-06-10T21:49:30.428222+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"467'11595","reported_seq":11691,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.537166+0000","last_change":"2023-06-10T21:48:54.603940+0000","last_active":"2023-06-10T21:49:46.537166+0000","last_peered":"2023-06-10T21:49:46.537166+0000","last_clean":"2023-06-10T21:47:17.279622+0000","last_became_active":"2023-06-10T21:48:54.513849+0000","last_became_peered":"2023-06-10T21:48:54.513849+0000","last_unstale":"2023-06-10T21:49:46.537166+0000","last_undegraded":"2023-06-10T21:49:46.537166+0000","last_fullsized":"2023-06-10T21:49:46.537166+0000","mapping_epoch":455,"log_start":"434'4745","ondisk_log_start":"434'4745","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:51:05.743014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119341056,"num_objects":1821,"num_object_clones":0,"num_object_copies":5463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3642,"num_objects_unfound":0,"num_objects_dirty":1821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11595,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1821}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":527,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430139+0000","last_change":"2023-06-10T21:47:18.326869+0000","last_active":"2023-06-10T21:49:30.430139+0000","last_peered":"2023-06-10T21:49:30.430139+0000","last_clean":"2023-06-10T21:49:30.430139+0000","last_became_active":"2023-06-10T21:47:18.326544+0000","last_became_peered":"2023-06-10T21:47:18.326544+0000","last_unstale":"2023-06-10T21:49:30.430139+0000","last_undegraded":"2023-06-10T21:49:30.430139+0000","last_fullsized":"2023-06-10T21:49:30.430139+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":57,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.466462+0000","last_change":"2023-06-10T21:49:18.398644+0000","last_active":"2023-06-10T21:49:30.466462+0000","last_peered":"2023-06-10T21:49:30.466462+0000","last_clean":"2023-06-10T21:49:30.466462+0000","last_became_active":"2023-06-10T21:49:18.398328+0000","last_became_peered":"2023-06-10T21:49:18.398328+0000","last_unstale":"2023-06-10T21:49:30.466462+0000","last_undegraded":"2023-06-10T21:49:30.466462+0000","last_fullsized":"2023-06-10T21:49:30.466462+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:44:21.732943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":39,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.465797+0000","last_change":"2023-06-10T21:47:18.334732+0000","last_active":"2023-06-10T21:49:30.465797+0000","last_peered":"2023-06-10T21:49:30.465797+0000","last_clean":"2023-06-10T21:49:30.465797+0000","last_became_active":"2023-06-10T21:47:18.334458+0000","last_became_peered":"2023-06-10T21:47:18.334458+0000","last_unstale":"2023-06-10T21:49:30.465797+0000","last_undegraded":"2023-06-10T21:49:30.465797+0000","last_fullsized":"2023-06-10T21:49:30.465797+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:31.688908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"467'11736","reported_seq":18099,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.619082+0000","last_change":"2023-06-10T21:49:19.770306+0000","last_active":"2023-06-10T21:49:46.619082+0000","last_peered":"2023-06-10T21:49:46.619082+0000","last_clean":"2023-06-10T21:47:17.293084+0000","last_became_active":"2023-06-10T21:49:19.689638+0000","last_became_peered":"2023-06-10T21:49:19.689638+0000","last_unstale":"2023-06-10T21:49:46.619082+0000","last_undegraded":"2023-06-10T21:49:46.619082+0000","last_fullsized":"2023-06-10T21:49:46.619082+0000","mapping_epoch":461,"log_start":"437'6418","ondisk_log_start":"437'6418","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:47:34.519285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1971,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11736,"num_write_kb":433664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3134,"num_bytes_recovered":205324288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":1816}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":524,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.430887+0000","last_change":"2023-06-10T21:48:53.377036+0000","last_active":"2023-06-10T21:49:30.430887+0000","last_peered":"2023-06-10T21:49:30.430887+0000","last_clean":"2023-06-10T21:49:30.430887+0000","last_became_active":"2023-06-10T21:48:53.375990+0000","last_became_peered":"2023-06-10T21:48:53.375990+0000","last_unstale":"2023-06-10T21:49:30.430887+0000","last_undegraded":"2023-06-10T21:49:30.430887+0000","last_fullsized":"2023-06-10T21:49:30.430887+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":367,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.475883+0000","last_change":"2023-06-10T21:49:18.406396+0000","last_active":"2023-06-10T21:49:41.475883+0000","last_peered":"2023-06-10T21:49:41.475883+0000","last_clean":"2023-06-10T21:49:41.475883+0000","last_became_active":"2023-06-10T21:49:18.406063+0000","last_became_peered":"2023-06-10T21:49:18.406063+0000","last_unstale":"2023-06-10T21:49:41.475883+0000","last_undegraded":"2023-06-10T21:49:41.475883+0000","last_fullsized":"2023-06-10T21:49:41.475883+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:06:02.929666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":48,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.467157+0000","last_change":"2023-06-10T21:49:18.403449+0000","last_active":"2023-06-10T21:49:30.467157+0000","last_peered":"2023-06-10T21:49:30.467157+0000","last_clean":"2023-06-10T21:49:30.467157+0000","last_became_active":"2023-06-10T21:49:18.402496+0000","last_became_peered":"2023-06-10T21:49:18.402496+0000","last_unstale":"2023-06-10T21:49:30.467157+0000","last_undegraded":"2023-06-10T21:49:30.467157+0000","last_fullsized":"2023-06-10T21:49:30.467157+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:50.868280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"466'11597","reported_seq":15939,"reported_epoch":467,"state":"peering","last_fresh":"2023-06-10T21:49:46.394121+0000","last_change":"2023-06-10T21:49:46.394287+0000","last_active":"2023-06-10T21:49:46.394121+0000","last_peered":"2023-06-10T21:49:46.281145+0000","last_clean":"2023-06-10T21:49:46.281145+0000","last_became_active":"2023-06-10T21:49:19.614693+0000","last_became_peered":"2023-06-10T21:49:19.614693+0000","last_unstale":"2023-06-10T21:49:46.394121+0000","last_undegraded":"2023-06-10T21:49:46.394121+0000","last_fullsized":"2023-06-10T21:49:46.394121+0000","mapping_epoch":467,"log_start":"437'6324","ondisk_log_start":"437'6324","created":420,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5273,"log_dups_size":0,"ondisk_log_size":5273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:59:13.473900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119603200,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11597,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2125,"num_bytes_recovered":139132928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":492,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.485966+0000","last_change":"2023-06-10T21:47:18.433743+0000","last_active":"2023-06-10T21:49:41.485966+0000","last_peered":"2023-06-10T21:49:41.485966+0000","last_clean":"2023-06-10T21:49:41.485966+0000","last_became_active":"2023-06-10T21:47:18.338606+0000","last_became_peered":"2023-06-10T21:47:18.338606+0000","last_unstale":"2023-06-10T21:49:41.485966+0000","last_undegraded":"2023-06-10T21:49:41.485966+0000","last_fullsized":"2023-06-10T21:49:41.485966+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:13:02.602599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428389+0000","last_change":"2023-06-10T21:48:53.380616+0000","last_active":"2023-06-10T21:49:30.428389+0000","last_peered":"2023-06-10T21:49:30.428389+0000","last_clean":"2023-06-10T21:49:30.428389+0000","last_became_active":"2023-06-10T21:48:53.380316+0000","last_became_peered":"2023-06-10T21:48:53.380316+0000","last_unstale":"2023-06-10T21:49:30.428389+0000","last_undegraded":"2023-06-10T21:49:30.428389+0000","last_fullsized":"2023-06-10T21:49:30.428389+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":335,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.429967+0000","last_change":"2023-06-10T21:49:18.406601+0000","last_active":"2023-06-10T21:49:30.429967+0000","last_peered":"2023-06-10T21:49:30.429967+0000","last_clean":"2023-06-10T21:49:30.429967+0000","last_became_active":"2023-06-10T21:49:18.406228+0000","last_became_peered":"2023-06-10T21:49:18.406228+0000","last_unstale":"2023-06-10T21:49:30.429967+0000","last_undegraded":"2023-06-10T21:49:30.429967+0000","last_fullsized":"2023-06-10T21:49:30.429967+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"467'11519","reported_seq":11604,"reported_epoch":467,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:49:46.554591+0000","last_change":"2023-06-10T21:48:53.937107+0000","last_active":"2023-06-10T21:49:46.554591+0000","last_peered":"2023-06-10T21:49:46.554591+0000","last_clean":"2023-06-10T21:47:17.285378+0000","last_became_active":"2023-06-10T21:48:53.660628+0000","last_became_peered":"2023-06-10T21:48:53.660628+0000","last_unstale":"2023-06-10T21:49:46.554591+0000","last_undegraded":"2023-06-10T21:49:46.554591+0000","last_fullsized":"2023-06-10T21:49:46.554591+0000","mapping_epoch":454,"log_start":"434'4614","ondisk_log_start":"434'4614","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:26:42.953015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116850688,"num_objects":1783,"num_object_clones":0,"num_object_copies":5349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3566,"num_objects_unfound":0,"num_objects_dirty":1783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11519,"num_write_kb":425664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":1783}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428451+0000","last_change":"2023-06-10T21:48:53.379918+0000","last_active":"2023-06-10T21:49:30.428451+0000","last_peered":"2023-06-10T21:49:30.428451+0000","last_clean":"2023-06-10T21:49:30.428451+0000","last_became_active":"2023-06-10T21:48:53.379608+0000","last_became_peered":"2023-06-10T21:48:53.379608+0000","last_unstale":"2023-06-10T21:49:30.428451+0000","last_undegraded":"2023-06-10T21:49:30.428451+0000","last_fullsized":"2023-06-10T21:49:30.428451+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":70,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.465645+0000","last_change":"2023-06-10T21:48:53.400742+0000","last_active":"2023-06-10T21:49:30.465645+0000","last_peered":"2023-06-10T21:49:30.465645+0000","last_clean":"2023-06-10T21:49:30.465645+0000","last_became_active":"2023-06-10T21:48:53.381285+0000","last_became_peered":"2023-06-10T21:48:53.381285+0000","last_unstale":"2023-06-10T21:49:30.465645+0000","last_undegraded":"2023-06-10T21:49:30.465645+0000","last_fullsized":"2023-06-10T21:49:30.465645+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:45.459010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":30,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427999+0000","last_change":"2023-06-10T21:49:18.399985+0000","last_active":"2023-06-10T21:49:30.427999+0000","last_peered":"2023-06-10T21:49:30.427999+0000","last_clean":"2023-06-10T21:49:30.427999+0000","last_became_active":"2023-06-10T21:49:18.399526+0000","last_became_peered":"2023-06-10T21:49:18.399526+0000","last_unstale":"2023-06-10T21:49:30.427999+0000","last_undegraded":"2023-06-10T21:49:30.427999+0000","last_fullsized":"2023-06-10T21:49:30.427999+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":38,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.467114+0000","last_change":"2023-06-10T21:47:18.324336+0000","last_active":"2023-06-10T21:49:30.467114+0000","last_peered":"2023-06-10T21:49:30.467114+0000","last_clean":"2023-06-10T21:49:30.467114+0000","last_became_active":"2023-06-10T21:47:18.321834+0000","last_became_peered":"2023-06-10T21:47:18.321834+0000","last_unstale":"2023-06-10T21:49:30.467114+0000","last_undegraded":"2023-06-10T21:49:30.467114+0000","last_fullsized":"2023-06-10T21:49:30.467114+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:40.514667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":380,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.480953+0000","last_change":"2023-06-10T21:49:18.398119+0000","last_active":"2023-06-10T21:49:41.480953+0000","last_peered":"2023-06-10T21:49:41.480953+0000","last_clean":"2023-06-10T21:49:41.480953+0000","last_became_active":"2023-06-10T21:49:18.397781+0000","last_became_peered":"2023-06-10T21:49:18.397781+0000","last_unstale":"2023-06-10T21:49:41.480953+0000","last_undegraded":"2023-06-10T21:49:41.480953+0000","last_fullsized":"2023-06-10T21:49:41.480953+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:27:32.540626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":38,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427914+0000","last_change":"2023-06-10T21:49:18.431026+0000","last_active":"2023-06-10T21:49:30.427914+0000","last_peered":"2023-06-10T21:49:30.427914+0000","last_clean":"2023-06-10T21:49:30.427914+0000","last_became_active":"2023-06-10T21:49:18.403054+0000","last_became_peered":"2023-06-10T21:49:18.403054+0000","last_unstale":"2023-06-10T21:49:30.427914+0000","last_undegraded":"2023-06-10T21:49:30.427914+0000","last_fullsized":"2023-06-10T21:49:30.427914+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":373,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478743+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:49:41.478743+0000","last_peered":"2023-06-10T21:49:41.478743+0000","last_clean":"2023-06-10T21:49:41.478743+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:49:41.478743+0000","last_undegraded":"2023-06-10T21:49:41.478743+0000","last_fullsized":"2023-06-10T21:49:41.478743+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":38,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.428303+0000","last_change":"2023-06-10T21:49:18.410072+0000","last_active":"2023-06-10T21:49:30.428303+0000","last_peered":"2023-06-10T21:49:30.428303+0000","last_clean":"2023-06-10T21:49:30.428303+0000","last_became_active":"2023-06-10T21:49:18.398826+0000","last_became_peered":"2023-06-10T21:49:18.398826+0000","last_unstale":"2023-06-10T21:49:30.428303+0000","last_undegraded":"2023-06-10T21:49:30.428303+0000","last_fullsized":"2023-06-10T21:49:30.428303+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":453,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478917+0000","last_change":"2023-06-10T21:47:18.335340+0000","last_active":"2023-06-10T21:49:41.478917+0000","last_peered":"2023-06-10T21:49:41.478917+0000","last_clean":"2023-06-10T21:49:41.478917+0000","last_became_active":"2023-06-10T21:47:18.335048+0000","last_became_peered":"2023-06-10T21:47:18.335048+0000","last_unstale":"2023-06-10T21:49:41.478917+0000","last_undegraded":"2023-06-10T21:49:41.478917+0000","last_fullsized":"2023-06-10T21:49:41.478917+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:11:27.198035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":132,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.482377+0000","last_change":"2023-06-10T21:48:53.450892+0000","last_active":"2023-06-10T21:49:41.482377+0000","last_peered":"2023-06-10T21:49:41.482377+0000","last_clean":"2023-06-10T21:49:41.482377+0000","last_became_active":"2023-06-10T21:48:53.450548+0000","last_became_peered":"2023-06-10T21:48:53.450548+0000","last_unstale":"2023-06-10T21:49:41.482377+0000","last_undegraded":"2023-06-10T21:49:41.482377+0000","last_fullsized":"2023-06-10T21:49:41.482377+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:36:57.682749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":466,"state":"active+clean","last_fresh":"2023-06-10T21:49:41.478772+0000","last_change":"2023-06-10T21:47:18.344279+0000","last_active":"2023-06-10T21:49:41.478772+0000","last_peered":"2023-06-10T21:49:41.478772+0000","last_clean":"2023-06-10T21:49:41.478772+0000","last_became_active":"2023-06-10T21:47:18.343646+0000","last_became_peered":"2023-06-10T21:47:18.343646+0000","last_unstale":"2023-06-10T21:49:41.478772+0000","last_undegraded":"2023-06-10T21:49:41.478772+0000","last_fullsized":"2023-06-10T21:49:41.478772+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:43.284066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":90,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.465319+0000","last_change":"2023-06-10T21:49:18.424381+0000","last_active":"2023-06-10T21:49:30.465319+0000","last_peered":"2023-06-10T21:49:30.465319+0000","last_clean":"2023-06-10T21:49:30.465319+0000","last_became_active":"2023-06-10T21:49:18.403001+0000","last_became_peered":"2023-06-10T21:49:18.403001+0000","last_unstale":"2023-06-10T21:49:30.465319+0000","last_undegraded":"2023-06-10T21:49:30.465319+0000","last_fullsized":"2023-06-10T21:49:30.465319+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:47.222958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":465,"state":"active+clean","last_fresh":"2023-06-10T21:49:30.427471+0000","last_change":"2023-06-10T21:48:53.382733+0000","last_active":"2023-06-10T21:49:30.427471+0000","last_peered":"2023-06-10T21:49:30.427471+0000","last_clean":"2023-06-10T21:49:30.427471+0000","last_became_active":"2023-06-10T21:48:53.382249+0000","last_became_peered":"2023-06-10T21:48:53.382249+0000","last_unstale":"2023-06-10T21:49:30.427471+0000","last_undegraded":"2023-06-10T21:49:30.427471+0000","last_fullsized":"2023-06-10T21:49:30.427471+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1901068312,"num_objects":29009,"num_object_clones":0,"num_object_copies":87027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":33800,"num_objects_unfound":0,"num_objects_dirty":29009,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186699,"num_write_kb":6902593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21254,"num_bytes_recovered":1392050176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6588280832,"data_stored":6588268616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97141,"ondisk_log_size":97141,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":28260,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53512,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430832,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1772456,"kb_used_data":1324896,"kb_used_omap":10,"kb_used_meta":447541,"kb_avail":91972696,"statfs":{"total":95995035648,"available":94180040704,"internally_reserved":0,"allocated":1356693504,"data_stored":1355933428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":458282045},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:49:06 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":2.617,"15min":2.1000000000000001},"min":{"1min":0.252,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":6.3209999999999997,"5min":170.05199999999999,"15min":170.05199999999999},"last":2.6309999999999998},{"interface":"front","average":{"1min":3.3109999999999999,"5min":3.746,"15min":2.7389999999999999},"min":{"1min":0.26900000000000002,"5min":0.246,"15min":0.246},"max":{"1min":12.534000000000001,"5min":170.126,"15min":170.126},"last":0.318}]},{"osd":1,"last update":"Sat Jun 10 21:49:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7999999999999998,"5min":1.411,"15min":2.0219999999999998},"min":{"1min":0.311,"5min":0.27800000000000002,"15min":0.25600000000000001},"max":{"1min":13.180999999999999,"5min":13.180999999999999,"15min":60.863},"last":2.669},{"interface":"front","average":{"1min":2.109,"5min":1.6220000000000001,"15min":2.3399999999999999},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":7.3280000000000003,"5min":8.8680000000000003,"15min":61.148000000000003},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:48:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4700000000000002,"5min":9.9269999999999996,"15min":6.3140000000000001},"min":{"1min":0.34000000000000002,"5min":0.309,"15min":0.27600000000000002},"max":{"1min":30.734000000000002,"5min":218.90600000000001,"15min":218.90600000000001},"last":4.577},{"interface":"front","average":{"1min":4.2430000000000003,"5min":10.151999999999999,"15min":6.3949999999999996},"min":{"1min":0.46200000000000002,"5min":0.38,"15min":0.33900000000000002},"max":{"1min":30.452999999999999,"5min":218.851,"15min":218.851},"last":1.143}]},{"osd":3,"last update":"Sat Jun 10 21:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":5.9729999999999999,"15min":5.4299999999999997},"min":{"1min":0.34899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.2350000000000003,"5min":218.35300000000001,"15min":218.35300000000001},"last":2.6920000000000002},{"interface":"front","average":{"1min":3.4089999999999998,"5min":6.6500000000000004,"15min":5.5060000000000002},"min":{"1min":0.34399999999999997,"5min":0.33300000000000002,"15min":0.308},"max":{"1min":11.409000000000001,"5min":218.74600000000001,"15min":218.74600000000001},"last":5.6580000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.1479999999999999,"15min":0.82699999999999996},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":3.4609999999999999,"5min":4.8479999999999999,"15min":13.429},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.268,"5min":1.355,"15min":0.98799999999999999},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.532,"5min":12.279999999999999,"15min":22.77},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:49:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.3100000000000001,"15min":8.5549999999999997},"min":{"1min":0.315,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.3279999999999998,"5min":7.984,"15min":131.619},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.5549999999999999,"15min":8.5169999999999995},"min":{"1min":0.33000000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":8.4659999999999993,"5min":10.603999999999999,"15min":134.928},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":0.84399999999999997},"min":{"1min":0.433,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":6.3029999999999999,"5min":7.9660000000000002,"15min":10.890000000000001},"last":2.8780000000000001},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.363,"15min":0.95799999999999996},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.30599999999999999},"max":{"1min":7.3949999999999996,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.502}]}]},{"osd":6,"up_from":42,"seq":180388627060,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818004,"kb_used_data":353612,"kb_used_omap":0,"kb_used_meta":464384,"kb_avail":92927148,"statfs":{"total":95995035648,"available":95157399552,"internally_reserved":0,"allocated":362098688,"data_stored":361357394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":475529216},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":2.0609999999999999,"15min":1.9239999999999999},"min":{"1min":0.28799999999999998,"5min":0.253,"15min":0.23599999999999999},"max":{"1min":7.6840000000000002,"5min":107.86,"15min":107.86},"last":4.0640000000000001},{"interface":"front","average":{"1min":9.7279999999999998,"5min":3.7480000000000002,"15min":2.718},"min":{"1min":0.27600000000000002,"5min":0.26700000000000002,"15min":0.26100000000000001},"max":{"1min":136.85900000000001,"5min":136.85900000000001,"15min":136.85900000000001},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Jun 10 21:48:54 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.101,"15min":1.9470000000000001},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.245},"max":{"1min":4.6260000000000003,"5min":7.3819999999999997,"15min":80.210999999999999},"last":4.1050000000000004},{"interface":"front","average":{"1min":1.4359999999999999,"5min":1.298,"15min":2.089},"min":{"1min":0.33700000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.8609999999999998,"5min":9.1920000000000002,"15min":79.894000000000005},"last":0.56299999999999994}]},{"osd":2,"last update":"Sat Jun 10 21:49:07 2023","interfaces":[{"interface":"back","average":{"1min":14.597,"5min":11.558999999999999,"15min":6.1769999999999996},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":142.255,"5min":225.119,"15min":225.119},"last":4.0250000000000004},{"interface":"front","average":{"1min":14.875999999999999,"5min":11.766,"15min":6.2599999999999998},"min":{"1min":0.438,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":142.12899999999999,"5min":225.137,"15min":225.137},"last":4.133}]},{"osd":3,"last update":"Sat Jun 10 21:49:32 2023","interfaces":[{"interface":"back","average":{"1min":27.033000000000001,"5min":12.029999999999999,"15min":7.3220000000000001},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":412.55200000000002,"5min":412.55200000000002,"15min":412.55200000000002},"last":4.2729999999999997},{"interface":"front","average":{"1min":27.202000000000002,"5min":11.993,"15min":7.1150000000000002},"min":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":412.41199999999998,"5min":412.41199999999998,"15min":412.41199999999998},"last":1.6060000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:49:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.0089999999999999,"15min":0.748},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.27500000000000002},"max":{"1min":7.5510000000000002,"5min":7.5510000000000002,"15min":7.5510000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.86399999999999999,"15min":0.75900000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.7040000000000002,"5min":8.4459999999999997,"15min":11.068},"last":2.7200000000000002}]},{"osd":5,"last update":"Sat Jun 10 21:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0369999999999999,"15min":1.173},"min":{"1min":0.34499999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0250000000000004,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.012,"15min":1.627},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.4630000000000001,"5min":6.5229999999999997,"15min":7.194},"last":7.2009999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:49:32 2023","interfaces":[{"interface":"back","average":{"1min":6.8140000000000001,"5min":3.9239999999999999,"15min":2.9700000000000002},"min":{"1min":0.34399999999999997,"5min":0.33600000000000002,"15min":0.316},"max":{"1min":97.108000000000004,"5min":189.042,"15min":189.042},"last":0.52700000000000002},{"interface":"front","average":{"1min":6.7699999999999996,"5min":4.2199999999999998,"15min":3.2120000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32000000000000001},"max":{"1min":97.299000000000007,"5min":190.83699999999999,"15min":190.83699999999999},"last":7.1159999999999997}]}]},{"osd":1,"up_from":0,"seq":450971566503,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":244,"seq":1047972020479,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1021816,"kb_used_data":385712,"kb_used_omap":10,"kb_used_meta":636085,"kb_avail":92723336,"statfs":{"total":95995035648,"available":94948696064,"internally_reserved":0,"allocated":394969088,"data_stored":393434032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":651351092},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":1.256,"15min":1.948},"min":{"1min":0.215,"5min":0.215,"15min":0.189},"max":{"1min":6.6429999999999998,"5min":6.6429999999999998,"15min":27.759},"last":0.25600000000000001},{"interface":"front","average":{"1min":2.9239999999999999,"5min":1.6020000000000001,"15min":2.7269999999999999},"min":{"1min":0.27500000000000002,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":8.7040000000000006,"5min":9.4800000000000004,"15min":31.419},"last":0.23699999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":14.917999999999999,"5min":7.157,"15min":7.173},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.253},"max":{"1min":135.24100000000001,"5min":135.24100000000001,"15min":135.24100000000001},"last":2.7040000000000002},{"interface":"front","average":{"1min":14.054,"5min":6.7030000000000003,"15min":6.5999999999999996},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":135.27099999999999,"5min":135.27099999999999,"15min":135.27099999999999},"last":0.35099999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:49:40 2023","interfaces":[{"interface":"back","average":{"1min":17.539999999999999,"5min":6.7839999999999998,"15min":6.1100000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":150.89699999999999,"5min":150.89699999999999,"15min":211.864},"last":2.742},{"interface":"front","average":{"1min":18.173999999999999,"5min":6.5190000000000001,"15min":5.7400000000000002},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":150.97200000000001,"5min":150.97200000000001,"15min":207.517},"last":0.28899999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5580000000000001,"15min":2.4260000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":8.6709999999999994,"5min":8.6709999999999994,"15min":87.054000000000002},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.538,"5min":1.3240000000000001,"15min":2.194},"min":{"1min":0.36799999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.9720000000000004,"5min":6.6139999999999999,"15min":93.649000000000001},"last":2.798}]},{"osd":5,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":2.258,"15min":4.5300000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":6.7290000000000001,"5min":70.903999999999996,"15min":92.149000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.425,"5min":2.464,"15min":5.181},"min":{"1min":0.375,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":5.0659999999999998,"5min":71.698999999999998,"15min":93.292000000000002},"last":2.8260000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0590000000000002,"5min":1.7130000000000001,"15min":2.5099999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.34300000000000003},"max":{"1min":8.6289999999999996,"5min":8.6289999999999996,"15min":87.019999999999996},"last":0.498},{"interface":"front","average":{"1min":1.629,"5min":1.607,"15min":2.331},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.0300000000000002,"5min":10.382,"15min":93.614999999999995},"last":2.7709999999999999}]},{"osd":7,"last update":"Sat Jun 10 21:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":3.746,"15min":3.653},"min":{"1min":0.42799999999999999,"5min":0.39500000000000002,"15min":0.34100000000000003},"max":{"1min":6.7000000000000002,"5min":191.566,"15min":191.566},"last":10.053000000000001},{"interface":"front","average":{"1min":3.2989999999999999,"5min":4.3090000000000002,"15min":3.9420000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.36299999999999999},"max":{"1min":11.757,"5min":191.58000000000001,"15min":191.58000000000001},"last":1.5129999999999999}]}]},{"osd":2,"up_from":120,"seq":515396075891,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407488,"kb_used_data":1178360,"kb_used_omap":7,"kb_used_meta":1229048,"kb_avail":91337664,"statfs":{"total":95995035648,"available":93529767936,"internally_reserved":0,"allocated":1206640640,"data_stored":1205162983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1258545233},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,1,0,0,1,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:49:07 2023","interfaces":[{"interface":"back","average":{"1min":3.5099999999999998,"5min":2.8839999999999999,"15min":4.0819999999999999},"min":{"1min":0.215,"5min":0.20899999999999999,"15min":0.19},"max":{"1min":15.614000000000001,"5min":103.19,"15min":171.33699999999999},"last":0.63},{"interface":"front","average":{"1min":2.516,"5min":2.3879999999999999,"15min":3.6779999999999999},"min":{"1min":0.30399999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.596,"5min":103.125,"15min":171.56200000000001},"last":0.61599999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:48:59 2023","interfaces":[{"interface":"back","average":{"1min":3.4830000000000001,"5min":1.9910000000000001,"15min":4.3369999999999997},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.217},"max":{"1min":15.573,"5min":15.573,"15min":171.286},"last":7.6580000000000004},{"interface":"front","average":{"1min":3.1030000000000002,"5min":1.901,"15min":3.1859999999999999},"min":{"1min":0.28399999999999997,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":15.685,"5min":15.685,"15min":85.501999999999995},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Jun 10 21:49:33 2023","interfaces":[{"interface":"back","average":{"1min":5.0730000000000004,"5min":4.9800000000000004,"15min":6.8029999999999999},"min":{"1min":0.32000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":46.786000000000001,"5min":79.353999999999999,"15min":171.78299999999999},"last":2.4350000000000001},{"interface":"front","average":{"1min":4.3310000000000004,"5min":4.4180000000000001,"15min":6.4960000000000004},"min":{"1min":0.26200000000000001,"5min":0.26000000000000001,"15min":0.24399999999999999},"max":{"1min":43.375,"5min":79.614000000000004,"15min":171.65100000000001},"last":2.3650000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:48:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.464,"15min":2.206},"min":{"1min":0.42899999999999999,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":15.712999999999999,"5min":15.712999999999999,"15min":171.48699999999999},"last":0.496},{"interface":"front","average":{"1min":2.8639999999999999,"5min":1.9610000000000001,"15min":2.9980000000000002},"min":{"1min":0.44500000000000001,"5min":0.36899999999999999,"15min":0.29999999999999999},"max":{"1min":15.641999999999999,"5min":15.641999999999999,"15min":171.18100000000001},"last":7.7839999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:49:07 2023","interfaces":[{"interface":"back","average":{"1min":16.672000000000001,"5min":7.8159999999999998,"15min":7.2599999999999998},"min":{"1min":0.40200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":130.864,"5min":157.756,"15min":157.756},"last":2.3999999999999999},{"interface":"front","average":{"1min":15.099,"5min":7.2800000000000002,"15min":6.6539999999999999},"min":{"1min":0.47499999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":131.06299999999999,"5min":157.59899999999999,"15min":157.59899999999999},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:48:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":1.7430000000000001,"15min":2.3660000000000001},"min":{"1min":0.47999999999999998,"5min":0.41999999999999998,"15min":0.307},"max":{"1min":15.930999999999999,"5min":15.930999999999999,"15min":171.441},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.3100000000000001,"5min":2.1560000000000001,"15min":3.1280000000000001},"min":{"1min":0.41499999999999998,"5min":0.39100000000000001,"15min":0.33000000000000002},"max":{"1min":15.91,"5min":15.91,"15min":171.24600000000001},"last":7.702}]},{"osd":7,"last update":"Sat Jun 10 21:48:59 2023","interfaces":[{"interface":"back","average":{"1min":10.255000000000001,"5min":5.3499999999999996,"15min":4.8769999999999998},"min":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.34200000000000003},"max":{"1min":135.86099999999999,"5min":135.86099999999999,"15min":219.131},"last":0.59999999999999998},{"interface":"front","average":{"1min":8.9789999999999992,"5min":4.9710000000000001,"15min":4.5960000000000001},"min":{"1min":0.41199999999999998,"5min":0.39700000000000002,"15min":0.32200000000000001},"max":{"1min":133.45400000000001,"5min":133.45400000000001,"15min":219.19},"last":0.54300000000000004}]}]},{"osd":3,"up_from":250,"seq":1073741824251,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2426720,"kb_used_data":1456792,"kb_used_omap":7,"kb_used_meta":969912,"kb_avail":91318432,"statfs":{"total":95995035648,"available":93510074368,"internally_reserved":0,"allocated":1491755008,"data_stored":1490126606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":993189907},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":13.359,"5min":4.1710000000000003,"15min":3.242},"min":{"1min":0.23799999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":236.97300000000001,"5min":236.97300000000001,"15min":236.97300000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":11.646000000000001,"5min":4.29,"15min":3.496},"min":{"1min":0.223,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":237.375,"5min":237.375,"15min":237.375},"last":5.3339999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.0819999999999999,"15min":2.6890000000000001},"min":{"1min":0.253,"5min":0.247,"15min":0.20499999999999999},"max":{"1min":14.896000000000001,"5min":18.116,"15min":94.766000000000005},"last":1.421},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1589999999999998,"15min":2.9119999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":16.562999999999999,"5min":50.264000000000003,"15min":94.691000000000003},"last":1.3959999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":8.6050000000000004,"5min":6.258,"15min":4.8479999999999999},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.23300000000000001},"max":{"1min":117.279,"5min":126.913,"15min":231.07900000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":10.301,"5min":6.5789999999999997,"15min":5.5709999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.22800000000000001},"max":{"1min":117.398,"5min":128.52199999999999,"15min":231.58600000000001},"last":0.29199999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.4209999999999998,"15min":1.752},"min":{"1min":0.317,"5min":0.317,"15min":0.28499999999999998},"max":{"1min":4.4809999999999999,"5min":50.295999999999999,"15min":50.295999999999999},"last":9.1850000000000005},{"interface":"front","average":{"1min":2.7509999999999999,"5min":2.548,"15min":2.1640000000000001},"min":{"1min":0.33700000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":14.938000000000001,"5min":50.381,"15min":50.381},"last":1.458}]},{"osd":5,"last update":"Sat Jun 10 21:49:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8199999999999998,"5min":1.732,"15min":1.304},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":50.399000000000001,"5min":50.399000000000001,"15min":50.399000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":4.4249999999999998,"5min":2.7589999999999999,"15min":2.722},"min":{"1min":0.42999999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":50.335000000000001,"5min":50.335000000000001,"15min":50.335000000000001},"last":5.4100000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.1000000000000001,"15min":2.4039999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32300000000000001},"max":{"1min":62.098999999999997,"5min":62.098999999999997,"15min":62.098999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":5.6070000000000002,"5min":2.8359999999999999,"15min":2.0190000000000001},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":62.139000000000003,"5min":62.139000000000003,"15min":62.139000000000003},"last":0.95999999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":5.7149999999999999,"5min":4.093,"15min":2.7850000000000001},"min":{"1min":0.48899999999999999,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":69.683000000000007,"5min":69.683000000000007,"15min":69.683000000000007},"last":0.495},{"interface":"front","average":{"1min":5.9320000000000004,"5min":3.6619999999999999,"15min":2.3650000000000002},"min":{"1min":0.435,"5min":0.374,"15min":0.316},"max":{"1min":69.998999999999995,"5min":69.998999999999995,"15min":69.998999999999995},"last":1.347}]}]},{"osd":4,"up_from":0,"seq":128849019516,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":0,"seq":1675037245522,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":384827392,"data_stored":384827392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1183453184,"data_stored":1183449112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1480130560,"data_stored":1480130560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1831145472,"data_stored":1831141400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":362676224,"data_stored":362676224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1346048000,"data_stored":1346043928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:49:52.390 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-10T21:49:52.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_9 pg_num 26 2023-06-10T21:49:52.479 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:52.532 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:52 smithi064 ceph-mon[180297]: pgmap v1723: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 29934/79716 objects misplaced (37.551%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:49:52.533 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:52 smithi064 ceph-mon[186706]: pgmap v1723: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 29934/79716 objects misplaced (37.551%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:49:52.545 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:52 smithi152 ceph-mon[156819]: pgmap v1723: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 638 op/s; 29934/79716 objects misplaced (37.551%); 8.9 MiB/s, 141 objects/s recovering 2023-06-10T21:49:52.763 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:52.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:52.869 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:49:52.878 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:52.941 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:53.114 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:49:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:49:52] "GET /metrics HTTP/1.1" 200 224270 "" "Prometheus/2.33.4" 2023-06-10T21:49:53.641 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:49:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:53.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:49:53.747 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:49:53.786 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:53.887 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:49:53.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:53.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:49:53.997 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:49:54.001 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:49:54.023 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:54.064 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:49:54.067 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1427613115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.068 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:54.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:49:54.169 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:54.203 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:54.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1427613115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1427613115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.409 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:49:54.566 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:54.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:54.675 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:54.748 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:54.917 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:54.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:55.027 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:55.062 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:55.141 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:55.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:55.246 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:49:55.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:55.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.304 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:55.305 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: osdmap e469: 8 total, 8 up, 5 in 2023-06-10T21:49:55.305 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:54 smithi152 ceph-mon[156819]: pgmap v1725: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 29934/79185 objects misplaced (37.803%); 8.9 MiB/s, 142 objects/s recovering 2023-06-10T21:49:55.314 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.410 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: osdmap e469: 8 total, 8 up, 5 in 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[186706]: pgmap v1725: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 29934/79185 objects misplaced (37.803%); 8.9 MiB/s, 142 objects/s recovering 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:49:55.411 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: failed to encode map e469 with expected crc 2023-06-10T21:49:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:49:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: osdmap e469: 8 total, 8 up, 5 in 2023-06-10T21:49:55.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:54 smithi064 ceph-mon[180297]: pgmap v1725: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 621 op/s; 29934/79185 objects misplaced (37.803%); 8.9 MiB/s, 142 objects/s recovering 2023-06-10T21:49:55.413 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:49:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:55.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:49:55.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:55.561 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:56.418 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:56.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:56.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:49:56.560 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:57.335 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:49:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:57.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:49:57.438 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:57 smithi064 ceph-mon[186706]: pgmap v1726: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 29783/78225 objects misplaced (38.074%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:49:57.438 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:57 smithi064 ceph-mon[180297]: pgmap v1726: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 29783/78225 objects misplaced (38.074%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:49:57.443 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:57.536 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:57.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:57 smithi152 ceph-mon[156819]: pgmap v1726: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 330 op/s; 29783/78225 objects misplaced (38.074%); 3.8 MiB/s, 60 objects/s recovering 2023-06-10T21:49:57.698 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:49:57.773 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:57.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:57.879 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:57.888 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:49:57.888 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:49:57.888 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:49:57.888 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156299 12.2 12:4770460a:::benchmark_data_smithi064_442287_object68249:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e359)", 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 567.66617895499996, 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.193165665, 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.889 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156299 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.890 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.891 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214945+0000", 2023-06-10T21:49:57.892 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214947+0000", 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.3539999999999998e-06 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.893 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214953+0000", 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6590000000000004e-06 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.214959+0000", 2023-06-10T21:49:57.894 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1920000000000002e-06 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215018+0000", 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9209000000000002e-05 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.895 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op must wait for map", 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.215029+0000", 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0444e-05 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222815+0000", 2023-06-10T21:49:57.896 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077857710000000004 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.222824+0000", 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2979999999999997e-06 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.897 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.391187+0000", 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1683627429999999 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.392188+0000", 2023-06-10T21:49:57.898 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010011899999999999 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408024+0000", 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015836071 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.899 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408111+0000", 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7112000000000002e-05 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.900 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156315 12.12 12:486ff777:::benchmark_data_smithi064_442287_object68265:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 567.47656422800003, 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0549169319999998, 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.901 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156315 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:49:57.902 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404560+0000", 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.903 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404558+0000", 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404567+0000", 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3510000000000002e-06 2023-06-10T21:49:57.904 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404580+0000", 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3585e-05 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.905 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404597+0000", 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6884999999999998e-05 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404651+0000", 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4268000000000003e-05 2023-06-10T21:49:57.906 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.404663+0000", 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1511999999999999e-05 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.907 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.408138+0000", 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0034749550000002 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430486+0000", 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.022347895 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.908 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440618+0000", 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010132474000000001 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.440713+0000", 2023-06-10T21:49:57.909 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4409999999999999e-05 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.441046+0000", 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033325899999999997 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.910 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.445516+0000", 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044697950000000004 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457707+0000", 2023-06-10T21:49:57.911 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012191306000000001 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457796+0000", 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8623999999999997e-05 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.912 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.458924+0000", 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0011283 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.458960+0000", 2023-06-10T21:49:57.913 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5641e-05 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459477+0000", 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00051747900000000005 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.914 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156317 12.12 12:4887ecdb:::benchmark_data_smithi064_442287_object68267:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 567.471351949, 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0511120819999999, 2023-06-10T21:49:57.915 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156317 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.916 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409772+0000", 2023-06-10T21:49:57.917 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409771+0000", 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.918 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409779+0000", 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4189999999999996e-06 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409792+0000", 2023-06-10T21:49:57.919 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3642000000000001e-05 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409801+0000", 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1789999999999997e-06 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.920 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409855+0000", 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3746999999999999e-05 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.409865+0000", 2023-06-10T21:49:57.921 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0232e-05 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430525+0000", 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0206594729999998 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.922 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.432926+0000", 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002401481 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.448701+0000", 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.015774939000000002 2023-06-10T21:49:57.923 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.448802+0000", 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100634 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:49:57.924 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.449172+0000", 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00037020999999999999 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451840+0000", 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002668241 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.925 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459503+0000", 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076622160000000003 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.926 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459525+0000", 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2195e-05 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460331+0000", 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080666600000000005 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.927 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460406+0000", 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4512999999999996e-05 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.460885+0000", 2023-06-10T21:49:57.928 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047858500000000001 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46562.0:156321 12.12 12:4965cc64:::benchmark_data_smithi064_442287_object68271:head [delete] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:49:57.929 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 567.40754133099995, 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.989335101, 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46562", 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/2704889249", 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 156321 2023-06-10T21:49:57.930 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.931 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473583+0000", 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473581+0000", 2023-06-10T21:49:57.932 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473589+0000", 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4819999999999997e-06 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.933 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473603+0000", 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4447000000000001e-05 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473614+0000", 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1249e-05 2023-06-10T21:49:57.934 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473664+0000", 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9842999999999997e-05 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.935 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:30.473675+0000", 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0786e-05 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.432987+0000", 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.959312341 2023-06-10T21:49:57.936 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for degraded object", 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.433685+0000", 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00069741300000000003 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.937 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451862+0000", 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018176859 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.451922+0000", 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0197e-05 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.938 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 1,5,7", 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.452223+0000", 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030085300000000001 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:57.939 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.457715+0000", 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054923799999999998 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459543+0000", 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0018279150000000001 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.940 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.459560+0000", 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7337999999999999e-05 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462361+0000", 2023-06-10T21:49:57.941 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002800641 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462413+0000", 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1984000000000002e-05 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.942 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.462918+0000", 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050521099999999998 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.943 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156335 12.a e360/358 12:540f4c4f:::benchmark_data_smithi064_442287_object68285:head v 360'9865, mlcod=0'0)", 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.66007815499995, 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1858911519999999, 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.944 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.945 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221046+0000", 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221045+0000", 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.946 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221053+0000", 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7559999999999999e-06 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221064+0000", 2023-06-10T21:49:57.947 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0668e-05 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.221072+0000", 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3250000000000008e-06 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.948 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.401905+0000", 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.180832348 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.401927+0000", 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2634e-05 2023-06-10T21:49:57.949 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406910+0000", 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049828210000000001 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.950 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406937+0000", 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7396e-05 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.951 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156351 12.a e360/358 12:57b5438e:::benchmark_data_smithi064_442287_object68301:head v 360'9866, mlcod=0'0)", 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.59878822200005, 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.124617532, 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.952 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282336+0000", 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.953 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282335+0000", 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.954 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.282343+0000", 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6960000000000005e-06 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287460+0000", 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051169090000000002 2023-06-10T21:49:57.955 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287467+0000", 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.272e-06 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.956 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.403319+0000", 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.115852015 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.403339+0000", 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9918000000000001e-05 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.957 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406942+0000", 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003602821 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.406954+0000", 2023-06-10T21:49:57.958 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1768e-05 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156352 12.a e360/358 12:57f76687:::benchmark_data_smithi064_442287_object68302:head v 360'9867, mlcod=0'0)", 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:49:57.959 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.59363491099998, 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.143004788, 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.960 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.961 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287489+0000", 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287496+0000", 2023-06-10T21:49:57.962 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7859999999999999e-06 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287504+0000", 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9799999999999998e-06 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.963 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.287509+0000", 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4240000000000001e-06 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.404422+0000", 2023-06-10T21:49:57.964 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1169133019999999 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.404444+0000", 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1957e-05 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.965 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430421+0000", 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.025976448999999999 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430494+0000", 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3517999999999996e-05 2023-06-10T21:49:57.966 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_repop(client.46562.0:156358 12.a e360/358 12:5a95b6f1:::benchmark_data_smithi064_442287_object68308:head v 360'9868, mlcod=0'0)", 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:49:57.967 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 566.577672689, 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.127077511, 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:49:57.968 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303452+0000", 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.969 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303451+0000", 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303458+0000", 2023-06-10T21:49:57.970 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1550000000000001e-06 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303471+0000", 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3037000000000001e-05 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.971 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:31.303479+0000", 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8900000000000007e-06 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.405165+0000", 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.101685818 2023-06-10T21:49:57.972 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.405186+0000", 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0908999999999999e-05 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.973 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430498+0000", 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.02531286 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:32.430529+0000", 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0797999999999999e-05 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.974 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106691 13.3 13:cd7d9036:::benchmark_data_smithi064_501823_object106690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 417.36102554000001, 2023-06-10T21:49:57.975 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.983989778, 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106691 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.976 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.977 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520098+0000", 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.978 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520145+0000", 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6650999999999998e-05 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520153+0000", 2023-06-10T21:49:57.979 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6559999999999992e-06 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520159+0000", 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4359999999999999e-06 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.980 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520202+0000", 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3389000000000001e-05 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520324+0000", 2023-06-10T21:49:57.981 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121074 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520472+0000", 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148715 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.982 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.526115+0000", 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056431629999999997 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547106+0000", 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020990175 2023-06-10T21:49:57.983 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547117+0000", 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.182e-05 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.984 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189400+0000", 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64228294200000002 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189417+0000", 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6770999999999999e-05 2023-06-10T21:49:57.985 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.503987+0000", 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314569638 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:57.986 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.504089+0000", 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101869 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106703 13.3 13:caa34abe:::benchmark_data_smithi064_501823_object106702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:49:57.987 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 417.34194085799999, 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.981275578, 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:49:57.988 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106703 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.989 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:57.990 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539182+0000", 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539209+0000", 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6724999999999999e-05 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.991 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542176+0000", 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002966991 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542190+0000", 2023-06-10T21:49:57.992 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4246999999999999e-05 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547147+0000", 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049566059999999997 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.993 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547155+0000", 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3969999999999994e-06 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189447+0000", 2023-06-10T21:49:57.994 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64229122500000002 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189456+0000", 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4609999999999997e-06 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.995 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505480+0000", 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.316023876 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505680+0000", 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000200366 2023-06-10T21:49:57.996 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505915+0000", 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000234653 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.997 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.508394+0000", 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024786389999999999 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.517040+0000", 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086467249999999992 2023-06-10T21:49:57.998 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.518965+0000", 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001924469 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:57.999 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519013+0000", 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000002e-05 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.520459+0000", 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001446073 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.000 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106693 13.2 13:481d9832:::benchmark_data_smithi064_501823_object106692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e391)", 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 416.69022626499998, 2023-06-10T21:49:58.001 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3284778340000001, 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106693 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.002 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.003 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190896+0000", 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.004 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190926+0000", 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9950999999999998e-05 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.005 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190941+0000", 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5157000000000001e-05 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190954+0000", 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2877999999999999e-05 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.006 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193380+0000", 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024257100000000002 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193393+0000", 2023-06-10T21:49:58.007 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2578000000000001e-05 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519272+0000", 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325879748 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.008 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519376+0000", 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010365300000000001 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.009 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113489 14.2 14:4e77c190:::benchmark_data_smithi064_559916_object5633:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.56565069300001, 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0400237479999999, 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.010 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113489 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:49:58.011 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.012 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315472+0000", 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315476+0000", 2023-06-10T21:49:58.013 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8809999999999998e-06 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315484+0000", 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8690000000000004e-06 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.014 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315490+0000", 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6720000000000001e-06 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315593+0000", 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103514 2023-06-10T21:49:58.015 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315724+0000", 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013101899999999999 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:49:58.016 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315831+0000", 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106902 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.318758+0000", 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002926508 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.017 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320772+0000", 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020147619999999998 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.018 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321248+0000", 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047574200000000003 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.361815+0000", 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040567060000000002 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.019 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.361827+0000", 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1287000000000001e-05 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332544+0000", 2023-06-10T21:49:58.020 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97071757299999994 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332558+0000", 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3631999999999999e-05 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.021 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355409+0000", 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022850839 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355497+0000", 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8868000000000004e-05 2023-06-10T21:49:58.022 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113492 14.2 14:4e2dedac:::benchmark_data_smithi064_559916_object5636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:49:58.023 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.560197999, 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0489783180000001, 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113492 2023-06-10T21:49:58.024 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.025 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320925+0000", 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:58.026 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320931+0000", 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3489999999999997e-06 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.027 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320944+0000", 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2369e-05 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320952+0000", 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5750000000000001e-06 2023-06-10T21:49:58.028 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321274+0000", 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000322009 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.029 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321413+0000", 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013840599999999999 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321538+0000", 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124993 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.030 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.384946+0000", 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063408185000000006 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.384958+0000", 2023-06-10T21:49:58.031 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2075e-05 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332592+0000", 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94763364299999997 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.032 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332600+0000", 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1370000000000002e-06 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355799+0000", 2023-06-10T21:49:58.033 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0231989180000001 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355949+0000", 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014996000000000001 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.034 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369808+0000", 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013859274 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369905+0000", 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6813999999999998e-05 2023-06-10T21:49:58.035 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113495 14.1 14:85da5a60:::benchmark_data_smithi064_559916_object5639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.55855065599999, 2023-06-10T21:49:58.036 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.126787014, 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113495 2023-06-10T21:49:58.037 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.038 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322573+0000", 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:58.039 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322579+0000", 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8270000000000003e-06 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.040 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322590+0000", 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1576000000000001e-05 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322598+0000", 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6040000000000004e-06 2023-06-10T21:49:58.041 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322655+0000", 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7009000000000003e-05 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.042 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322767+0000", 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112612 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322867+0000", 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9697000000000004e-05 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.043 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.327963+0000", 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050958289999999996 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.396532+0000", 2023-06-10T21:49:58.044 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068569243000000002 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.396544+0000", 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1412e-05 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.045 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334604+0000", 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93806067599999998 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334618+0000", 2023-06-10T21:49:58.046 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3395e-05 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.449264+0000", 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.114646585 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.047 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.449361+0000", 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6520000000000004e-05 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.048 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113506 14.2 14:437c8228:::benchmark_data_smithi064_559916_object5650:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.49149068700001, 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9829840920000001, 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.049 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113506 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.050 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.051 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389632+0000", 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389639+0000", 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9890000000000003e-06 2023-06-10T21:49:58.052 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389651+0000", 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1640999999999999e-05 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.053 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389660+0000", 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0599999999999997e-06 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389719+0000", 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9182999999999999e-05 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.054 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389729+0000", 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332652+0000", 2023-06-10T21:49:58.055 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.942923069 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332659+0000", 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2389999999999996e-06 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.056 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355991+0000", 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0233322730000001 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.356154+0000", 2023-06-10T21:49:58.057 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162709 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.356326+0000", 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017248599999999999 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.058 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.365479+0000", 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091523160000000006 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368469+0000", 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029906360000000001 2023-06-10T21:49:58.059 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369468+0000", 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00099876499999999998 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.060 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372462+0000", 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029935249999999999 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372521+0000", 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9165e-05 2023-06-10T21:49:58.061 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372618+0000", 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6997999999999995e-05 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.062 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.063 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.063 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113507 14.2 14:4867687b:::benchmark_data_smithi064_559916_object5651:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:49:58.063 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:49:58.063 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.49145179800001, 2023-06-10T21:49:58.063 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.983135852, 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.064 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113507 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.065 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.066 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389672+0000", 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389675+0000", 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5040000000000001e-06 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.067 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389680+0000", 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7849999999999997e-06 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389685+0000", 2023-06-10T21:49:58.068 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9180000000000002e-06 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389748+0000", 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2556000000000002e-05 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.069 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389754+0000", 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0560000000000001e-06 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332683+0000", 2023-06-10T21:49:58.070 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94292939200000003 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332690+0000", 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0119999999999999e-06 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.071 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359119+0000", 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0264282819999999 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359263+0000", 2023-06-10T21:49:58.072 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144466 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359414+0000", 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015090899999999999 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.073 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368481+0000", 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090673170000000001 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369506+0000", 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010247920000000001 2023-06-10T21:49:58.074 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369942+0000", 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043549000000000002 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.075 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372655+0000", 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002713783 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372710+0000", 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4558999999999997e-05 2023-06-10T21:49:58.076 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372808+0000", 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8418000000000005e-05 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.077 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113510 14.2 14:41be09f6:::benchmark_data_smithi064_559916_object5654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.48160062299999, 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.973454032, 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.078 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113510 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.079 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.080 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399522+0000", 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.081 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399528+0000", 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2519999999999999e-06 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399540+0000", 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2697999999999999e-05 2023-06-10T21:49:58.082 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399548+0000", 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9829999999999996e-06 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.083 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399590+0000", 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1582000000000003e-05 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399599+0000", 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5289999999999997e-06 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.084 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.333991+0000", 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93439153399999997 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334002+0000", 2023-06-10T21:49:58.085 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0492000000000001e-05 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362370+0000", 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0283688769999999 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.086 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362530+0000", 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159383 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362686+0000", 2023-06-10T21:49:58.087 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015642100000000001 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368494+0000", 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058078749999999997 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.088 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.371560+0000", 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030656170000000001 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372046+0000", 2023-06-10T21:49:58.089 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048609899999999998 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372850+0000", 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080377499999999995 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.090 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372896+0000", 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6505000000000003e-05 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372978+0000", 2023-06-10T21:49:58.091 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1731999999999996e-05 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113514 14.1 14:8819266d:::benchmark_data_smithi064_559916_object5658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:49:58.092 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 145.474827269, 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0529779540000002, 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.093 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113514 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.094 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.095 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406296+0000", 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406302+0000", 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.93e-06 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.096 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406316+0000", 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3675999999999999e-05 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406322+0000", 2023-06-10T21:49:58.097 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4699999999999999e-06 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406382+0000", 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9517e-05 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.098 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406393+0000", 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1188e-05 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334656+0000", 2023-06-10T21:49:58.099 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92826324699999996 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334666+0000", 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8989999999999996e-06 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.100 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453295+0000", 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1186290750000001 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453431+0000", 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013597699999999999 2023-06-10T21:49:58.101 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453547+0000", 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116291 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.102 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459016+0000", 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054681790000000001 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459027+0000", 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1032999999999999e-05 2023-06-10T21:49:58.103 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459070+0000", 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3897999999999999e-05 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.104 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459116+0000", 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5872999999999999e-05 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459167+0000", 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0689999999999997e-05 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.105 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459275+0000", 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108057 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.106 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113526 14.1 14:8a61b658:::benchmark_data_smithi064_559916_object5670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 144.43898276199999, 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.033151127, 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.107 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113526 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.108 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.109 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442140+0000", 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442146+0000", 2023-06-10T21:49:58.110 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.579e-06 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442163+0000", 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6396999999999999e-05 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.111 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442172+0000", 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4679999999999998e-06 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442233+0000", 2023-06-10T21:49:58.112 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0779000000000002e-05 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442250+0000", 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6546999999999999e-05 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.113 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456413+0000", 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0141633699999999 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456538+0000", 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012505200000000001 2023-06-10T21:49:58.114 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456651+0000", 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112871 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.115 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467864+0000", 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01121276 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467974+0000", 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011081 2023-06-10T21:49:58.116 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467990+0000", 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5242000000000001e-05 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.117 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475090+0000", 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071004320000000003 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475156+0000", 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6271000000000003e-05 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475293+0000", 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013633100000000001 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:191575 14.9 14:9518e930:::benchmark_data_smithi064_559916_object83719:head [delete] snapc 0=[] ondisk+write+known_if_redirected e469)", 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:55.519024+0000", 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 2.3621002450000002, 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.24499712700000001, 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:49:58.120 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 191575 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:49:58.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519024+0000", 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519024+0000", 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519023+0000", 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519028+0000", 2023-06-10T21:49:58.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2020000000000003e-06 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519040+0000", 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1770000000000001e-05 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519048+0000", 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519096+0000", 2023-06-10T21:49:58.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8236999999999999e-05 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519228+0000", 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013195599999999999 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,5,6", 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.519373+0000", 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144518 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.523019+0000", 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003646706 2023-06-10T21:49:58.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.525578+0000", 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002558374 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.525877+0000", 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00029878700000000001 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.526510+0000", 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00063382900000000003 2023-06-10T21:49:58.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.763841+0000", 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.23733026199999999 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:49:58.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.763898+0000", 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6993999999999997e-05 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:55.764021+0000", 2023-06-10T21:49:58.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000123604 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:49:58.132 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:49:58.133 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:58.555 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:49:58.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:58.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:49:58.673 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:49:58.771 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:49:58.826 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:49:58.874 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:49:58.901 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:49:59.177 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:49:59.265 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:49:59.307 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:49:59.353 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:49:59.398 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:49:59.440 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:49:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.449 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.450 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:49:59.483 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[180297]: pgmap v1727: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26067/71511 objects misplaced (36.452%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:49:59.484 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/387831683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:59.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.484 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.485 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[186706]: pgmap v1727: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26067/71511 objects misplaced (36.452%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:49:59.485 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:49:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/387831683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:59 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:49:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:59 smithi152 ceph-mon[156819]: pgmap v1727: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 519 op/s; 26067/71511 objects misplaced (36.452%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:49:59.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:49:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/387831683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:49:59.777 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:49:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:49:59.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:49:59.891 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:49:59.980 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:00.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:00 smithi064 ceph-mon[180297]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.350 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:00 smithi064 ceph-mon[180297]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.442 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:50:00.625 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:00 smithi064 ceph-mon[186706]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:00 smithi064 ceph-mon[186706]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:00 smithi152 ceph-mon[156819]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:00 smithi152 ceph-mon[156819]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-06-10T21:50:00.851 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:00.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:00.962 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:01.023 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:01.052 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:50:01.108 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:01.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:01.215 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:01.244 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:01.331 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:01.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:01.444 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:01.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[186706]: pgmap v1728: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 25916/70047 objects misplaced (36.998%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:50:01.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.519 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[180297]: pgmap v1728: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 25916/70047 objects misplaced (36.998%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:50:01.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.520 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:01 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.527 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:01 smithi152 ceph-mon[156819]: pgmap v1728: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 7.3 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 523 op/s; 25916/70047 objects misplaced (36.998%); 5.7 MiB/s, 90 objects/s recovering 2023-06-10T21:50:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:01 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:01.770 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:01.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:01.881 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:01.929 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:01.968 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:01.992 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:01.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:02.095 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:02.122 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:02.163 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:02.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:02.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:02.267 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:02.304 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:02.509 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:02.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:02.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:02.663 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:02.689 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:02 smithi064 ceph-mon[186706]: pgmap v1729: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 22764/63975 objects misplaced (35.583%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:02.694 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:02 smithi064 ceph-mon[180297]: pgmap v1729: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 22764/63975 objects misplaced (35.583%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:02.740 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:02 smithi152 ceph-mon[156819]: pgmap v1729: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 524 op/s; 22764/63975 objects misplaced (35.583%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:02.757 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:02.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:02.866 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:02.932 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:50:02.940 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:02] "GET /metrics HTTP/1.1" 200 224270 "" "Prometheus/2.33.4" 2023-06-10T21:50:02.949 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:03.286 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:50:03.286 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:50:03.502 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 14 pg_num to 26 2023-06-10T21:50:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/569560925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/569560925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:03.672 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:03.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/569560925' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:03 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]: dispatch 2023-06-10T21:50:03.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:03.786 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:03.820 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:03.904 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:03.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:04.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:04.081 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:04.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:50:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:50:04.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: osdmap e470: 8 total, 8 up, 5 in 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: pgmap v1731: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 520 op/s; 22764/63555 objects misplaced (35.818%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:50:04.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: osdmap e470: 8 total, 8 up, 5 in 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.912 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: pgmap v1731: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 520 op/s; 22764/63555 objects misplaced (35.818%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:04.913 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:04 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:50:04.929 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:50:04.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: osdmap e470: 8 total, 8 up, 5 in 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: failed to encode map e470 with expected crc 2023-06-10T21:50:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: pgmap v1731: 89 pgs: 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 520 op/s; 22764/63555 objects misplaced (35.818%); 4.0 MiB/s, 63 objects/s recovering 2023-06-10T21:50:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:04 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-10T21:50:05.033 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:05.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:05.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:05.141 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:05.176 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:05.223 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:05.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:05.326 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:05.356 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:05.454 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:05.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:05.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:05.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:05.583 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:05.585 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:05.628 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:05.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:05.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:05.737 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:50:05.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[186706]: osdmap e471: 8 total, 8 up, 5 in 2023-06-10T21:50:05.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[186706]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.738 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[186706]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.739 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[186706]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.739 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:50:05.739 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[180297]: osdmap e471: 8 total, 8 up, 5 in 2023-06-10T21:50:05.739 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[180297]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.740 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[180297]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.740 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:05 smithi064 ceph-mon[180297]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.759 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:05 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-10T21:50:05.759 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:05 smithi152 ceph-mon[156819]: osdmap e471: 8 total, 8 up, 5 in 2023-06-10T21:50:05.760 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:05 smithi152 ceph-mon[156819]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.760 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:05 smithi152 ceph-mon[156819]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.760 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:05 smithi152 ceph-mon[156819]: failed to encode map e471 with expected crc 2023-06-10T21:50:05.763 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:05.808 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:05.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:05.912 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:05.937 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:06.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:06.224 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:06.256 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:06.406 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:50:06.417 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:06.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:06.524 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:06.569 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:06.606 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:50:06.606 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:50:06.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e471 with expected crc 2023-06-10T21:50:06.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.812 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.8 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:06.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: osdmap e472: 8 total, 8 up, 5 in 2023-06-10T21:50:06.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.0 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.0 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.7 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.814 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.7 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.1 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:06.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.6 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:06.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.9 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.816 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.9 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.816 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.4 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:06.816 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.2 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.817 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.2 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.817 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: 14.5 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:06.817 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:06.817 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: pgmap v1734: 99 pgs: 10 unknown, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 22623/62646 objects misplaced (36.112%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:50:06.818 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:06.818 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:06.818 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: osdmap e473: 8 total, 8 up, 5 in 2023-06-10T21:50:06.818 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.819 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.819 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:06 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.823 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:06.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e471 with expected crc 2023-06-10T21:50:06.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.8 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:06.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: osdmap e472: 8 total, 8 up, 5 in 2023-06-10T21:50:06.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.0 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.0 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.829 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.7 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.830 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.7 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.1 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:06.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.6 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:06.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.9 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.9 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.4 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:06.833 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.2 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.2 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.834 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: 14.5 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:06.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:06.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: pgmap v1734: 99 pgs: 10 unknown, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 22623/62646 objects misplaced (36.112%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:50:06.835 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:06.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:06.836 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: osdmap e473: 8 total, 8 up, 5 in 2023-06-10T21:50:06.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.837 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.838 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e471 with expected crc 2023-06-10T21:50:06.838 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.8 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:06.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: osdmap e472: 8 total, 8 up, 5 in 2023-06-10T21:50:06.839 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.0 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.0 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:06.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.840 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.7 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.841 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.7 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:06.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:06.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.1 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:06.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.6 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:06.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.9 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.9 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:06.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.4 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:06.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.2 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.2 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:06.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: 14.5 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:06.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:06.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: pgmap v1734: 99 pgs: 10 unknown, 1 active+remapped+backfilling, 12 active+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 411 op/s; 22623/62646 objects misplaced (36.112%); 2.8 MiB/s, 44 objects/s recovering 2023-06-10T21:50:06.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:06.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3948612061' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:06.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: osdmap e473: 8 total, 8 up, 5 in 2023-06-10T21:50:06.845 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.846 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:06 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:06.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:06.948 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:06.977 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:07.021 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:07.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:07.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:07.125 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:07.151 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:07.354 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:07.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:07.405 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:50:07.459 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:07.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e472 with expected crc 2023-06-10T21:50:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: osdmap e474: 8 total, 8 up, 5 in 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e474 with expected crc 2023-06-10T21:50:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:07 smithi152 ceph-mon[156819]: failed to encode map e474 with expected crc 2023-06-10T21:50:07.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e472 with expected crc 2023-06-10T21:50:07.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: osdmap e474: 8 total, 8 up, 5 in 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e474 with expected crc 2023-06-10T21:50:07.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[186706]: failed to encode map e474 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e472 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e473 with expected crc 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: osdmap e474: 8 total, 8 up, 5 in 2023-06-10T21:50:07.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e474 with expected crc 2023-06-10T21:50:07.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:07 smithi064 ceph-mon[180297]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.035 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:50:08.125 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:08.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:08.232 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:08.315 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:08.431 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:08.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:08.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:08.568 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:08.698 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.10 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.10 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.18 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.12 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.699 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.15 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.12 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.19 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.19 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.17 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.17 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.16 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:08.700 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.14 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:08.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: 14.11 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:08.701 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:08 smithi152 ceph-mon[156819]: pgmap v1737: 99 pgs: 9 remapped+peering, 1 unknown, 12 active+remapped+backfill_wait, 1 active+remapped, 76 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 52 MiB/s wr, 1.88k op/s; 15342/61011 objects misplaced (25.146%); 7.4 MiB/s, 119 objects/s recovering 2023-06-10T21:50:08.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.10 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.10 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.18 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.12 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.15 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.12 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.19 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.756 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.19 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.17 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.17 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.16 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.14 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: 14.11 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:08.757 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[180297]: pgmap v1737: 99 pgs: 9 remapped+peering, 1 unknown, 12 active+remapped+backfill_wait, 1 active+remapped, 76 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 52 MiB/s wr, 1.88k op/s; 15342/61011 objects misplaced (25.146%); 7.4 MiB/s, 119 objects/s recovering 2023-06-10T21:50:08.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.10 continuing backfill to osd.0 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.10 continuing backfill to osd.6 from (436'5420,470'12241] MIN to 470'12241 2023-06-10T21:50:08.758 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.18 continuing backfill to osd.0 from (436'5361,470'12256] MIN to 470'12256 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.12 continuing backfill to osd.0 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.15 continuing backfill to osd.0 from (442'7063,470'12354] MIN to 470'12354 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.12 continuing backfill to osd.6 from (442'6923,470'12227] MIN to 470'12227 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.19 continuing backfill to osd.0 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.19 continuing backfill to osd.6 from (442'6918,470'12248] MIN to 470'12248 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.17 continuing backfill to osd.0 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.759 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.17 continuing backfill to osd.3 from (436'5326,470'12168] MIN to 470'12168 2023-06-10T21:50:08.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.16 continuing backfill to osd.0 from (435'5200,470'12048] MIN to 470'12048 2023-06-10T21:50:08.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: failed to encode map e474 with expected crc 2023-06-10T21:50:08.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.14 continuing backfill to osd.6 from (442'6792,470'12103] MIN to 470'12103 2023-06-10T21:50:08.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: 14.11 continuing backfill to osd.6 from (442'7150,470'12475] MIN to 470'12475 2023-06-10T21:50:08.760 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:08 smithi064 ceph-mon[186706]: pgmap v1737: 99 pgs: 9 remapped+peering, 1 unknown, 12 active+remapped+backfill_wait, 1 active+remapped, 76 active+clean; 1.2 GiB data, 6.9 GiB used, 440 GiB / 447 GiB avail; 52 MiB/s wr, 1.88k op/s; 15342/61011 objects misplaced (25.146%); 7.4 MiB/s, 119 objects/s recovering 2023-06-10T21:50:08.779 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:08.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:08.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:08.883 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:08.910 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:08.952 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:08.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:08.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:09.055 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:09.081 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:09.136 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:09.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:09.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:09.242 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:09.290 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:09.452 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:09.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:09.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:09.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:09.968 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:09.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:10.065 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:50:10.066 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:50:10.081 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:10.125 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:10.318 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:10.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:10.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:10.405 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:50:10.423 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:10.424 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:50:10.444 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:50:10.454 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:10.465 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:50:10.475 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:50:10.475 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:50:10.476 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-06-10T21:50:10.476 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-10T21:50:10.476 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-10T21:50:10.476 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-06-10T21:50:10.487 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:50:10.504 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:10.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:10.507 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:50:10.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:10.632 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:50:10.680 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:50:10.681 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:10.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.689 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.690 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.690 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.690 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.690 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:50:10.827 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:50:10.901 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:10.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:11.006 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:11.023 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-10T21:50:11.083 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:11.198 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:11.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:11.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:11.270 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:50:11.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:11.364 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: pgmap v1738: 99 pgs: 1 active+remapped+backfilling, 9 remapped+peering, 11 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 240 op/s; 14573/58434 objects misplaced (24.939%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:50:11.365 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2560378257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:11.366 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:11.412 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: pgmap v1738: 99 pgs: 1 active+remapped+backfilling, 9 remapped+peering, 11 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 240 op/s; 14573/58434 objects misplaced (24.939%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:50:11.413 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2560378257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: pgmap v1738: 99 pgs: 1 active+remapped+backfilling, 9 remapped+peering, 11 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.2 GiB data, 6.5 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 240 op/s; 14573/58434 objects misplaced (24.939%); 5.6 MiB/s, 89 objects/s recovering 2023-06-10T21:50:11.414 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2560378257' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:11.518 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:11.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:11.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:11.696 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:12.119 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:12.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:12.224 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:12.253 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:12.389 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:12.466 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:12.535 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:12.545 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:12 smithi152 ceph-mon[156819]: pgmap v1739: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 16292/49839 objects misplaced (32.689%); 4.8 MiB/s, 76 objects/s recovering 2023-06-10T21:50:12.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:12.646 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:12.724 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:12 smithi064 ceph-mon[180297]: pgmap v1739: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 16292/49839 objects misplaced (32.689%); 4.8 MiB/s, 76 objects/s recovering 2023-06-10T21:50:12.725 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:12 smithi064 ceph-mon[186706]: pgmap v1739: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 624 op/s; 16292/49839 objects misplaced (32.689%); 4.8 MiB/s, 76 objects/s recovering 2023-06-10T21:50:12.739 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:12.856 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:12.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:12.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:12.961 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:13.037 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:13.061 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:12] "GET /metrics HTTP/1.1" 200 224254 "" "Prometheus/2.33.4" 2023-06-10T21:50:13.164 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:13.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:13.225 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:50:13.225 INFO:teuthology.orchestra.run.smithi152.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-10T21:50:13.226 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:50:13.271 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:13.333 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:14.079 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:14.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:14.183 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:14.187 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:50:14.211 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:14.837 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:14.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:14.945 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:15.011 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:15.117 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:15.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:15.231 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:15.297 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:15.402 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:15 smithi152 ceph-mon[156819]: pgmap v1740: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 557 op/s; 16292/49254 objects misplaced (33.078%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:50:15.402 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:15 smithi064 ceph-mon[180297]: pgmap v1740: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 557 op/s; 16292/49254 objects misplaced (33.078%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:50:15.485 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:50:15.486 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:50:15.486 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:50:15.486 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:50:15.505 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:15.509 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:15.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:15.614 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:15.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:15 smithi064 ceph-mon[186706]: pgmap v1740: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1.0 GiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 557 op/s; 16292/49254 objects misplaced (33.078%); 4.1 MiB/s, 65 objects/s recovering 2023-06-10T21:50:15.678 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:15.765 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:15.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:15.878 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:15.975 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:16.126 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:16.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:16.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:16.232 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:16.258 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:16.302 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:16.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:16.409 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:16.458 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:50:16.498 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:16.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2611179403' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:16.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:16.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2611179403' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:16.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2611179403' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:16.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:17.044 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-10T21:50:17.166 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:17.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:17.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:17.287 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:17.340 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:17.406 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:17.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:17.513 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:17.541 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:17.678 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: pgmap v1741: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1004 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 16108/48246 objects misplaced (33.387%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:50:17.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:17.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:50:17.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: failed to encode map e475 with expected crc 2023-06-10T21:50:17.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:17.679 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: osdmap e475: 8 total, 8 up, 5 in 2023-06-10T21:50:17.680 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[186706]: failed to encode map e475 with expected crc 2023-06-10T21:50:17.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: pgmap v1741: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1004 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 16108/48246 objects misplaced (33.387%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:50:17.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:17.680 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:50:17.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: failed to encode map e475 with expected crc 2023-06-10T21:50:17.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:17.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: osdmap e475: 8 total, 8 up, 5 in 2023-06-10T21:50:17.681 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:17 smithi064 ceph-mon[180297]: failed to encode map e475 with expected crc 2023-06-10T21:50:17.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: pgmap v1741: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 1004 MiB data, 6.0 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 482 op/s; 16108/48246 objects misplaced (33.387%); 3.9 MiB/s, 62 objects/s recovering 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: failed to encode map e475 with expected crc 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: osdmap e475: 8 total, 8 up, 5 in 2023-06-10T21:50:17.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:17 smithi152 ceph-mon[156819]: failed to encode map e475 with expected crc 2023-06-10T21:50:18.006 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:18.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:18.118 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:18.172 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:18.249 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:18.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:18.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:18.361 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:18.423 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:18.468 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:18.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:18.572 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:18.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:18.643 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:18.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:18.748 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:18.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:19.098 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:19.105 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:50:19.105 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:50:19.106 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2023-06-10T21:50:19.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:50:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:19.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:19.209 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:19.250 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:19.298 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:19.330 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:19.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:19.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:19.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[180297]: pgmap v1743: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 815 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 60 op/s; 12347/39165 objects misplaced (31.526%); 950 KiB/s, 14 objects/s recovering 2023-06-10T21:50:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:19.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:19 smithi064 ceph-mon[186706]: pgmap v1743: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 815 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 60 op/s; 12347/39165 objects misplaced (31.526%); 950 KiB/s, 14 objects/s recovering 2023-06-10T21:50:19.442 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:19.554 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:19 smithi152 ceph-mon[156819]: pgmap v1743: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 815 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 60 op/s; 12347/39165 objects misplaced (31.526%); 950 KiB/s, 14 objects/s recovering 2023-06-10T21:50:19.728 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:19.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:19.837 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:19.907 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:20.009 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:50:20.016 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:50:20.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:50:20.101 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:50:20.130 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:20.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:20.147 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:50:20.196 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:50:20.237 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:50:20.239 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:20.274 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:20.275 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:50:20.309 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:50:20.324 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.325 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.326 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:50:20.354 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:20.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:20.464 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:20.515 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:20 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1802382020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:20.515 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:20 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1802382020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:20.520 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:20 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1802382020' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:20.531 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:20.622 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:20.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:20.737 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:20.802 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:20.868 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:20.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:20.973 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:21.002 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:21.251 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:21 smithi064 ceph-mon[180297]: pgmap v1744: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 777 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 12167/37353 objects misplaced (32.573%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:21.516 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:21 smithi064 ceph-mon[186706]: pgmap v1744: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 777 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 12167/37353 objects misplaced (32.573%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:21 smithi152 ceph-mon[156819]: pgmap v1744: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 777 MiB data, 5.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 702 op/s; 12167/37353 objects misplaced (32.573%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:22.296 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:22.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:22.403 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:22.433 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:22.453 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:22.520 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:22.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:22.631 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:22.694 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:22 smithi064 ceph-mon[186706]: pgmap v1745: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 574 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 740 op/s; 8310/27615 objects misplaced (30.092%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:22.697 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:22 smithi064 ceph-mon[180297]: pgmap v1745: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 574 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 740 op/s; 8310/27615 objects misplaced (30.092%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:22.700 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:22.901 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:22.926 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:22.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:22.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:22 smithi152 ceph-mon[156819]: pgmap v1745: 99 pgs: 1 active+remapped+backfilling, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 574 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 740 op/s; 8310/27615 objects misplaced (30.092%); 1.4 MiB/s, 21 objects/s recovering 2023-06-10T21:50:22.983 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:50:22.984 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:50:23.000 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:22] "GET /metrics HTTP/1.1" 200 224267 "" "Prometheus/2.33.4" 2023-06-10T21:50:23.032 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:23.070 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:23.137 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:23.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:23.248 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:23.295 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:23.385 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:23.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:23.491 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:23.576 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:24.480 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:50:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[186706]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[186706]: osdmap e476: 8 total, 8 up, 5 in 2023-06-10T21:50:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[186706]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[186706]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[186706]: pgmap v1747: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 553 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 8220/26580 objects misplaced (30.926%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[180297]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[180297]: osdmap e476: 8 total, 8 up, 5 in 2023-06-10T21:50:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[180297]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[180297]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:24 smithi064 ceph-mon[180297]: pgmap v1747: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 553 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 8220/26580 objects misplaced (30.926%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:24.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:24 smithi152 ceph-mon[156819]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:24 smithi152 ceph-mon[156819]: osdmap e476: 8 total, 8 up, 5 in 2023-06-10T21:50:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:24 smithi152 ceph-mon[156819]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:24 smithi152 ceph-mon[156819]: failed to encode map e476 with expected crc 2023-06-10T21:50:24.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:24 smithi152 ceph-mon[156819]: pgmap v1747: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 553 MiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 902 op/s; 8220/26580 objects misplaced (30.926%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:25.066 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:25.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:25.170 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:25.233 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:25.597 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:25.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:25.626 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-10T21:50:25.713 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:25.752 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:25.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.794 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[186706]: osdmap e477: 8 total, 8 up, 5 in 2023-06-10T21:50:25.806 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.807 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:25 smithi064 ceph-mon[180297]: osdmap e477: 8 total, 8 up, 5 in 2023-06-10T21:50:25.819 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:25.824 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: failed to encode map e477 with expected crc 2023-06-10T21:50:25.825 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:25 smithi152 ceph-mon[156819]: osdmap e477: 8 total, 8 up, 5 in 2023-06-10T21:50:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:25.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:25.927 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:25.972 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:26.030 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:26.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:26.136 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:26.192 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:26.274 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:26.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:26.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:26.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:26.453 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:26.789 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:26.879 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:26.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:26.989 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:27.045 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:27.112 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:27.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:27.220 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:27.266 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:27.335 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:27.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:27.441 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:27.532 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:27.616 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:50:27.624 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:50:27.653 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:27 smithi064 ceph-mon[180297]: pgmap v1749: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 539 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 550 op/s; 8220/25935 objects misplaced (31.695%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:27.653 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:27 smithi064 ceph-mon[186706]: pgmap v1749: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 539 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 550 op/s; 8220/25935 objects misplaced (31.695%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:27.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:27 smithi152 ceph-mon[156819]: pgmap v1749: 99 pgs: 1 peering, 20 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 539 MiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 550 op/s; 8220/25935 objects misplaced (31.695%); 1.5 MiB/s, 24 objects/s recovering 2023-06-10T21:50:27.737 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:28.460 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:50:28.645 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:28.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: failed to encode map e478 with expected crc 2023-06-10T21:50:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: osdmap e478: 8 total, 8 up, 5 in 2023-06-10T21:50:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: from='client.57223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:50:28.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/137316723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:28 smithi152 ceph-mon[156819]: pgmap v1751: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 1 peering, 18 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 359 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 574 op/s; 4151/17271 objects misplaced (24.035%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:50:28.750 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:28.785 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: failed to encode map e478 with expected crc 2023-06-10T21:50:28.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: osdmap e478: 8 total, 8 up, 5 in 2023-06-10T21:50:28.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: from='client.57223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:50:28.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/137316723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.786 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[186706]: pgmap v1751: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 1 peering, 18 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 359 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 574 op/s; 4151/17271 objects misplaced (24.035%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: failed to encode map e478 with expected crc 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: osdmap e478: 8 total, 8 up, 5 in 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: from='client.57223 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/137316723' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:50:28.787 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:28 smithi064 ceph-mon[180297]: pgmap v1751: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 1 peering, 18 active+remapped+backfill_wait, 1 active+remapped, 77 active+clean; 359 MiB data, 4.7 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 574 op/s; 4151/17271 objects misplaced (24.035%); 5.2 MiB/s, 83 objects/s recovering 2023-06-10T21:50:28.790 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:29.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: failed to encode map e478 with expected crc 2023-06-10T21:50:29.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:50:29.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:29 smithi152 ceph-mon[156819]: osdmap e479: 8 total, 8 up, 5 in 2023-06-10T21:50:29.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: failed to encode map e478 with expected crc 2023-06-10T21:50:29.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:29.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[180297]: osdmap e479: 8 total, 8 up, 5 in 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: failed to encode map e478 with expected crc 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:29.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: failed to encode map e479 with expected crc 2023-06-10T21:50:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:50:29.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:29 smithi064 ceph-mon[186706]: osdmap e479: 8 total, 8 up, 5 in 2023-06-10T21:50:30.358 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1750,"stamp":"2023-06-10T21:50:27.436042+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":376248230,"num_objects":5757,"num_object_clones":0,"num_object_copies":17271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4151,"num_objects_unfound":0,"num_objects_dirty":5757,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":210236,"num_write_kb":6905039,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23801,"num_bytes_recovered":1554021180,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160371,"ondisk_log_size":160371,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":4924308,"kb_used_data":1343852,"kb_used_omap":37,"kb_used_meta":3580122,"kb_avail":463801452,"statfs":{"total":479975178240,"available":474932686848,"internally_reserved":0,"allocated":1376104448,"data_stored":1369795504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38699,"internal_metadata":3666045141},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-438697984,"num_objects":-6694,"num_object_clones":0,"num_object_copies":-20082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-8016,"num_objects_unfound":0,"num_objects_dirty":-6694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6694,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":500,"num_bytes_recovered":32702488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.099121"},"pg_stats":[{"pgid":"14.17","version":"477'12634","reported_seq":15419,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.597984+0000","last_change":"2023-06-10T21:50:07.470735+0000","last_active":"2023-06-10T21:50:26.597984+0000","last_peered":"2023-06-10T21:50:26.597984+0000","last_clean":"2023-06-10T21:47:17.256756+0000","last_became_active":"2023-06-10T21:50:07.431457+0000","last_became_peered":"2023-06-10T21:50:07.431457+0000","last_unstale":"2023-06-10T21:50:26.597984+0000","last_undegraded":"2023-06-10T21:50:26.597984+0000","last_fullsized":"2023-06-10T21:50:26.597984+0000","mapping_epoch":473,"log_start":"436'5326","ondisk_log_start":"436'5326","created":471,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7308,"log_dups_size":0,"ondisk_log_size":7308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":192,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6550,"num_write_kb":214784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":671,"num_bytes_recovered":43941888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":162}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":527,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.488744+0000","last_change":"2023-06-10T21:47:18.342858+0000","last_active":"2023-06-10T21:50:23.488744+0000","last_peered":"2023-06-10T21:50:23.488744+0000","last_clean":"2023-06-10T21:50:23.488744+0000","last_became_active":"2023-06-10T21:47:18.342378+0000","last_became_peered":"2023-06-10T21:47:18.342378+0000","last_unstale":"2023-06-10T21:50:23.488744+0000","last_undegraded":"2023-06-10T21:50:23.488744+0000","last_fullsized":"2023-06-10T21:50:23.488744+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:58:30.423525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483965+0000","last_change":"2023-06-10T21:48:53.473755+0000","last_active":"2023-06-10T21:50:23.483965+0000","last_peered":"2023-06-10T21:50:23.483965+0000","last_clean":"2023-06-10T21:50:23.483965+0000","last_became_active":"2023-06-10T21:48:53.473408+0000","last_became_peered":"2023-06-10T21:48:53.473408+0000","last_unstale":"2023-06-10T21:50:23.483965+0000","last_undegraded":"2023-06-10T21:50:23.483965+0000","last_fullsized":"2023-06-10T21:50:23.483965+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:37:52.851231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"477'12552","reported_seq":12800,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.641054+0000","last_change":"2023-06-10T21:50:07.477914+0000","last_active":"2023-06-10T21:50:26.641054+0000","last_peered":"2023-06-10T21:50:26.641054+0000","last_clean":"2023-06-10T21:47:32.300137+0000","last_became_active":"2023-06-10T21:50:07.430922+0000","last_became_peered":"2023-06-10T21:50:07.430922+0000","last_unstale":"2023-06-10T21:50:26.641054+0000","last_undegraded":"2023-06-10T21:50:26.641054+0000","last_fullsized":"2023-06-10T21:50:26.641054+0000","mapping_epoch":473,"log_start":"442'6792","ondisk_log_start":"442'6792","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11567104,"num_objects":177,"num_object_clones":0,"num_object_copies":531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":162,"num_objects_unfound":0,"num_objects_dirty":177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6501,"num_write_kb":213664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2392064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":177}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":428,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483807+0000","last_change":"2023-06-10T21:48:53.404523+0000","last_active":"2023-06-10T21:50:23.483807+0000","last_peered":"2023-06-10T21:50:23.483807+0000","last_clean":"2023-06-10T21:50:23.483807+0000","last_became_active":"2023-06-10T21:48:53.403684+0000","last_became_peered":"2023-06-10T21:48:53.403684+0000","last_unstale":"2023-06-10T21:50:23.483807+0000","last_undegraded":"2023-06-10T21:50:23.483807+0000","last_fullsized":"2023-06-10T21:50:23.483807+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:06:20.451907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":76,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441126+0000","last_change":"2023-06-10T21:49:18.402971+0000","last_active":"2023-06-10T21:50:16.441126+0000","last_peered":"2023-06-10T21:50:16.441126+0000","last_clean":"2023-06-10T21:50:16.441126+0000","last_became_active":"2023-06-10T21:49:18.402645+0000","last_became_peered":"2023-06-10T21:49:18.402645+0000","last_unstale":"2023-06-10T21:50:16.441126+0000","last_undegraded":"2023-06-10T21:50:16.441126+0000","last_fullsized":"2023-06-10T21:50:16.441126+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:57:01.556949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.15","version":"477'12819","reported_seq":13209,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.539743+0000","last_change":"2023-06-10T21:50:07.464826+0000","last_active":"2023-06-10T21:50:26.539743+0000","last_peered":"2023-06-10T21:50:26.539743+0000","last_clean":"2023-06-10T21:47:17.264551+0000","last_became_active":"2023-06-10T21:50:07.430865+0000","last_became_peered":"2023-06-10T21:50:07.430865+0000","last_unstale":"2023-06-10T21:50:26.539743+0000","last_undegraded":"2023-06-10T21:50:26.539743+0000","last_fullsized":"2023-06-10T21:50:26.539743+0000","mapping_epoch":473,"log_start":"442'7063","ondisk_log_start":"442'7063","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9699328,"num_objects":148,"num_object_clones":0,"num_object_copies":444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":122,"num_objects_unfound":0,"num_objects_dirty":148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6642,"num_write_kb":217280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":4980736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":148}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":142,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.481414+0000","last_change":"2023-06-10T21:47:18.339120+0000","last_active":"2023-06-10T21:50:23.481414+0000","last_peered":"2023-06-10T21:50:23.481414+0000","last_clean":"2023-06-10T21:50:23.481414+0000","last_became_active":"2023-06-10T21:47:18.338758+0000","last_became_peered":"2023-06-10T21:47:18.338758+0000","last_unstale":"2023-06-10T21:50:23.481414+0000","last_undegraded":"2023-06-10T21:50:23.481414+0000","last_fullsized":"2023-06-10T21:50:23.481414+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:10:24.375802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435826+0000","last_change":"2023-06-10T21:49:18.402969+0000","last_active":"2023-06-10T21:50:16.435826+0000","last_peered":"2023-06-10T21:50:16.435826+0000","last_clean":"2023-06-10T21:50:16.435826+0000","last_became_active":"2023-06-10T21:49:18.402624+0000","last_became_peered":"2023-06-10T21:49:18.402624+0000","last_unstale":"2023-06-10T21:50:16.435826+0000","last_undegraded":"2023-06-10T21:50:16.435826+0000","last_fullsized":"2023-06-10T21:50:16.435826+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441622+0000","last_change":"2023-06-10T21:49:18.403030+0000","last_active":"2023-06-10T21:50:16.441622+0000","last_peered":"2023-06-10T21:50:16.441622+0000","last_clean":"2023-06-10T21:50:16.441622+0000","last_became_active":"2023-06-10T21:49:18.402590+0000","last_became_peered":"2023-06-10T21:49:18.402590+0000","last_unstale":"2023-06-10T21:50:16.441622+0000","last_undegraded":"2023-06-10T21:50:16.441622+0000","last_fullsized":"2023-06-10T21:50:16.441622+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:55.929958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":175,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.481438+0000","last_change":"2023-06-10T21:49:18.398386+0000","last_active":"2023-06-10T21:50:23.481438+0000","last_peered":"2023-06-10T21:50:23.481438+0000","last_clean":"2023-06-10T21:50:23.481438+0000","last_became_active":"2023-06-10T21:49:18.398058+0000","last_became_peered":"2023-06-10T21:49:18.398058+0000","last_unstale":"2023-06-10T21:50:23.481438+0000","last_undegraded":"2023-06-10T21:50:23.481438+0000","last_fullsized":"2023-06-10T21:50:23.481438+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:12:47.857409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.429696+0000","last_change":"2023-06-10T21:49:18.403576+0000","last_active":"2023-06-10T21:50:16.429696+0000","last_peered":"2023-06-10T21:50:16.429696+0000","last_clean":"2023-06-10T21:50:16.429696+0000","last_became_active":"2023-06-10T21:49:18.403071+0000","last_became_peered":"2023-06-10T21:49:18.403071+0000","last_unstale":"2023-06-10T21:50:16.429696+0000","last_undegraded":"2023-06-10T21:50:16.429696+0000","last_fullsized":"2023-06-10T21:50:16.429696+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":239,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483736+0000","last_change":"2023-06-10T21:48:53.403919+0000","last_active":"2023-06-10T21:50:23.483736+0000","last_peered":"2023-06-10T21:50:23.483736+0000","last_clean":"2023-06-10T21:50:23.483736+0000","last_became_active":"2023-06-10T21:48:53.403593+0000","last_became_peered":"2023-06-10T21:48:53.403593+0000","last_unstale":"2023-06-10T21:50:23.483736+0000","last_undegraded":"2023-06-10T21:50:23.483736+0000","last_fullsized":"2023-06-10T21:50:23.483736+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:59.593873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.18","version":"477'12751","reported_seq":14898,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.568612+0000","last_change":"2023-06-10T21:50:07.479333+0000","last_active":"2023-06-10T21:50:26.568612+0000","last_peered":"2023-06-10T21:50:26.568612+0000","last_clean":"2023-06-10T21:47:17.280051+0000","last_became_active":"2023-06-10T21:50:07.433798+0000","last_became_peered":"2023-06-10T21:50:07.433798+0000","last_unstale":"2023-06-10T21:50:26.568612+0000","last_undegraded":"2023-06-10T21:50:26.568612+0000","last_fullsized":"2023-06-10T21:50:26.568612+0000","mapping_epoch":473,"log_start":"436'5361","ondisk_log_start":"436'5361","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7390,"log_dups_size":0,"ondisk_log_size":7390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9306112,"num_objects":142,"num_object_clones":0,"num_object_copies":426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6623,"num_write_kb":216480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":516,"num_bytes_recovered":33783808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","0","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":142}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":248,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.439459+0000","last_change":"2023-06-10T21:47:18.431486+0000","last_active":"2023-06-10T21:50:16.439459+0000","last_peered":"2023-06-10T21:50:16.439459+0000","last_clean":"2023-06-10T21:50:16.439459+0000","last_became_active":"2023-06-10T21:47:18.326725+0000","last_became_peered":"2023-06-10T21:47:18.326725+0000","last_unstale":"2023-06-10T21:50:16.439459+0000","last_undegraded":"2023-06-10T21:50:16.439459+0000","last_fullsized":"2023-06-10T21:50:16.439459+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435340+0000","last_change":"2023-06-10T21:48:53.381964+0000","last_active":"2023-06-10T21:50:16.435340+0000","last_peered":"2023-06-10T21:50:16.435340+0000","last_clean":"2023-06-10T21:50:16.435340+0000","last_became_active":"2023-06-10T21:48:53.381436+0000","last_became_peered":"2023-06-10T21:48:53.381436+0000","last_unstale":"2023-06-10T21:50:16.435340+0000","last_undegraded":"2023-06-10T21:50:16.435340+0000","last_fullsized":"2023-06-10T21:50:16.435340+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.19","version":"477'12708","reported_seq":19092,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.615497+0000","last_change":"2023-06-10T21:50:07.499173+0000","last_active":"2023-06-10T21:50:26.615497+0000","last_peered":"2023-06-10T21:50:26.615497+0000","last_clean":"2023-06-10T21:47:17.293084+0000","last_became_active":"2023-06-10T21:50:07.443456+0000","last_became_peered":"2023-06-10T21:50:07.443456+0000","last_unstale":"2023-06-10T21:50:26.615497+0000","last_undegraded":"2023-06-10T21:50:26.615497+0000","last_fullsized":"2023-06-10T21:50:26.615497+0000","mapping_epoch":473,"log_start":"442'6918","ondisk_log_start":"442'6918","created":471,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5790,"log_dups_size":0,"ondisk_log_size":5790,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12582912,"num_objects":192,"num_object_clones":0,"num_object_copies":576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":192,"num_objects_unfound":0,"num_objects_dirty":192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6584,"num_write_kb":216832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1567,"num_bytes_recovered":102662144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,2,5],"avail_no_missing":["3","0","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":192}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":437,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.430797+0000","last_change":"2023-06-10T21:48:53.849265+0000","last_active":"2023-06-10T21:50:16.430797+0000","last_peered":"2023-06-10T21:50:16.430797+0000","last_clean":"2023-06-10T21:50:16.430797+0000","last_became_active":"2023-06-10T21:48:53.630188+0000","last_became_peered":"2023-06-10T21:48:53.630188+0000","last_unstale":"2023-06-10T21:50:16.430797+0000","last_undegraded":"2023-06-10T21:50:16.430797+0000","last_fullsized":"2023-06-10T21:50:16.430797+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23580,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435564+0000","last_change":"2023-06-10T21:49:18.402957+0000","last_active":"2023-06-10T21:50:16.435564+0000","last_peered":"2023-06-10T21:50:16.435564+0000","last_clean":"2023-06-10T21:50:16.435564+0000","last_became_active":"2023-06-10T21:49:18.402621+0000","last_became_peered":"2023-06-10T21:49:18.402621+0000","last_unstale":"2023-06-10T21:50:16.435564+0000","last_undegraded":"2023-06-10T21:50:16.435564+0000","last_fullsized":"2023-06-10T21:50:16.435564+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":565,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.429531+0000","last_change":"2023-06-10T21:48:53.936290+0000","last_active":"2023-06-10T21:50:16.429531+0000","last_peered":"2023-06-10T21:50:16.429531+0000","last_clean":"2023-06-10T21:50:16.429531+0000","last_became_active":"2023-06-10T21:48:53.631956+0000","last_became_peered":"2023-06-10T21:48:53.631956+0000","last_unstale":"2023-06-10T21:50:16.429531+0000","last_undegraded":"2023-06-10T21:50:16.429531+0000","last_fullsized":"2023-06-10T21:50:16.429531+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":427,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482329+0000","last_change":"2023-06-10T21:39:50.273016+0000","last_active":"2023-06-10T21:50:23.482329+0000","last_peered":"2023-06-10T21:50:23.482329+0000","last_clean":"2023-06-10T21:50:23.482329+0000","last_became_active":"2023-06-10T21:39:50.272344+0000","last_became_peered":"2023-06-10T21:39:50.272344+0000","last_unstale":"2023-06-10T21:50:23.482329+0000","last_undegraded":"2023-06-10T21:50:23.482329+0000","last_fullsized":"2023-06-10T21:50:23.482329+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:35:00.662457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":49,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.437876+0000","last_change":"2023-06-10T21:47:18.322156+0000","last_active":"2023-06-10T21:50:16.437876+0000","last_peered":"2023-06-10T21:50:16.437876+0000","last_clean":"2023-06-10T21:50:16.437876+0000","last_became_active":"2023-06-10T21:47:18.320854+0000","last_became_peered":"2023-06-10T21:47:18.320854+0000","last_unstale":"2023-06-10T21:50:16.437876+0000","last_undegraded":"2023-06-10T21:50:16.437876+0000","last_fullsized":"2023-06-10T21:50:16.437876+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:07:26.646183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":361,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.439566+0000","last_change":"2023-06-10T21:48:53.385839+0000","last_active":"2023-06-10T21:50:16.439566+0000","last_peered":"2023-06-10T21:50:16.439566+0000","last_clean":"2023-06-10T21:50:16.439566+0000","last_became_active":"2023-06-10T21:48:53.385521+0000","last_became_peered":"2023-06-10T21:48:53.385521+0000","last_unstale":"2023-06-10T21:50:16.439566+0000","last_undegraded":"2023-06-10T21:50:16.439566+0000","last_fullsized":"2023-06-10T21:50:16.439566+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":506,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.428633+0000","last_change":"2023-06-10T21:48:53.380288+0000","last_active":"2023-06-10T21:50:16.428633+0000","last_peered":"2023-06-10T21:50:16.428633+0000","last_clean":"2023-06-10T21:50:16.428633+0000","last_became_active":"2023-06-10T21:48:53.376181+0000","last_became_peered":"2023-06-10T21:48:53.376181+0000","last_unstale":"2023-06-10T21:50:16.428633+0000","last_undegraded":"2023-06-10T21:50:16.428633+0000","last_fullsized":"2023-06-10T21:50:16.428633+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.440897+0000","last_change":"2023-06-10T21:47:33.372564+0000","last_active":"2023-06-10T21:50:16.440897+0000","last_peered":"2023-06-10T21:50:16.440897+0000","last_clean":"2023-06-10T21:50:16.440897+0000","last_became_active":"2023-06-10T21:47:33.371667+0000","last_became_peered":"2023-06-10T21:47:33.371667+0000","last_unstale":"2023-06-10T21:50:16.440897+0000","last_undegraded":"2023-06-10T21:50:16.440897+0000","last_fullsized":"2023-06-10T21:50:16.440897+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:02.002487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":41,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.434750+0000","last_change":"2023-06-10T21:48:53.680926+0000","last_active":"2023-06-10T21:50:16.434750+0000","last_peered":"2023-06-10T21:50:16.434750+0000","last_clean":"2023-06-10T21:50:16.434750+0000","last_became_active":"2023-06-10T21:48:53.379343+0000","last_became_peered":"2023-06-10T21:48:53.379343+0000","last_unstale":"2023-06-10T21:50:16.434750+0000","last_undegraded":"2023-06-10T21:50:16.434750+0000","last_fullsized":"2023-06-10T21:50:16.434750+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":320,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.498762+0000","last_change":"2023-06-10T21:48:53.387743+0000","last_active":"2023-06-10T21:50:23.498762+0000","last_peered":"2023-06-10T21:50:23.498762+0000","last_clean":"2023-06-10T21:50:23.498762+0000","last_became_active":"2023-06-10T21:48:53.385099+0000","last_became_peered":"2023-06-10T21:48:53.385099+0000","last_unstale":"2023-06-10T21:50:23.498762+0000","last_undegraded":"2023-06-10T21:50:23.498762+0000","last_fullsized":"2023-06-10T21:50:23.498762+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:50:59.882320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.2","version":"477'12662","reported_seq":17045,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.618529+0000","last_change":"2023-06-10T21:50:05.727732+0000","last_active":"2023-06-10T21:50:26.618529+0000","last_peered":"2023-06-10T21:50:26.618529+0000","last_clean":"2023-06-10T21:47:17.271926+0000","last_became_active":"2023-06-10T21:50:05.708438+0000","last_became_peered":"2023-06-10T21:50:05.708438+0000","last_unstale":"2023-06-10T21:50:26.618529+0000","last_undegraded":"2023-06-10T21:50:26.618529+0000","last_fullsized":"2023-06-10T21:50:26.618529+0000","mapping_epoch":454,"log_start":"442'6923","ondisk_log_start":"442'6923","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12156928,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6548,"num_write_kb":215488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":712,"num_bytes_recovered":46596096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[3,7,5],"avail_no_missing":["3","0","5","6","7"],"object_location_counts":[{"shards":"3,5,7","objects":185}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":294,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.431031+0000","last_change":"2023-06-10T21:49:18.419142+0000","last_active":"2023-06-10T21:50:16.431031+0000","last_peered":"2023-06-10T21:50:16.431031+0000","last_clean":"2023-06-10T21:50:16.431031+0000","last_became_active":"2023-06-10T21:49:18.403225+0000","last_became_peered":"2023-06-10T21:49:18.403225+0000","last_unstale":"2023-06-10T21:50:16.431031+0000","last_undegraded":"2023-06-10T21:50:16.431031+0000","last_fullsized":"2023-06-10T21:50:16.431031+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":198,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.427967+0000","last_change":"2023-06-10T21:48:53.381781+0000","last_active":"2023-06-10T21:50:16.427967+0000","last_peered":"2023-06-10T21:50:16.427967+0000","last_clean":"2023-06-10T21:50:16.427967+0000","last_became_active":"2023-06-10T21:48:53.381469+0000","last_became_peered":"2023-06-10T21:48:53.381469+0000","last_unstale":"2023-06-10T21:50:16.427967+0000","last_undegraded":"2023-06-10T21:50:16.427967+0000","last_fullsized":"2023-06-10T21:50:16.427967+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"476'12665","reported_seq":28967,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:24.051155+0000","last_change":"2023-06-10T21:50:05.727945+0000","last_active":"2023-06-10T21:50:24.051155+0000","last_peered":"2023-06-10T21:50:24.051155+0000","last_clean":"2023-06-10T21:50:24.051155+0000","last_became_active":"2023-06-10T21:50:05.708813+0000","last_became_peered":"2023-06-10T21:50:05.708813+0000","last_unstale":"2023-06-10T21:50:24.051155+0000","last_undegraded":"2023-06-10T21:50:24.051155+0000","last_fullsized":"2023-06-10T21:50:24.051155+0000","mapping_epoch":460,"log_start":"445'7341","ondisk_log_start":"445'7341","created":420,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13205504,"num_objects":201,"num_object_clones":0,"num_object_copies":603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6530,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4042,"num_bytes_recovered":264896512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":294,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.429785+0000","last_change":"2023-06-10T21:47:33.362906+0000","last_active":"2023-06-10T21:50:16.429785+0000","last_peered":"2023-06-10T21:50:16.429785+0000","last_clean":"2023-06-10T21:50:16.429785+0000","last_became_active":"2023-06-10T21:47:33.349060+0000","last_became_peered":"2023-06-10T21:47:33.349060+0000","last_unstale":"2023-06-10T21:50:16.429785+0000","last_undegraded":"2023-06-10T21:50:16.429785+0000","last_fullsized":"2023-06-10T21:50:16.429785+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.434643+0000","last_change":"2023-06-10T21:48:53.379866+0000","last_active":"2023-06-10T21:50:16.434643+0000","last_peered":"2023-06-10T21:50:16.434643+0000","last_clean":"2023-06-10T21:50:16.434643+0000","last_became_active":"2023-06-10T21:48:53.379538+0000","last_became_peered":"2023-06-10T21:48:53.379538+0000","last_unstale":"2023-06-10T21:50:16.434643+0000","last_undegraded":"2023-06-10T21:50:16.434643+0000","last_fullsized":"2023-06-10T21:50:16.434643+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.0","version":"477'12660","reported_seq":12774,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.541009+0000","last_change":"2023-06-10T21:50:05.732600+0000","last_active":"2023-06-10T21:50:26.541009+0000","last_peered":"2023-06-10T21:50:26.541009+0000","last_clean":"2023-06-10T21:47:17.279925+0000","last_became_active":"2023-06-10T21:50:05.707101+0000","last_became_peered":"2023-06-10T21:50:05.707101+0000","last_unstale":"2023-06-10T21:50:26.541009+0000","last_undegraded":"2023-06-10T21:50:26.541009+0000","last_fullsized":"2023-06-10T21:50:26.541009+0000","mapping_epoch":461,"log_start":"436'5420","ondisk_log_start":"436'5420","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7240,"log_dups_size":0,"ondisk_log_size":7240,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11370496,"num_objects":173,"num_object_clones":0,"num_object_copies":519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":346,"num_objects_unfound":0,"num_objects_dirty":173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6539,"num_write_kb":214816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":173}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":581,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.439479+0000","last_change":"2023-06-10T21:48:53.675024+0000","last_active":"2023-06-10T21:50:16.439479+0000","last_peered":"2023-06-10T21:50:16.439479+0000","last_clean":"2023-06-10T21:50:16.439479+0000","last_became_active":"2023-06-10T21:48:53.385337+0000","last_became_peered":"2023-06-10T21:48:53.385337+0000","last_unstale":"2023-06-10T21:50:16.439479+0000","last_undegraded":"2023-06-10T21:50:16.439479+0000","last_fullsized":"2023-06-10T21:50:16.439479+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435559+0000","last_change":"2023-06-10T21:49:18.401260+0000","last_active":"2023-06-10T21:50:16.435559+0000","last_peered":"2023-06-10T21:50:16.435559+0000","last_clean":"2023-06-10T21:50:16.435559+0000","last_became_active":"2023-06-10T21:49:18.400755+0000","last_became_peered":"2023-06-10T21:49:18.400755+0000","last_unstale":"2023-06-10T21:50:16.435559+0000","last_undegraded":"2023-06-10T21:50:16.435559+0000","last_fullsized":"2023-06-10T21:50:16.435559+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"477'12937","reported_seq":13185,"reported_epoch":477,"state":"active+remapped+backfilling","last_fresh":"2023-06-10T21:50:26.668621+0000","last_change":"2023-06-10T21:50:26.447224+0000","last_active":"2023-06-10T21:50:26.668621+0000","last_peered":"2023-06-10T21:50:26.668621+0000","last_clean":"2023-06-10T21:47:32.299728+0000","last_became_active":"2023-06-10T21:50:05.705953+0000","last_became_peered":"2023-06-10T21:50:05.705953+0000","last_unstale":"2023-06-10T21:50:26.668621+0000","last_undegraded":"2023-06-10T21:50:26.668621+0000","last_fullsized":"2023-06-10T21:50:26.668621+0000","mapping_epoch":454,"log_start":"442'7150","ondisk_log_start":"442'7150","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10911744,"num_objects":166,"num_object_clones":0,"num_object_copies":498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":124,"num_objects_unfound":0,"num_objects_dirty":166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6699,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":3407872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":166}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":155,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.481969+0000","last_change":"2023-06-10T21:49:18.395063+0000","last_active":"2023-06-10T21:50:23.481969+0000","last_peered":"2023-06-10T21:50:23.481969+0000","last_clean":"2023-06-10T21:50:23.481969+0000","last_became_active":"2023-06-10T21:49:18.394738+0000","last_became_peered":"2023-06-10T21:49:18.394738+0000","last_unstale":"2023-06-10T21:50:23.481969+0000","last_undegraded":"2023-06-10T21:50:23.481969+0000","last_fullsized":"2023-06-10T21:50:23.481969+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:47:33.832467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.440133+0000","last_change":"2023-06-10T21:49:18.403580+0000","last_active":"2023-06-10T21:50:16.440133+0000","last_peered":"2023-06-10T21:50:16.440133+0000","last_clean":"2023-06-10T21:50:16.440133+0000","last_became_active":"2023-06-10T21:49:18.403128+0000","last_became_peered":"2023-06-10T21:49:18.403128+0000","last_unstale":"2023-06-10T21:50:16.440133+0000","last_undegraded":"2023-06-10T21:50:16.440133+0000","last_fullsized":"2023-06-10T21:50:16.440133+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:25:18.227144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"477'12478","reported_seq":17577,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.602947+0000","last_change":"2023-06-10T21:50:05.551623+0000","last_active":"2023-06-10T21:50:26.602947+0000","last_peered":"2023-06-10T21:50:26.602947+0000","last_clean":"2023-06-10T21:47:17.257047+0000","last_became_active":"2023-06-10T21:50:05.521436+0000","last_became_peered":"2023-06-10T21:50:05.521436+0000","last_unstale":"2023-06-10T21:50:26.602947+0000","last_undegraded":"2023-06-10T21:50:26.602947+0000","last_fullsized":"2023-06-10T21:50:26.602947+0000","mapping_epoch":439,"log_start":"435'5200","ondisk_log_start":"435'5200","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7278,"log_dups_size":0,"ondisk_log_size":7278,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10878976,"num_objects":166,"num_object_clones":0,"num_object_copies":498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6454,"num_write_kb":211840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1252,"num_bytes_recovered":81985536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","0","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":166}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":150,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.481811+0000","last_change":"2023-06-10T21:47:18.440477+0000","last_active":"2023-06-10T21:50:23.481811+0000","last_peered":"2023-06-10T21:50:23.481811+0000","last_clean":"2023-06-10T21:50:23.481811+0000","last_became_active":"2023-06-10T21:47:18.339247+0000","last_became_peered":"2023-06-10T21:47:18.339247+0000","last_unstale":"2023-06-10T21:50:23.481811+0000","last_undegraded":"2023-06-10T21:50:23.481811+0000","last_fullsized":"2023-06-10T21:50:23.481811+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:44:14.648072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":142,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482254+0000","last_change":"2023-06-10T21:48:53.389236+0000","last_active":"2023-06-10T21:50:23.482254+0000","last_peered":"2023-06-10T21:50:23.482254+0000","last_clean":"2023-06-10T21:50:23.482254+0000","last_became_active":"2023-06-10T21:48:53.388982+0000","last_became_peered":"2023-06-10T21:48:53.388982+0000","last_unstale":"2023-06-10T21:50:23.482254+0000","last_undegraded":"2023-06-10T21:50:23.482254+0000","last_fullsized":"2023-06-10T21:50:23.482254+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:02:53.667590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.7","version":"477'12596","reported_seq":15381,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.612823+0000","last_change":"2023-06-10T21:50:05.538762+0000","last_active":"2023-06-10T21:50:26.612823+0000","last_peered":"2023-06-10T21:50:26.612823+0000","last_clean":"2023-06-10T21:47:17.256756+0000","last_became_active":"2023-06-10T21:50:05.516344+0000","last_became_peered":"2023-06-10T21:50:05.516344+0000","last_unstale":"2023-06-10T21:50:26.612823+0000","last_undegraded":"2023-06-10T21:50:26.612823+0000","last_fullsized":"2023-06-10T21:50:26.612823+0000","mapping_epoch":443,"log_start":"436'5326","ondisk_log_start":"436'5326","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7270,"log_dups_size":0,"ondisk_log_size":7270,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13107200,"num_objects":200,"num_object_clones":0,"num_object_copies":600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":270,"num_objects_unfound":0,"num_objects_dirty":200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":214784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":670,"num_bytes_recovered":43941888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":200}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435278+0000","last_change":"2023-06-10T21:49:18.394539+0000","last_active":"2023-06-10T21:50:16.435278+0000","last_peered":"2023-06-10T21:50:16.435278+0000","last_clean":"2023-06-10T21:50:16.435278+0000","last_became_active":"2023-06-10T21:49:18.394003+0000","last_became_peered":"2023-06-10T21:49:18.394003+0000","last_unstale":"2023-06-10T21:50:16.435278+0000","last_undegraded":"2023-06-10T21:50:16.435278+0000","last_fullsized":"2023-06-10T21:50:16.435278+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":535,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.493194+0000","last_change":"2023-06-10T21:48:53.472531+0000","last_active":"2023-06-10T21:50:23.493194+0000","last_peered":"2023-06-10T21:50:23.493194+0000","last_clean":"2023-06-10T21:50:23.493194+0000","last_became_active":"2023-06-10T21:48:53.453752+0000","last_became_peered":"2023-06-10T21:48:53.453752+0000","last_unstale":"2023-06-10T21:50:23.493194+0000","last_undegraded":"2023-06-10T21:50:23.493194+0000","last_fullsized":"2023-06-10T21:50:23.493194+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:41:13.163563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.4","version":"475'12489","reported_seq":13507,"reported_epoch":476,"state":"peering","last_fresh":"2023-06-10T21:50:23.484889+0000","last_change":"2023-06-10T21:50:23.484774+0000","last_active":"2023-06-10T21:50:23.484583+0000","last_peered":"2023-06-10T21:50:23.452993+0000","last_clean":"2023-06-10T21:50:23.452993+0000","last_became_active":"2023-06-10T21:50:05.707117+0000","last_became_peered":"2023-06-10T21:50:05.707117+0000","last_unstale":"2023-06-10T21:50:23.484889+0000","last_undegraded":"2023-06-10T21:50:23.484889+0000","last_fullsized":"2023-06-10T21:50:23.484889+0000","mapping_epoch":476,"log_start":"442'6792","ondisk_log_start":"442'6792","created":420,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5697,"log_dups_size":0,"ondisk_log_size":5697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15695872,"num_objects":239,"num_object_clones":0,"num_object_copies":717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6437,"num_write_kb":213664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":417,"num_bytes_recovered":27361280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":504,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.493188+0000","last_change":"2023-06-10T21:49:18.401055+0000","last_active":"2023-06-10T21:50:23.493188+0000","last_peered":"2023-06-10T21:50:23.493188+0000","last_clean":"2023-06-10T21:50:23.493188+0000","last_became_active":"2023-06-10T21:49:18.400215+0000","last_became_peered":"2023-06-10T21:49:18.400215+0000","last_unstale":"2023-06-10T21:50:23.493188+0000","last_undegraded":"2023-06-10T21:50:23.493188+0000","last_fullsized":"2023-06-10T21:50:23.493188+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:06:57.351053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.438415+0000","last_change":"2023-06-10T21:49:18.399132+0000","last_active":"2023-06-10T21:50:16.438415+0000","last_peered":"2023-06-10T21:50:16.438415+0000","last_clean":"2023-06-10T21:50:16.438415+0000","last_became_active":"2023-06-10T21:49:18.398793+0000","last_became_peered":"2023-06-10T21:49:18.398793+0000","last_unstale":"2023-06-10T21:50:16.438415+0000","last_undegraded":"2023-06-10T21:50:16.438415+0000","last_fullsized":"2023-06-10T21:50:16.438415+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.5","version":"477'12793","reported_seq":13302,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.668368+0000","last_change":"2023-06-10T21:50:05.755962+0000","last_active":"2023-06-10T21:50:26.668368+0000","last_peered":"2023-06-10T21:50:26.668368+0000","last_clean":"2023-06-10T21:47:17.264551+0000","last_became_active":"2023-06-10T21:50:05.520629+0000","last_became_peered":"2023-06-10T21:50:05.520629+0000","last_unstale":"2023-06-10T21:50:26.668368+0000","last_undegraded":"2023-06-10T21:50:26.668368+0000","last_fullsized":"2023-06-10T21:50:26.668368+0000","mapping_epoch":439,"log_start":"442'7063","ondisk_log_start":"442'7063","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11403264,"num_objects":174,"num_object_clones":0,"num_object_copies":522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":122,"num_objects_unfound":0,"num_objects_dirty":174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6616,"num_write_kb":217280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"3,5,7","objects":174}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":350,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.428150+0000","last_change":"2023-06-10T21:49:18.411617+0000","last_active":"2023-06-10T21:50:16.428150+0000","last_peered":"2023-06-10T21:50:16.428150+0000","last_clean":"2023-06-10T21:50:16.428150+0000","last_became_active":"2023-06-10T21:49:18.411293+0000","last_became_peered":"2023-06-10T21:49:18.411293+0000","last_unstale":"2023-06-10T21:50:16.428150+0000","last_undegraded":"2023-06-10T21:50:16.428150+0000","last_fullsized":"2023-06-10T21:50:16.428150+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483379+0000","last_change":"2023-06-10T21:49:18.400628+0000","last_active":"2023-06-10T21:50:23.483379+0000","last_peered":"2023-06-10T21:50:23.483379+0000","last_clean":"2023-06-10T21:50:23.483379+0000","last_became_active":"2023-06-10T21:49:18.400273+0000","last_became_peered":"2023-06-10T21:49:18.400273+0000","last_unstale":"2023-06-10T21:50:23.483379+0000","last_undegraded":"2023-06-10T21:50:23.483379+0000","last_fullsized":"2023-06-10T21:50:23.483379+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:36:43.396235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.8","version":"477'12694","reported_seq":14848,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.568815+0000","last_change":"2023-06-10T21:50:05.546594+0000","last_active":"2023-06-10T21:50:26.568815+0000","last_peered":"2023-06-10T21:50:26.568815+0000","last_clean":"2023-06-10T21:47:17.280051+0000","last_became_active":"2023-06-10T21:50:05.520421+0000","last_became_peered":"2023-06-10T21:50:05.520421+0000","last_unstale":"2023-06-10T21:50:26.568815+0000","last_undegraded":"2023-06-10T21:50:26.568815+0000","last_fullsized":"2023-06-10T21:50:26.568815+0000","mapping_epoch":439,"log_start":"436'5361","ondisk_log_start":"436'5361","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7333,"log_dups_size":0,"ondisk_log_size":7333,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":45,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":216480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":516,"num_bytes_recovered":33783808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":199}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":49,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.445833+0000","last_change":"2023-06-10T21:47:18.325919+0000","last_active":"2023-06-10T21:50:16.445833+0000","last_peered":"2023-06-10T21:50:16.445833+0000","last_clean":"2023-06-10T21:50:16.445833+0000","last_became_active":"2023-06-10T21:47:18.325473+0000","last_became_peered":"2023-06-10T21:47:18.325473+0000","last_unstale":"2023-06-10T21:50:16.445833+0000","last_undegraded":"2023-06-10T21:50:16.445833+0000","last_fullsized":"2023-06-10T21:50:16.445833+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:29:40.800760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":119,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-06-10T21:50:24.481474+0000","last_change":"2023-06-10T21:49:18.414653+0000","last_active":"2023-06-10T21:50:24.481474+0000","last_peered":"2023-06-10T21:50:24.481474+0000","last_clean":"2023-06-10T21:50:24.481474+0000","last_became_active":"2023-06-10T21:49:18.414276+0000","last_became_peered":"2023-06-10T21:49:18.414276+0000","last_unstale":"2023-06-10T21:50:24.481474+0000","last_undegraded":"2023-06-10T21:50:24.481474+0000","last_fullsized":"2023-06-10T21:50:24.481474+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:25:00.827868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.437795+0000","last_change":"2023-06-10T21:47:33.366140+0000","last_active":"2023-06-10T21:50:16.437795+0000","last_peered":"2023-06-10T21:50:16.437795+0000","last_clean":"2023-06-10T21:50:16.437795+0000","last_became_active":"2023-06-10T21:47:33.365711+0000","last_became_peered":"2023-06-10T21:47:33.365711+0000","last_unstale":"2023-06-10T21:50:16.437795+0000","last_undegraded":"2023-06-10T21:50:16.437795+0000","last_fullsized":"2023-06-10T21:50:16.437795+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:02:40.888164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.e","version":"477'13451","reported_seq":13555,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.593977+0000","last_change":"2023-06-10T21:49:49.817038+0000","last_active":"2023-06-10T21:50:26.593977+0000","last_peered":"2023-06-10T21:50:26.593977+0000","last_clean":"2023-06-10T21:47:17.288812+0000","last_became_active":"2023-06-10T21:48:53.526973+0000","last_became_peered":"2023-06-10T21:48:53.526973+0000","last_unstale":"2023-06-10T21:50:26.593977+0000","last_undegraded":"2023-06-10T21:50:26.593977+0000","last_fullsized":"2023-06-10T21:50:26.593977+0000","mapping_epoch":454,"log_start":"445'7366","ondisk_log_start":"445'7366","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6085,"log_dups_size":0,"ondisk_log_size":6085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:49:28.977308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":762,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13451,"num_write_kb":442624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":381}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435476+0000","last_change":"2023-06-10T21:49:18.401074+0000","last_active":"2023-06-10T21:50:16.435476+0000","last_peered":"2023-06-10T21:50:16.435476+0000","last_clean":"2023-06-10T21:50:16.435476+0000","last_became_active":"2023-06-10T21:49:18.400317+0000","last_became_peered":"2023-06-10T21:49:18.400317+0000","last_unstale":"2023-06-10T21:50:16.435476+0000","last_undegraded":"2023-06-10T21:50:16.435476+0000","last_fullsized":"2023-06-10T21:50:16.435476+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.440434+0000","last_change":"2023-06-10T21:49:18.395157+0000","last_active":"2023-06-10T21:50:16.440434+0000","last_peered":"2023-06-10T21:50:16.440434+0000","last_clean":"2023-06-10T21:50:16.440434+0000","last_became_active":"2023-06-10T21:49:18.394783+0000","last_became_peered":"2023-06-10T21:49:18.394783+0000","last_unstale":"2023-06-10T21:50:16.440434+0000","last_undegraded":"2023-06-10T21:50:16.440434+0000","last_fullsized":"2023-06-10T21:50:16.440434+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:19:02.332466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":353,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.493386+0000","last_change":"2023-06-10T21:48:53.384662+0000","last_active":"2023-06-10T21:50:23.493386+0000","last_peered":"2023-06-10T21:50:23.493386+0000","last_clean":"2023-06-10T21:50:23.493386+0000","last_became_active":"2023-06-10T21:48:53.383318+0000","last_became_peered":"2023-06-10T21:48:53.383318+0000","last_unstale":"2023-06-10T21:50:23.493386+0000","last_undegraded":"2023-06-10T21:50:23.493386+0000","last_fullsized":"2023-06-10T21:50:23.493386+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:01:32.045450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"477'13184","reported_seq":14726,"reported_epoch":477,"state":"active+clean+remapped","last_fresh":"2023-06-10T21:50:26.545649+0000","last_change":"2023-06-10T21:50:26.444870+0000","last_active":"2023-06-10T21:50:26.545649+0000","last_peered":"2023-06-10T21:50:26.545649+0000","last_clean":"2023-06-10T21:50:26.545649+0000","last_became_active":"2023-06-10T21:49:18.533901+0000","last_became_peered":"2023-06-10T21:49:18.533901+0000","last_unstale":"2023-06-10T21:50:26.545649+0000","last_undegraded":"2023-06-10T21:50:26.545649+0000","last_fullsized":"2023-06-10T21:50:26.545649+0000","mapping_epoch":460,"log_start":"445'7138","ondisk_log_start":"445'7138","created":420,"last_epoch_clean":477,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:10:01.366032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20643864,"num_objects":316,"num_object_clones":0,"num_object_copies":948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":316,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13184,"num_write_kb":431937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":529,"num_bytes_recovered":34537496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[2,5,7],"avail_no_missing":["2","0","5","6","7"],"object_location_counts":[{"shards":"2,5,7","objects":316}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":373,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.493097+0000","last_change":"2023-06-10T21:47:18.337074+0000","last_active":"2023-06-10T21:50:23.493097+0000","last_peered":"2023-06-10T21:50:23.493097+0000","last_clean":"2023-06-10T21:50:23.493097+0000","last_became_active":"2023-06-10T21:47:18.336698+0000","last_became_peered":"2023-06-10T21:47:18.336698+0000","last_unstale":"2023-06-10T21:50:23.493097+0000","last_undegraded":"2023-06-10T21:50:23.493097+0000","last_fullsized":"2023-06-10T21:50:23.493097+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:58.371008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":387,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.490727+0000","last_change":"2023-06-10T21:48:53.380090+0000","last_active":"2023-06-10T21:50:23.490727+0000","last_peered":"2023-06-10T21:50:23.490727+0000","last_clean":"2023-06-10T21:50:23.490727+0000","last_became_active":"2023-06-10T21:48:53.379772+0000","last_became_peered":"2023-06-10T21:48:53.379772+0000","last_unstale":"2023-06-10T21:50:23.490727+0000","last_undegraded":"2023-06-10T21:50:23.490727+0000","last_fullsized":"2023-06-10T21:50:23.490727+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:22:14.985441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441537+0000","last_change":"2023-06-10T21:49:18.398464+0000","last_active":"2023-06-10T21:50:16.441537+0000","last_peered":"2023-06-10T21:50:16.441537+0000","last_clean":"2023-06-10T21:50:16.441537+0000","last_became_active":"2023-06-10T21:49:18.397993+0000","last_became_peered":"2023-06-10T21:49:18.397993+0000","last_unstale":"2023-06-10T21:50:16.441537+0000","last_undegraded":"2023-06-10T21:50:16.441537+0000","last_fullsized":"2023-06-10T21:50:16.441537+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:14:41.415970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.f","version":"476'13020","reported_seq":18191,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:24.361092+0000","last_change":"2023-06-10T21:49:30.458667+0000","last_active":"2023-06-10T21:50:24.361092+0000","last_peered":"2023-06-10T21:50:24.361092+0000","last_clean":"2023-06-10T21:50:24.361092+0000","last_became_active":"2023-06-10T21:49:30.432070+0000","last_became_peered":"2023-06-10T21:49:30.432070+0000","last_unstale":"2023-06-10T21:50:24.361092+0000","last_undegraded":"2023-06-10T21:50:24.361092+0000","last_fullsized":"2023-06-10T21:50:24.361092+0000","mapping_epoch":464,"log_start":"447'8187","ondisk_log_start":"447'8187","created":420,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4833,"log_dups_size":0,"ondisk_log_size":4833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13020,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2542,"num_bytes_recovered":166592512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":444,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-06-10T21:50:24.491779+0000","last_change":"2023-06-10T21:49:18.400868+0000","last_active":"2023-06-10T21:50:24.491779+0000","last_peered":"2023-06-10T21:50:24.491779+0000","last_clean":"2023-06-10T21:50:24.491779+0000","last_became_active":"2023-06-10T21:49:18.400214+0000","last_became_peered":"2023-06-10T21:49:18.400214+0000","last_unstale":"2023-06-10T21:50:24.491779+0000","last_undegraded":"2023-06-10T21:50:24.491779+0000","last_fullsized":"2023-06-10T21:50:24.491779+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:12:40.417709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":722,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.428043+0000","last_change":"2023-06-10T21:48:56.439594+0000","last_active":"2023-06-10T21:50:16.428043+0000","last_peered":"2023-06-10T21:50:16.428043+0000","last_clean":"2023-06-10T21:50:16.428043+0000","last_became_active":"2023-06-10T21:48:56.439122+0000","last_became_peered":"2023-06-10T21:48:56.439122+0000","last_unstale":"2023-06-10T21:50:16.428043+0000","last_undegraded":"2023-06-10T21:50:16.428043+0000","last_fullsized":"2023-06-10T21:50:16.428043+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":166,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482671+0000","last_change":"2023-06-10T21:49:18.398769+0000","last_active":"2023-06-10T21:50:23.482671+0000","last_peered":"2023-06-10T21:50:23.482671+0000","last_clean":"2023-06-10T21:50:23.482671+0000","last_became_active":"2023-06-10T21:49:18.398564+0000","last_became_peered":"2023-06-10T21:49:18.398564+0000","last_unstale":"2023-06-10T21:50:23.482671+0000","last_undegraded":"2023-06-10T21:50:23.482671+0000","last_fullsized":"2023-06-10T21:50:23.482671+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:25:52.639389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.436107+0000","last_change":"2023-06-10T21:48:53.379480+0000","last_active":"2023-06-10T21:50:16.436107+0000","last_peered":"2023-06-10T21:50:16.436107+0000","last_clean":"2023-06-10T21:50:16.436107+0000","last_became_active":"2023-06-10T21:48:53.379153+0000","last_became_peered":"2023-06-10T21:48:53.379153+0000","last_unstale":"2023-06-10T21:50:16.436107+0000","last_undegraded":"2023-06-10T21:50:16.436107+0000","last_fullsized":"2023-06-10T21:50:16.436107+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"477'13080","reported_seq":13189,"reported_epoch":477,"state":"active+remapped","last_fresh":"2023-06-10T21:50:26.580084+0000","last_change":"2023-06-10T21:50:05.549919+0000","last_active":"2023-06-10T21:50:26.580084+0000","last_peered":"2023-06-10T21:50:26.580084+0000","last_clean":"2023-06-10T21:47:17.279622+0000","last_became_active":"2023-06-10T21:48:54.513849+0000","last_became_peered":"2023-06-10T21:48:54.513849+0000","last_unstale":"2023-06-10T21:50:26.580084+0000","last_undegraded":"2023-06-10T21:50:26.580084+0000","last_fullsized":"2023-06-10T21:50:26.580084+0000","mapping_epoch":455,"log_start":"444'7059","ondisk_log_start":"444'7059","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:51:05.743014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22020096,"num_objects":336,"num_object_clones":0,"num_object_copies":1008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":672,"num_objects_unfound":0,"num_objects_dirty":336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13080,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[2,5,7],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":336}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":537,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.428059+0000","last_change":"2023-06-10T21:47:18.326869+0000","last_active":"2023-06-10T21:50:16.428059+0000","last_peered":"2023-06-10T21:50:16.428059+0000","last_clean":"2023-06-10T21:50:16.428059+0000","last_became_active":"2023-06-10T21:47:18.326544+0000","last_became_peered":"2023-06-10T21:47:18.326544+0000","last_unstale":"2023-06-10T21:50:16.428059+0000","last_undegraded":"2023-06-10T21:50:16.428059+0000","last_fullsized":"2023-06-10T21:50:16.428059+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.436739+0000","last_change":"2023-06-10T21:49:18.398644+0000","last_active":"2023-06-10T21:50:16.436739+0000","last_peered":"2023-06-10T21:50:16.436739+0000","last_clean":"2023-06-10T21:50:16.436739+0000","last_became_active":"2023-06-10T21:49:18.398328+0000","last_became_peered":"2023-06-10T21:49:18.398328+0000","last_unstale":"2023-06-10T21:50:16.436739+0000","last_undegraded":"2023-06-10T21:50:16.436739+0000","last_fullsized":"2023-06-10T21:50:16.436739+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:44:21.732943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":49,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441170+0000","last_change":"2023-06-10T21:47:18.334732+0000","last_active":"2023-06-10T21:50:16.441170+0000","last_peered":"2023-06-10T21:50:16.441170+0000","last_clean":"2023-06-10T21:50:16.441170+0000","last_became_active":"2023-06-10T21:47:18.334458+0000","last_became_peered":"2023-06-10T21:47:18.334458+0000","last_unstale":"2023-06-10T21:50:16.441170+0000","last_undegraded":"2023-06-10T21:50:16.441170+0000","last_fullsized":"2023-06-10T21:50:16.441170+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:24:31.688908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"477'12724","reported_seq":19107,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.616956+0000","last_change":"2023-06-10T21:50:05.732394+0000","last_active":"2023-06-10T21:50:26.616956+0000","last_peered":"2023-06-10T21:50:26.616956+0000","last_clean":"2023-06-10T21:47:17.293084+0000","last_became_active":"2023-06-10T21:50:05.706032+0000","last_became_peered":"2023-06-10T21:50:05.706032+0000","last_unstale":"2023-06-10T21:50:26.616956+0000","last_undegraded":"2023-06-10T21:50:26.616956+0000","last_fullsized":"2023-06-10T21:50:26.616956+0000","mapping_epoch":461,"log_start":"442'6918","ondisk_log_start":"442'6918","created":420,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534336,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":176,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6600,"num_write_kb":216832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1567,"num_bytes_recovered":102662144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,2,5],"avail_no_missing":["3","0","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":176}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":534,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.439519+0000","last_change":"2023-06-10T21:48:53.377036+0000","last_active":"2023-06-10T21:50:16.439519+0000","last_peered":"2023-06-10T21:50:16.439519+0000","last_clean":"2023-06-10T21:50:16.439519+0000","last_became_active":"2023-06-10T21:48:53.375990+0000","last_became_peered":"2023-06-10T21:48:53.375990+0000","last_unstale":"2023-06-10T21:50:16.439519+0000","last_undegraded":"2023-06-10T21:50:16.439519+0000","last_fullsized":"2023-06-10T21:50:16.439519+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":377,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.493477+0000","last_change":"2023-06-10T21:49:18.406396+0000","last_active":"2023-06-10T21:50:23.493477+0000","last_peered":"2023-06-10T21:50:23.493477+0000","last_clean":"2023-06-10T21:50:23.493477+0000","last_became_active":"2023-06-10T21:49:18.406063+0000","last_became_peered":"2023-06-10T21:49:18.406063+0000","last_unstale":"2023-06-10T21:50:23.493477+0000","last_undegraded":"2023-06-10T21:50:23.493477+0000","last_fullsized":"2023-06-10T21:50:23.493477+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:06:02.929666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.440328+0000","last_change":"2023-06-10T21:49:18.403449+0000","last_active":"2023-06-10T21:50:16.440328+0000","last_peered":"2023-06-10T21:50:16.440328+0000","last_clean":"2023-06-10T21:50:16.440328+0000","last_became_active":"2023-06-10T21:49:18.402496+0000","last_became_peered":"2023-06-10T21:49:18.402496+0000","last_unstale":"2023-06-10T21:50:16.440328+0000","last_undegraded":"2023-06-10T21:50:16.440328+0000","last_fullsized":"2023-06-10T21:50:16.440328+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:34:50.868280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"477'13081","reported_seq":17441,"reported_epoch":477,"state":"active+clean","last_fresh":"2023-06-10T21:50:26.649792+0000","last_change":"2023-06-10T21:49:47.425823+0000","last_active":"2023-06-10T21:50:26.649792+0000","last_peered":"2023-06-10T21:50:26.649792+0000","last_clean":"2023-06-10T21:50:26.649792+0000","last_became_active":"2023-06-10T21:49:47.396908+0000","last_became_peered":"2023-06-10T21:49:47.396908+0000","last_unstale":"2023-06-10T21:50:26.649792+0000","last_undegraded":"2023-06-10T21:50:26.649792+0000","last_fullsized":"2023-06-10T21:50:26.649792+0000","mapping_epoch":467,"log_start":"448'8529","ondisk_log_start":"448'8529","created":420,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":4552,"log_dups_size":0,"ondisk_log_size":4552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:42:49.372274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22347776,"num_objects":341,"num_object_clones":0,"num_object_copies":1023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13081,"num_write_kb":429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2125,"num_bytes_recovered":139132928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":502,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483319+0000","last_change":"2023-06-10T21:47:18.433743+0000","last_active":"2023-06-10T21:50:23.483319+0000","last_peered":"2023-06-10T21:50:23.483319+0000","last_clean":"2023-06-10T21:50:23.483319+0000","last_became_active":"2023-06-10T21:47:18.338606+0000","last_became_peered":"2023-06-10T21:47:18.338606+0000","last_unstale":"2023-06-10T21:50:23.483319+0000","last_undegraded":"2023-06-10T21:50:23.483319+0000","last_fullsized":"2023-06-10T21:50:23.483319+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:13:02.602599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.438346+0000","last_change":"2023-06-10T21:48:53.380616+0000","last_active":"2023-06-10T21:50:16.438346+0000","last_peered":"2023-06-10T21:50:16.438346+0000","last_clean":"2023-06-10T21:50:16.438346+0000","last_became_active":"2023-06-10T21:48:53.380316+0000","last_became_peered":"2023-06-10T21:48:53.380316+0000","last_unstale":"2023-06-10T21:50:16.438346+0000","last_undegraded":"2023-06-10T21:50:16.438346+0000","last_fullsized":"2023-06-10T21:50:16.438346+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":345,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.427939+0000","last_change":"2023-06-10T21:49:18.406601+0000","last_active":"2023-06-10T21:50:16.427939+0000","last_peered":"2023-06-10T21:50:16.427939+0000","last_clean":"2023-06-10T21:50:16.427939+0000","last_became_active":"2023-06-10T21:49:18.406228+0000","last_became_peered":"2023-06-10T21:49:18.406228+0000","last_unstale":"2023-06-10T21:50:16.427939+0000","last_undegraded":"2023-06-10T21:50:16.427939+0000","last_fullsized":"2023-06-10T21:50:16.427939+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.b","version":"477'12974","reported_seq":13720,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.580337+0000","last_change":"2023-06-10T21:50:05.544045+0000","last_active":"2023-06-10T21:50:26.580337+0000","last_peered":"2023-06-10T21:50:26.580337+0000","last_clean":"2023-06-10T21:47:17.285378+0000","last_became_active":"2023-06-10T21:48:53.660628+0000","last_became_peered":"2023-06-10T21:48:53.660628+0000","last_unstale":"2023-06-10T21:50:26.580337+0000","last_undegraded":"2023-06-10T21:50:26.580337+0000","last_fullsized":"2023-06-10T21:50:26.580337+0000","mapping_epoch":454,"log_start":"442'6921","ondisk_log_start":"442'6921","created":420,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:26:42.953015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21495808,"num_objects":328,"num_object_clones":0,"num_object_copies":984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":526,"num_objects_unfound":0,"num_objects_dirty":328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12974,"num_write_kb":425664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":218,"num_bytes_recovered":14221312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":328}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435934+0000","last_change":"2023-06-10T21:48:53.379918+0000","last_active":"2023-06-10T21:50:16.435934+0000","last_peered":"2023-06-10T21:50:16.435934+0000","last_clean":"2023-06-10T21:50:16.435934+0000","last_became_active":"2023-06-10T21:48:53.379608+0000","last_became_peered":"2023-06-10T21:48:53.379608+0000","last_unstale":"2023-06-10T21:50:16.435934+0000","last_undegraded":"2023-06-10T21:50:16.435934+0000","last_fullsized":"2023-06-10T21:50:16.435934+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":80,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441065+0000","last_change":"2023-06-10T21:48:53.400742+0000","last_active":"2023-06-10T21:50:16.441065+0000","last_peered":"2023-06-10T21:50:16.441065+0000","last_clean":"2023-06-10T21:50:16.441065+0000","last_became_active":"2023-06-10T21:48:53.381285+0000","last_became_peered":"2023-06-10T21:48:53.381285+0000","last_unstale":"2023-06-10T21:50:16.441065+0000","last_undegraded":"2023-06-10T21:50:16.441065+0000","last_fullsized":"2023-06-10T21:50:16.441065+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:05:45.459010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,3],"acting":[0,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":40,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.434715+0000","last_change":"2023-06-10T21:49:18.399985+0000","last_active":"2023-06-10T21:50:16.434715+0000","last_peered":"2023-06-10T21:50:16.434715+0000","last_clean":"2023-06-10T21:50:16.434715+0000","last_became_active":"2023-06-10T21:49:18.399526+0000","last_became_peered":"2023-06-10T21:49:18.399526+0000","last_unstale":"2023-06-10T21:50:16.434715+0000","last_undegraded":"2023-06-10T21:50:16.434715+0000","last_fullsized":"2023-06-10T21:50:16.434715+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.16","version":"477'12468","reported_seq":17560,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.584740+0000","last_change":"2023-06-10T21:50:07.477085+0000","last_active":"2023-06-10T21:50:26.584740+0000","last_peered":"2023-06-10T21:50:26.584740+0000","last_clean":"2023-06-10T21:47:17.257047+0000","last_became_active":"2023-06-10T21:50:07.436797+0000","last_became_peered":"2023-06-10T21:50:07.436797+0000","last_unstale":"2023-06-10T21:50:26.584740+0000","last_undegraded":"2023-06-10T21:50:26.584740+0000","last_fullsized":"2023-06-10T21:50:26.584740+0000","mapping_epoch":473,"log_start":"435'5200","ondisk_log_start":"435'5200","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7268,"log_dups_size":0,"ondisk_log_size":7268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534336,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6444,"num_write_kb":211840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1251,"num_bytes_recovered":81985536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[2,3,5],"avail_no_missing":["2","0","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":176}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":49,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.440235+0000","last_change":"2023-06-10T21:47:18.324336+0000","last_active":"2023-06-10T21:50:16.440235+0000","last_peered":"2023-06-10T21:50:16.440235+0000","last_clean":"2023-06-10T21:50:16.440235+0000","last_became_active":"2023-06-10T21:47:18.321834+0000","last_became_peered":"2023-06-10T21:47:18.321834+0000","last_unstale":"2023-06-10T21:50:16.440235+0000","last_undegraded":"2023-06-10T21:50:16.440235+0000","last_fullsized":"2023-06-10T21:50:16.440235+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:34:40.514667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":391,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482462+0000","last_change":"2023-06-10T21:49:18.398119+0000","last_active":"2023-06-10T21:50:23.482462+0000","last_peered":"2023-06-10T21:50:23.482462+0000","last_clean":"2023-06-10T21:50:23.482462+0000","last_became_active":"2023-06-10T21:49:18.397781+0000","last_became_peered":"2023-06-10T21:49:18.397781+0000","last_unstale":"2023-06-10T21:50:23.482462+0000","last_undegraded":"2023-06-10T21:50:23.482462+0000","last_fullsized":"2023-06-10T21:50:23.482462+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:27:32.540626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.11","version":"477'12952","reported_seq":13134,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.666084+0000","last_change":"2023-06-10T21:50:07.504883+0000","last_active":"2023-06-10T21:50:26.666084+0000","last_peered":"2023-06-10T21:50:26.666084+0000","last_clean":"2023-06-10T21:47:32.299728+0000","last_became_active":"2023-06-10T21:50:07.465236+0000","last_became_peered":"2023-06-10T21:50:07.465236+0000","last_unstale":"2023-06-10T21:50:26.666084+0000","last_undegraded":"2023-06-10T21:50:26.666084+0000","last_fullsized":"2023-06-10T21:50:26.666084+0000","mapping_epoch":473,"log_start":"442'7150","ondisk_log_start":"442'7150","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9928704,"num_objects":152,"num_object_clones":0,"num_object_copies":456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":144,"num_objects_unfound":0,"num_objects_dirty":152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6715,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1310720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":152}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":48,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435779+0000","last_change":"2023-06-10T21:49:18.431026+0000","last_active":"2023-06-10T21:50:16.435779+0000","last_peered":"2023-06-10T21:50:16.435779+0000","last_clean":"2023-06-10T21:50:16.435779+0000","last_became_active":"2023-06-10T21:49:18.403054+0000","last_became_peered":"2023-06-10T21:49:18.403054+0000","last_unstale":"2023-06-10T21:50:16.435779+0000","last_undegraded":"2023-06-10T21:50:16.435779+0000","last_fullsized":"2023-06-10T21:50:16.435779+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":383,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.481967+0000","last_change":"2023-06-10T21:39:50.285771+0000","last_active":"2023-06-10T21:50:23.481967+0000","last_peered":"2023-06-10T21:50:23.481967+0000","last_clean":"2023-06-10T21:50:23.481967+0000","last_became_active":"2023-06-10T21:39:50.272210+0000","last_became_peered":"2023-06-10T21:39:50.272210+0000","last_unstale":"2023-06-10T21:50:23.481967+0000","last_undegraded":"2023-06-10T21:50:23.481967+0000","last_fullsized":"2023-06-10T21:50:23.481967+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:06:02.703467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.10","version":"477'12686","reported_seq":12801,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.602110+0000","last_change":"2023-06-10T21:50:07.479297+0000","last_active":"2023-06-10T21:50:26.602110+0000","last_peered":"2023-06-10T21:50:26.602110+0000","last_clean":"2023-06-10T21:47:17.279925+0000","last_became_active":"2023-06-10T21:50:07.431643+0000","last_became_peered":"2023-06-10T21:50:07.431643+0000","last_unstale":"2023-06-10T21:50:26.602110+0000","last_undegraded":"2023-06-10T21:50:26.602110+0000","last_fullsized":"2023-06-10T21:50:26.602110+0000","mapping_epoch":473,"log_start":"436'5420","ondisk_log_start":"436'5420","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":7266,"log_dups_size":0,"ondisk_log_size":7266,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9666560,"num_objects":148,"num_object_clones":0,"num_object_copies":444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":296,"num_objects_unfound":0,"num_objects_dirty":148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":214816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,3,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":148}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":48,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.436160+0000","last_change":"2023-06-10T21:49:18.410072+0000","last_active":"2023-06-10T21:50:16.436160+0000","last_peered":"2023-06-10T21:50:16.436160+0000","last_clean":"2023-06-10T21:50:16.436160+0000","last_became_active":"2023-06-10T21:49:18.398826+0000","last_became_peered":"2023-06-10T21:49:18.398826+0000","last_unstale":"2023-06-10T21:50:16.436160+0000","last_undegraded":"2023-06-10T21:50:16.436160+0000","last_fullsized":"2023-06-10T21:50:16.436160+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":463,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482802+0000","last_change":"2023-06-10T21:47:18.335340+0000","last_active":"2023-06-10T21:50:23.482802+0000","last_peered":"2023-06-10T21:50:23.482802+0000","last_clean":"2023-06-10T21:50:23.482802+0000","last_became_active":"2023-06-10T21:47:18.335048+0000","last_became_peered":"2023-06-10T21:47:18.335048+0000","last_unstale":"2023-06-10T21:50:23.482802+0000","last_undegraded":"2023-06-10T21:50:23.482802+0000","last_fullsized":"2023-06-10T21:50:23.482802+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:11:27.198035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.13","version":"476'12627","reported_seq":28929,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:24.008824+0000","last_change":"2023-06-10T21:50:05.755773+0000","last_active":"2023-06-10T21:50:24.008824+0000","last_peered":"2023-06-10T21:50:24.008824+0000","last_clean":"2023-06-10T21:50:24.008824+0000","last_became_active":"2023-06-10T21:50:05.730763+0000","last_became_peered":"2023-06-10T21:50:05.730763+0000","last_unstale":"2023-06-10T21:50:24.008824+0000","last_undegraded":"2023-06-10T21:50:24.008824+0000","last_fullsized":"2023-06-10T21:50:24.008824+0000","mapping_epoch":471,"log_start":"445'7341","ondisk_log_start":"445'7341","created":471,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5286,"log_dups_size":0,"ondisk_log_size":5286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15695872,"num_objects":240,"num_object_clones":0,"num_object_copies":720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6493,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4042,"num_bytes_recovered":264896512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.482128+0000","last_change":"2023-06-10T21:48:53.450892+0000","last_active":"2023-06-10T21:50:23.482128+0000","last_peered":"2023-06-10T21:50:23.482128+0000","last_clean":"2023-06-10T21:50:23.482128+0000","last_became_active":"2023-06-10T21:48:53.450548+0000","last_became_peered":"2023-06-10T21:48:53.450548+0000","last_unstale":"2023-06-10T21:50:23.482128+0000","last_undegraded":"2023-06-10T21:50:23.482128+0000","last_fullsized":"2023-06-10T21:50:23.482128+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:36:57.682749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":157,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-06-10T21:50:23.483682+0000","last_change":"2023-06-10T21:47:18.344279+0000","last_active":"2023-06-10T21:50:23.483682+0000","last_peered":"2023-06-10T21:50:23.483682+0000","last_clean":"2023-06-10T21:50:23.483682+0000","last_became_active":"2023-06-10T21:47:18.343646+0000","last_became_peered":"2023-06-10T21:47:18.343646+0000","last_unstale":"2023-06-10T21:50:23.483682+0000","last_undegraded":"2023-06-10T21:50:23.483682+0000","last_fullsized":"2023-06-10T21:50:23.483682+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:06:43.284066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"477'12679","reported_seq":17060,"reported_epoch":477,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-10T21:50:26.609415+0000","last_change":"2023-06-10T21:50:07.473239+0000","last_active":"2023-06-10T21:50:26.609415+0000","last_peered":"2023-06-10T21:50:26.609415+0000","last_clean":"2023-06-10T21:47:17.271926+0000","last_became_active":"2023-06-10T21:50:07.430860+0000","last_became_peered":"2023-06-10T21:50:07.430860+0000","last_unstale":"2023-06-10T21:50:26.609415+0000","last_undegraded":"2023-06-10T21:50:26.609415+0000","last_fullsized":"2023-06-10T21:50:26.609415+0000","mapping_epoch":473,"log_start":"442'6923","ondisk_log_start":"442'6923","created":471,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T21:45:43.457627+0000","last_clean_scrub_stamp":"2023-06-10T21:45:43.457627+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11042816,"num_objects":169,"num_object_clones":0,"num_object_copies":507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":215488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":712,"num_bytes_recovered":46596096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[3,5,7],"avail_no_missing":["3","0","5","6","7"],"object_location_counts":[{"shards":"3,5,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":100,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.441007+0000","last_change":"2023-06-10T21:49:18.424381+0000","last_active":"2023-06-10T21:50:16.441007+0000","last_peered":"2023-06-10T21:50:16.441007+0000","last_clean":"2023-06-10T21:50:16.441007+0000","last_became_active":"2023-06-10T21:49:18.403001+0000","last_became_peered":"2023-06-10T21:49:18.403001+0000","last_unstale":"2023-06-10T21:50:16.441007+0000","last_undegraded":"2023-06-10T21:50:16.441007+0000","last_fullsized":"2023-06-10T21:50:16.441007+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:40:47.222958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-06-10T21:50:16.435031+0000","last_change":"2023-06-10T21:48:53.382733+0000","last_active":"2023-06-10T21:50:16.435031+0000","last_peered":"2023-06-10T21:50:16.435031+0000","last_clean":"2023-06-10T21:50:16.435031+0000","last_became_active":"2023-06-10T21:48:53.382249+0000","last_became_peered":"2023-06-10T21:48:53.382249+0000","last_unstale":"2023-06-10T21:50:16.435031+0000","last_undegraded":"2023-06-10T21:50:16.435031+0000","last_fullsized":"2023-06-10T21:50:16.435031+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":375652376,"num_objects":5733,"num_object_clones":0,"num_object_copies":17199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4151,"num_objects_unfound":0,"num_objects_dirty":5733,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209975,"num_write_kb":6902593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23674,"num_bytes_recovered":1550450712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1804808192,"data_stored":1804795976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160176,"ondisk_log_size":160176,"up":78,"acting":78,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":115,"num_bytes_recovered":28260,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53512,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430840,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409224,"kb_used_data":306496,"kb_used_omap":10,"kb_used_meta":102709,"kb_avail":93335928,"statfs":{"total":95995035648,"available":95575990272,"internally_reserved":0,"allocated":313851904,"data_stored":313076217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":105174077},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:50:07 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":3.0990000000000002,"15min":2.1549999999999998},"min":{"1min":0.27100000000000002,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":30.902999999999999,"5min":170.05199999999999,"15min":170.05199999999999},"last":1.165},{"interface":"front","average":{"1min":3.6360000000000001,"5min":4.0140000000000002,"15min":2.758},"min":{"1min":0.318,"5min":0.246,"15min":0.246},"max":{"1min":31.236000000000001,"5min":170.126,"15min":170.126},"last":7.79}]},{"osd":1,"last update":"Sat Jun 10 21:49:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7999999999999998,"5min":1.411,"15min":2.0219999999999998},"min":{"1min":0.311,"5min":0.27800000000000002,"15min":0.25600000000000001},"max":{"1min":13.180999999999999,"5min":13.180999999999999,"15min":60.863},"last":1.1859999999999999},{"interface":"front","average":{"1min":2.109,"5min":1.6220000000000001,"15min":2.3399999999999999},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.28599999999999998},"max":{"1min":7.3280000000000003,"5min":8.8680000000000003,"15min":61.148000000000003},"last":0.51000000000000001}]},{"osd":2,"last update":"Sat Jun 10 21:49:49 2023","interfaces":[{"interface":"back","average":{"1min":3.786,"5min":9.8930000000000007,"15min":6.1239999999999997},"min":{"1min":0.46700000000000003,"5min":0.309,"15min":0.27600000000000002},"max":{"1min":8.1489999999999991,"5min":218.90600000000001,"15min":218.90600000000001},"last":1.1160000000000001},{"interface":"front","average":{"1min":2.722,"5min":9.7989999999999995,"15min":6.1059999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":12.385999999999999,"5min":218.851,"15min":218.851},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:49:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":5.9729999999999999,"15min":5.4299999999999997},"min":{"1min":0.34899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.2350000000000003,"5min":218.35300000000001,"15min":218.35300000000001},"last":4.4279999999999999},{"interface":"front","average":{"1min":3.4089999999999998,"5min":6.6500000000000004,"15min":5.5060000000000002},"min":{"1min":0.34399999999999997,"5min":0.33300000000000002,"15min":0.308},"max":{"1min":11.409000000000001,"5min":218.74600000000001,"15min":218.74600000000001},"last":7.6349999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.1479999999999999,"15min":0.82699999999999996},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.253},"max":{"1min":3.4609999999999999,"5min":4.8479999999999999,"15min":13.429},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.268,"5min":1.355,"15min":0.98799999999999999},"min":{"1min":0.34000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.532,"5min":12.279999999999999,"15min":22.77},"last":1.0800000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0740000000000001,"15min":7.8140000000000001},"min":{"1min":0.32800000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.5700000000000003,"5min":5.5700000000000003,"15min":131.619},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.4249999999999998,"5min":1.633,"15min":7.8789999999999996},"min":{"1min":0.33500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":18.277999999999999,"5min":18.277999999999999,"15min":134.928},"last":4.5449999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.095,"15min":0.84399999999999997},"min":{"1min":0.433,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":6.3029999999999999,"5min":7.9660000000000002,"15min":10.890000000000001},"last":4.3899999999999997},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.363,"15min":0.95799999999999996},"min":{"1min":0.41299999999999998,"5min":0.34100000000000003,"15min":0.30599999999999999},"max":{"1min":7.3949999999999996,"5min":8.1020000000000003,"15min":8.1020000000000003},"last":0.93400000000000005}]}]},{"osd":6,"up_from":42,"seq":180388627068,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":693352,"kb_used_data":192160,"kb_used_omap":0,"kb_used_meta":501184,"kb_avail":93051800,"statfs":{"total":95995035648,"available":95285043200,"internally_reserved":0,"allocated":196771840,"data_stored":196018806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":513212416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:50:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8779999999999999,"5min":2.181,"15min":1.5720000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.415,"5min":107.86,"15min":107.86},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.5620000000000001,"5min":3.762,"15min":2.2559999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.5259999999999998,"5min":136.85900000000001,"15min":136.85900000000001},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:49:58 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.2330000000000001,"15min":1.885},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.245},"max":{"1min":6.3559999999999999,"5min":7.3819999999999997,"15min":80.210999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.6989999999999998,"5min":1.458,"15min":2.0950000000000002},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":79.894000000000005},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:50:08 2023","interfaces":[{"interface":"back","average":{"1min":16.369,"5min":12.162000000000001,"15min":7.1020000000000003},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":265.37099999999998,"5min":265.37099999999998,"15min":265.37099999999998},"last":1.3160000000000001},{"interface":"front","average":{"1min":16.515999999999998,"5min":12.313000000000001,"15min":7.194},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":265.32900000000001,"5min":265.32900000000001,"15min":265.32900000000001},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:49:32 2023","interfaces":[{"interface":"back","average":{"1min":27.033000000000001,"5min":12.029999999999999,"15min":7.3220000000000001},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":412.55200000000002,"5min":412.55200000000002,"15min":412.55200000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":27.202000000000002,"5min":11.993,"15min":7.1150000000000002},"min":{"1min":0.42799999999999999,"5min":0.39900000000000002,"15min":0.33400000000000002},"max":{"1min":412.41199999999998,"5min":412.41199999999998,"15min":412.41199999999998},"last":1.6919999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:50:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.0449999999999999,"15min":0.82099999999999995},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.27500000000000002},"max":{"1min":6.5640000000000001,"5min":7.5510000000000002,"15min":7.5510000000000002},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.8140000000000001,"5min":0.89300000000000002,"15min":0.85399999999999998},"min":{"1min":0.34799999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.8179999999999996,"5min":7.8179999999999996,"15min":11.068},"last":1.2869999999999999}]},{"osd":5,"last update":"Sat Jun 10 21:50:08 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":1.073,"15min":1.2090000000000001},"min":{"1min":0.27700000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.343,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.0980000000000001,"15min":1.6359999999999999},"min":{"1min":0.33800000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.3209999999999997,"5min":7.3209999999999997,"15min":7.3209999999999997},"last":1.256}]},{"osd":7,"last update":"Sat Jun 10 21:49:32 2023","interfaces":[{"interface":"back","average":{"1min":6.8140000000000001,"5min":3.9239999999999999,"15min":2.9700000000000002},"min":{"1min":0.34399999999999997,"5min":0.33600000000000002,"15min":0.316},"max":{"1min":97.108000000000004,"5min":189.042,"15min":189.042},"last":0.38500000000000001},{"interface":"front","average":{"1min":6.7699999999999996,"5min":4.2199999999999998,"15min":3.2120000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32000000000000001},"max":{"1min":97.299000000000007,"5min":190.83699999999999,"15min":190.83699999999999},"last":3.6000000000000001}]}]},{"osd":1,"up_from":0,"seq":450971566511,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":244,"seq":1047972020487,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975784,"kb_used_data":285536,"kb_used_omap":10,"kb_used_meta":690229,"kb_avail":92769368,"statfs":{"total":95995035648,"available":94995832832,"internally_reserved":0,"allocated":292388864,"data_stored":290816401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":706794548},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.4399999999999999,"15min":1.8540000000000001},"min":{"1min":0.246,"5min":0.215,"15min":0.189},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":27.759},"last":1.4019999999999999},{"interface":"front","average":{"1min":1.76,"5min":1.8320000000000001,"15min":2.581},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20300000000000001},"max":{"1min":11.337999999999999,"5min":11.337999999999999,"15min":31.419},"last":1.7829999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":11.997999999999999,"5min":9.3010000000000002,"15min":7.2539999999999996},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":156.31100000000001,"5min":156.31100000000001,"15min":156.31100000000001},"last":1.5760000000000001},{"interface":"front","average":{"1min":11.657999999999999,"5min":8.7460000000000004,"15min":6.734},"min":{"1min":0.311,"5min":0.25800000000000001,"15min":0.248},"max":{"1min":155.92099999999999,"5min":155.92099999999999,"15min":155.92099999999999},"last":1.4610000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:49:40 2023","interfaces":[{"interface":"back","average":{"1min":17.539999999999999,"5min":6.7839999999999998,"15min":6.1100000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.251},"max":{"1min":150.89699999999999,"5min":150.89699999999999,"15min":211.864},"last":0.52700000000000002},{"interface":"front","average":{"1min":18.173999999999999,"5min":6.5190000000000001,"15min":5.7400000000000002},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":150.97200000000001,"5min":150.97200000000001,"15min":207.517},"last":1.4350000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":1.387,"5min":1.655,"15min":2.3500000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":5.71,"5min":8.6709999999999994,"15min":87.054000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.583,"5min":1.397,"15min":2.0499999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35099999999999998},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":93.649000000000001},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":2.3620000000000001,"15min":4.2859999999999996},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.1899999999999995,"5min":70.903999999999996,"15min":92.149000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.5369999999999999,"15min":4.8680000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.9969999999999999,"5min":71.698999999999998,"15min":93.292000000000002},"last":0.64300000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":13.237,"5min":4.1340000000000003,"15min":3.2189999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":224.54599999999999,"5min":224.54599999999999,"15min":224.54599999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":13.845000000000001,"5min":4.0579999999999998,"15min":2.968},"min":{"1min":0.40500000000000003,"5min":0.377,"15min":0.377},"max":{"1min":224.56100000000001,"5min":224.56100000000001,"15min":224.56100000000001},"last":0.55800000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:50:16 2023","interfaces":[{"interface":"back","average":{"1min":25.242999999999999,"5min":8.5510000000000002,"15min":5.1840000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.34100000000000003},"max":{"1min":227.565,"5min":227.565,"15min":227.565},"last":0.50800000000000001},{"interface":"front","average":{"1min":24.539000000000001,"5min":9.0060000000000002,"15min":5.407},"min":{"1min":0.38700000000000001,"5min":0.377,"15min":0.36299999999999999},"max":{"1min":233.90100000000001,"5min":233.90100000000001,"15min":233.90100000000001},"last":1.3320000000000001}]}]},{"osd":2,"up_from":120,"seq":515396075899,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1488716,"kb_used_data":226500,"kb_used_omap":7,"kb_used_meta":1261944,"kb_avail":92256436,"statfs":{"total":95995035648,"available":94470590464,"internally_reserved":0,"allocated":231936000,"data_stored":230409400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1292230737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:50:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":2.9079999999999999,"15min":3.6779999999999999},"min":{"1min":0.23000000000000001,"5min":0.20899999999999999,"15min":0.19},"max":{"1min":9.7699999999999996,"5min":103.19,"15min":171.33699999999999},"last":1.373},{"interface":"front","average":{"1min":1.4490000000000001,"5min":2.4380000000000002,"15min":3.2989999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.056,"5min":103.125,"15min":171.56200000000001},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Jun 10 21:49:59 2023","interfaces":[{"interface":"back","average":{"1min":4.4500000000000002,"5min":2.5910000000000002,"15min":4.3300000000000001},"min":{"1min":0.26400000000000001,"5min":0.23599999999999999,"15min":0.217},"max":{"1min":29.315999999999999,"5min":29.315999999999999,"15min":171.286},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.0640000000000001,"15min":2.9790000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":8.4879999999999995,"5min":15.685,"15min":85.501999999999995},"last":1.407}]},{"osd":3,"last update":"Sat Jun 10 21:49:33 2023","interfaces":[{"interface":"back","average":{"1min":5.0730000000000004,"5min":4.9800000000000004,"15min":6.8029999999999999},"min":{"1min":0.32000000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":46.786000000000001,"5min":79.353999999999999,"15min":171.78299999999999},"last":1.448},{"interface":"front","average":{"1min":4.3310000000000004,"5min":4.4180000000000001,"15min":6.4960000000000004},"min":{"1min":0.26200000000000001,"5min":0.26000000000000001,"15min":0.24399999999999999},"max":{"1min":43.375,"5min":79.614000000000004,"15min":171.65100000000001},"last":0.61299999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:49:59 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.3600000000000001,"15min":2.1850000000000001},"min":{"1min":0.38800000000000001,"5min":0.30499999999999999,"15min":0.28100000000000003},"max":{"1min":3.7530000000000001,"5min":15.712999999999999,"15min":171.48699999999999},"last":2.2090000000000001},{"interface":"front","average":{"1min":4.117,"5min":2.4039999999999999,"15min":3.0840000000000001},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.312},"max":{"1min":29.164000000000001,"5min":29.164000000000001,"15min":171.18100000000001},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:50:10 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":7.7409999999999997,"15min":6.907},"min":{"1min":0.46999999999999997,"5min":0.311,"15min":0.311},"max":{"1min":5.2809999999999997,"5min":157.756,"15min":157.756},"last":2.7370000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":7.093,"15min":6.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.7989999999999999,"5min":157.59899999999999,"15min":157.59899999999999},"last":2.8620000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:49:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.6870000000000001,"15min":2.3740000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":8.6219999999999999,"5min":15.930999999999999,"15min":171.441},"last":2.6320000000000001},{"interface":"front","average":{"1min":3.9849999999999999,"5min":2.5630000000000002,"15min":3.2029999999999998},"min":{"1min":0.46700000000000003,"5min":0.39100000000000001,"15min":0.34399999999999997},"max":{"1min":13.997,"5min":15.91,"15min":171.24600000000001},"last":2.665}]},{"osd":7,"last update":"Sat Jun 10 21:49:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":5.1769999999999996,"15min":4.6420000000000003},"min":{"1min":0.49299999999999999,"5min":0.46600000000000003,"15min":0.34200000000000003},"max":{"1min":8.4649999999999999,"5min":135.86099999999999,"15min":219.131},"last":6.9450000000000003},{"interface":"front","average":{"1min":1.0880000000000001,"5min":4.6029999999999998,"15min":4.3970000000000002},"min":{"1min":0.40999999999999998,"5min":0.39700000000000002,"15min":0.32200000000000001},"max":{"1min":3.899,"5min":133.45400000000001,"15min":219.19},"last":4.8659999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824259,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357232,"kb_used_data":333160,"kb_used_omap":7,"kb_used_meta":1024056,"kb_avail":92387920,"statfs":{"total":95995035648,"available":94605230080,"internally_reserved":0,"allocated":341155840,"data_stored":339474680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":1048633363},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":13.359,"5min":4.1710000000000003,"15min":3.242},"min":{"1min":0.23799999999999999,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":236.97300000000001,"5min":236.97300000000001,"15min":236.97300000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":11.646000000000001,"5min":4.29,"15min":3.496},"min":{"1min":0.223,"5min":0.223,"15min":0.20799999999999999},"max":{"1min":237.375,"5min":237.375,"15min":237.375},"last":2.0419999999999998}]},{"osd":1,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":2.0819999999999999,"15min":2.6890000000000001},"min":{"1min":0.253,"5min":0.247,"15min":0.20499999999999999},"max":{"1min":14.896000000000001,"5min":18.116,"15min":94.766000000000005},"last":2.0920000000000001},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.1589999999999998,"15min":2.9119999999999999},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":16.562999999999999,"5min":50.264000000000003,"15min":94.691000000000003},"last":0.441}]},{"osd":2,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":8.6050000000000004,"5min":6.258,"15min":4.8479999999999999},"min":{"1min":0.29399999999999998,"5min":0.27500000000000002,"15min":0.23300000000000001},"max":{"1min":117.279,"5min":126.913,"15min":231.07900000000001},"last":4.7199999999999998},{"interface":"front","average":{"1min":10.301,"5min":6.5789999999999997,"15min":5.5709999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.22800000000000001},"max":{"1min":117.398,"5min":128.52199999999999,"15min":231.58600000000001},"last":2.9649999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.4209999999999998,"15min":1.752},"min":{"1min":0.317,"5min":0.317,"15min":0.28499999999999998},"max":{"1min":4.4809999999999999,"5min":50.295999999999999,"15min":50.295999999999999},"last":2.1629999999999998},{"interface":"front","average":{"1min":2.7509999999999999,"5min":2.548,"15min":2.1640000000000001},"min":{"1min":0.33700000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":14.938000000000001,"5min":50.381,"15min":50.381},"last":2.1360000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:50:13 2023","interfaces":[{"interface":"back","average":{"1min":3.819,"5min":2.27,"15min":1.4850000000000001},"min":{"1min":0.34899999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":16.251000000000001,"5min":50.399000000000001,"15min":50.399000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.1899999999999999,"5min":2.7440000000000002,"15min":2.6880000000000002},"min":{"1min":0.45200000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.2009999999999996,"5min":50.335000000000001,"15min":50.335000000000001},"last":2.1139999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":5.3090000000000002,"5min":3.1000000000000001,"15min":2.4039999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32300000000000001},"max":{"1min":62.098999999999997,"5min":62.098999999999997,"15min":62.098999999999997},"last":2.177},{"interface":"front","average":{"1min":5.6070000000000002,"5min":2.8359999999999999,"15min":2.0190000000000001},"min":{"1min":0.39300000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":62.139000000000003,"5min":62.139000000000003,"15min":62.139000000000003},"last":0.68600000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:49:43 2023","interfaces":[{"interface":"back","average":{"1min":5.7149999999999999,"5min":4.093,"15min":2.7850000000000001},"min":{"1min":0.48899999999999999,"5min":0.38400000000000001,"15min":0.33900000000000002},"max":{"1min":69.683000000000007,"5min":69.683000000000007,"15min":69.683000000000007},"last":2.2120000000000002},{"interface":"front","average":{"1min":5.9320000000000004,"5min":3.6619999999999999,"15min":2.3650000000000002},"min":{"1min":0.435,"5min":0.374,"15min":0.316},"max":{"1min":69.998999999999995,"5min":69.998999999999995,"15min":69.998999999999995},"last":0.73299999999999998}]}]},{"osd":4,"up_from":0,"seq":128849019524,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":0,"seq":1675037245530,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":288817152,"data_stored":288817152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":224006144,"data_stored":224002072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":334626816,"data_stored":334626816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":461377536,"data_stored":461373464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":188416000,"data_stored":188416000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":307564544,"data_stored":307560472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:50:30.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set unique_pool_9 pgp_num 26 2023-06-10T21:50:30.408 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:30.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:30.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:30.513 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:30.539 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:30.582 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:30.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: osdmap e480: 8 total, 8 up, 5 in 2023-06-10T21:50:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[186706]: pgmap v1754: 99 pgs: 1 active+remapped+backfilling, 1 peering, 17 active+remapped+backfill_wait, 1 active+remapped, 79 active+clean; 327 MiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 4151/15759 objects misplaced (26.341%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: osdmap e480: 8 total, 8 up, 5 in 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:30 smithi064 ceph-mon[180297]: pgmap v1754: 99 pgs: 1 active+remapped+backfilling, 1 peering, 17 active+remapped+backfill_wait, 1 active+remapped, 79 active+clean; 327 MiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 4151/15759 objects misplaced (26.341%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:50:30.689 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:30.715 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: osdmap e480: 8 total, 8 up, 5 in 2023-06-10T21:50:30.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.716 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: failed to encode map e480 with expected crc 2023-06-10T21:50:30.717 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:30 smithi152 ceph-mon[156819]: pgmap v1754: 99 pgs: 1 active+remapped+backfilling, 1 peering, 17 active+remapped+backfill_wait, 1 active+remapped, 79 active+clean; 327 MiB data, 4.5 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 4151/15759 objects misplaced (26.341%); 4.5 MiB/s, 71 objects/s recovering 2023-06-10T21:50:30.718 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:30.784 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:30.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:30.893 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:30.930 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:31.584 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:31.639 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:31.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:31.754 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:31.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: osdmap e481: 8 total, 8 up, 5 in 2023-06-10T21:50:31.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.786 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.787 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:31 smithi152 ceph-mon[156819]: osdmap e482: 8 total, 8 up, 5 in 2023-06-10T21:50:31.787 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:50:31.788 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:31.816 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:50:31.845 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:50:31.850 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:31.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:31.880 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:50:31.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: osdmap e481: 8 total, 8 up, 5 in 2023-06-10T21:50:31.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[180297]: osdmap e482: 8 total, 8 up, 5 in 2023-06-10T21:50:31.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: osdmap e481: 8 total, 8 up, 5 in 2023-06-10T21:50:31.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: failed to encode map e481 with expected crc 2023-06-10T21:50:31.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:31 smithi064 ceph-mon[186706]: osdmap e482: 8 total, 8 up, 5 in 2023-06-10T21:50:31.911 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:50:31.941 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:50:31.956 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:31.969 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:50:31.991 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.994 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:50:31.995 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:32.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/782098863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:32.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: pgmap v1757: 99 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 1 peering, 13 active+remapped+backfill_wait, 1 active+remapped, 81 active+clean; 256 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 410 op/s; 2181/12348 objects misplaced (17.663%); 10 MiB/s, 168 objects/s recovering 2023-06-10T21:50:32.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[180297]: osdmap e483: 8 total, 8 up, 5 in 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/782098863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: pgmap v1757: 99 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 1 peering, 13 active+remapped+backfill_wait, 1 active+remapped, 81 active+clean; 256 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 410 op/s; 2181/12348 objects misplaced (17.663%); 10 MiB/s, 168 objects/s recovering 2023-06-10T21:50:32.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:32 smithi064 ceph-mon[186706]: osdmap e483: 8 total, 8 up, 5 in 2023-06-10T21:50:32.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: failed to encode map e482 with expected crc 2023-06-10T21:50:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/782098863' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: pgmap v1757: 99 pgs: 1 active+clean+remapped, 2 active+remapped+backfilling, 1 peering, 13 active+remapped+backfill_wait, 1 active+remapped, 81 active+clean; 256 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 410 op/s; 2181/12348 objects misplaced (17.663%); 10 MiB/s, 168 objects/s recovering 2023-06-10T21:50:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:32 smithi152 ceph-mon[156819]: osdmap e483: 8 total, 8 up, 5 in 2023-06-10T21:50:33.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:32] "GET /metrics HTTP/1.1" 200 224267 "" "Prometheus/2.33.4" 2023-06-10T21:50:33.204 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-10T21:50:33.440 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:33.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:33.489 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: failed to encode map e482 with expected crc 2023-06-10T21:50:33.489 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.489 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.489 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:33.490 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:33.490 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[180297]: osdmap e484: 8 total, 8 up, 5 in 2023-06-10T21:50:33.490 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: failed to encode map e482 with expected crc 2023-06-10T21:50:33.490 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.490 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.491 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:33.491 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:33.491 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:33 smithi064 ceph-mon[186706]: osdmap e484: 8 total, 8 up, 5 in 2023-06-10T21:50:33.552 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:33.611 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: failed to encode map e482 with expected crc 2023-06-10T21:50:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: failed to encode map e483 with expected crc 2023-06-10T21:50:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:33.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:33 smithi152 ceph-mon[156819]: osdmap e484: 8 total, 8 up, 5 in 2023-06-10T21:50:34.452 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:34.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:34.558 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:34.586 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:34.629 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:34.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.713 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: pgmap v1760: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 218 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 435 op/s; 1496/10533 objects misplaced (14.203%); 20 MiB/s, 325 objects/s recovering 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[186706]: osdmap e485: 8 total, 8 up, 5 in 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: pgmap v1760: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 218 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 435 op/s; 1496/10533 objects misplaced (14.203%); 20 MiB/s, 325 objects/s recovering 2023-06-10T21:50:34.715 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:34 smithi064 ceph-mon[180297]: osdmap e485: 8 total, 8 up, 5 in 2023-06-10T21:50:34.734 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:34.764 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:34.867 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:34.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: failed to encode map e483 with expected crc 2023-06-10T21:50:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: failed to encode map e484 with expected crc 2023-06-10T21:50:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: pgmap v1760: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 218 MiB data, 4.3 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 435 op/s; 1496/10533 objects misplaced (14.203%); 20 MiB/s, 325 objects/s recovering 2023-06-10T21:50:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:34 smithi152 ceph-mon[156819]: osdmap e485: 8 total, 8 up, 5 in 2023-06-10T21:50:34.971 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:34.999 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:35.455 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:50:35.455 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "MOSDPGPush(12.a 363/358 [PushOp(12:53f3f12a:::benchmark_data_smithi064_442287_object74883:head, version: 322'4732, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(12:53f3f12a:::benchmark_data_smithi064_442287_object74883:head@322'4732, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 599.09803947299997, 2023-06-10T21:50:35.456 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25852421799999997, 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "started", 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:50:35.457 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302178+0000", 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.458 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302176+0000", 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.302218+0000", 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1696000000000002e-05 2023-06-10T21:50:35.459 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558593+0000", 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.25637484599999999 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.460 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558629+0000", 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5955999999999999e-05 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558742+0000", 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000113353 2023-06-10T21:50:35.461 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.558758+0000", 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5642e-05 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:35.462 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:40:36.560702+0000", 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019444530000000001 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:35.463 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:50:35.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: failed to encode map e484 with expected crc 2023-06-10T21:50:35.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.731 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.732 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[186706]: osdmap e486: 8 total, 8 up, 5 in 2023-06-10T21:50:35.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: failed to encode map e484 with expected crc 2023-06-10T21:50:35.732 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:35 smithi064 ceph-mon[180297]: osdmap e486: 8 total, 8 up, 5 in 2023-06-10T21:50:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: failed to encode map e484 with expected crc 2023-06-10T21:50:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: failed to encode map e485 with expected crc 2023-06-10T21:50:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:35 smithi152 ceph-mon[156819]: osdmap e486: 8 total, 8 up, 5 in 2023-06-10T21:50:36.032 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:36.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:36.137 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:36.210 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:36.486 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:36.495 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:36.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:36.531 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-10T21:50:36.537 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:50:36.600 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:36.691 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:36.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[186706]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[186706]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[186706]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.773 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[186706]: pgmap v1763: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 213 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 1496/10269 objects misplaced (14.568%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:50:36.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[180297]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.773 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[180297]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[180297]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.774 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:36 smithi064 ceph-mon[180297]: pgmap v1763: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 213 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 1496/10269 objects misplaced (14.568%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:50:36.862 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:36.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:36.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:36 smithi152 ceph-mon[156819]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:36 smithi152 ceph-mon[156819]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:36 smithi152 ceph-mon[156819]: failed to encode map e486 with expected crc 2023-06-10T21:50:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:36 smithi152 ceph-mon[156819]: pgmap v1763: 99 pgs: 2 peering, 7 active+remapped+backfill_wait, 1 active+remapped, 89 active+clean; 213 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 173 op/s; 1496/10269 objects misplaced (14.568%); 9.8 MiB/s, 157 objects/s recovering 2023-06-10T21:50:36.988 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:37.068 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:37.473 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:37.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:37.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:37.591 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:37.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:37 smithi064 ceph-mon[180297]: failed to encode map e486 with expected crc 2023-06-10T21:50:37.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:37 smithi064 ceph-mon[186706]: failed to encode map e486 with expected crc 2023-06-10T21:50:37.647 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:37.709 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:37.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:37.813 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 2023-06-10T21:50:37.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:37 smithi152 ceph-mon[156819]: failed to encode map e486 with expected crc 2023-06-10T21:50:37.840 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:37.882 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4 found: no such container 2023-06-10T21:50:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:37.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.4' 2023-06-10T21:50:37.987 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:38.017 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:38.062 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:38.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:38.166 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:38.193 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:38.375 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:38.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:38.424 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:38.479 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:38.506 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:38.510 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:38.548 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:38.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:38.563 INFO:teuthology.orchestra.run.smithi064.stderr:set pool 14 pgp_num to 26 2023-06-10T21:50:38.638 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: osdmap e487: 8 total, 8 up, 5 in 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.639 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[180297]: pgmap v1765: 99 pgs: 1 active+remapped+backfilling, 2 peering, 2 active+remapped+backfill_wait, 94 active+clean; 170 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 390/8223 objects misplaced (4.743%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:50:38.640 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: osdmap e487: 8 total, 8 up, 5 in 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.641 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-06-10T21:50:38.642 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:38 smithi064 ceph-mon[186706]: pgmap v1765: 99 pgs: 1 active+remapped+backfilling, 2 peering, 2 active+remapped+backfill_wait, 94 active+clean; 170 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 390/8223 objects misplaced (4.743%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:50:38.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:38.686 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:38.745 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:38.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:38.850 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:38.901 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: osdmap e487: 8 total, 8 up, 5 in 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: failed to encode map e487 with expected crc 2023-06-10T21:50:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]: dispatch 2023-06-10T21:50:38.946 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:38 smithi152 ceph-mon[156819]: pgmap v1765: 99 pgs: 1 active+remapped+backfilling, 2 peering, 2 active+remapped+backfill_wait, 94 active+clean; 170 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 278 op/s; 390/8223 objects misplaced (4.743%); 15 MiB/s, 237 objects/s recovering 2023-06-10T21:50:39.479 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:39.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:39.592 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:39.593 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Removed 107853 objects 2023-06-10T21:50:39.593 INFO:tasks.radosbench.radosbench.0.smithi064.stdout:Clean up completed and total clean up time :195.365 2023-06-10T21:50:39.620 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:39.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[186706]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.825 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[186706]: osdmap e488: 8 total, 8 up, 5 in 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[186706]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[186706]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[180297]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-06-10T21:50:39.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[180297]: osdmap e488: 8 total, 8 up, 5 in 2023-06-10T21:50:39.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[180297]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:39 smithi064 ceph-mon[180297]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.873 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:39.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:39.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:39.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:39 smithi152 ceph-mon[156819]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:39 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1955969403' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "26"}]': finished 2023-06-10T21:50:39.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:39 smithi152 ceph-mon[156819]: osdmap e488: 8 total, 8 up, 5 in 2023-06-10T21:50:39.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:39 smithi152 ceph-mon[156819]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:39 smithi152 ceph-mon[156819]: failed to encode map e488 with expected crc 2023-06-10T21:50:39.978 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:40.008 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:40.208 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:40.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:40.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:40.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:40.341 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:40.577 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is set 2023-06-10T21:50:40.841 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-06-10T21:50:40.842 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-06-10T21:50:40.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[186706]: failed to encode map e488 with expected crc 2023-06-10T21:50:40.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3963671249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[186706]: pgmap v1767: 99 pgs: 99 active+clean; 57 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[180297]: failed to encode map e488 with expected crc 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3963671249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.868 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[180297]: pgmap v1767: 99 pgs: 99 active+clean; 57 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:50:40.869 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:50:40.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:40 smithi152 ceph-mon[156819]: failed to encode map e488 with expected crc 2023-06-10T21:50:40.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:40 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3963671249' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:40 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:40 smithi152 ceph-mon[156819]: pgmap v1767: 99 pgs: 99 active+clean; 57 MiB data, 3.8 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 440 op/s; 8.6 MiB/s, 137 objects/s recovering 2023-06-10T21:50:40.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-06-10T21:50:41.657 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:41.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:41.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:41.762 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:41.792 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:41.834 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:41.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: osdmap e489: 8 total, 8 up, 5 in 2023-06-10T21:50:41.837 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:41 smithi152 ceph-mon[156819]: osdmap e490: 8 total, 8 up, 5 in 2023-06-10T21:50:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:41.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:41.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:50:41.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: osdmap e489: 8 total, 8 up, 5 in 2023-06-10T21:50:41.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[186706]: osdmap e490: 8 total, 8 up, 5 in 2023-06-10T21:50:41.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:41.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: failed to encode map e489 with expected crc 2023-06-10T21:50:41.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:41.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2023-06-10T21:50:41.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: osdmap e489: 8 total, 8 up, 5 in 2023-06-10T21:50:41.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:41 smithi064 ceph-mon[180297]: osdmap e490: 8 total, 8 up, 5 in 2023-06-10T21:50:41.939 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:41.972 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:42.070 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:42.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:42.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:42.176 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:42.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:42.265 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:42.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:42.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:42.352 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:42.371 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:42.400 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:42.460 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:42.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:42.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:42.566 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:42.599 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:42.697 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:42.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:42.709 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:50:42.709 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:50:42.710 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:50:42.710 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:50:42.802 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:42.831 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: pgmap v1770: 99 pgs: 99 active+clean; 25 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 3.6 MiB/s, 56 objects/s recovering 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:50:42.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[186706]: osdmap e491: 8 total, 8 up, 5 in 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: pgmap v1770: 99 pgs: 99 active+clean; 25 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 3.6 MiB/s, 56 objects/s recovering 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:50:42.833 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:50:42.834 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:42 smithi064 ceph-mon[180297]: osdmap e491: 8 total, 8 up, 5 in 2023-06-10T21:50:42.834 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:42.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: failed to encode map e490 with expected crc 2023-06-10T21:50:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: pgmap v1770: 99 pgs: 99 active+clean; 25 MiB data, 3.6 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 492 op/s; 3.6 MiB/s, 56 objects/s recovering 2023-06-10T21:50:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-06-10T21:50:42.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "18"}]': finished 2023-06-10T21:50:42.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:42 smithi152 ceph-mon[156819]: osdmap e491: 8 total, 8 up, 5 in 2023-06-10T21:50:42.995 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:50:43.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:50:43.152 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:42] "GET /metrics HTTP/1.1" 200 224291 "" "Prometheus/2.33.4" 2023-06-10T21:50:43.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:50:43.170 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.4: {} 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.171 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.172 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:50:43.625 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:43.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:43.715 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:50:43.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:43.786 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:43.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[180297]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[180297]: 14.10 starting backfill to osd.7 from (0'0,0'0] MAX to 488'12839 2023-06-10T21:50:43.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[180297]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3810100095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[180297]: osdmap e492: 8 total, 8 up, 5 in 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[186706]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[186706]: 14.10 starting backfill to osd.7 from (0'0,0'0] MAX to 488'12839 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[186706]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3810100095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:43.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:43 smithi064 ceph-mon[186706]: osdmap e492: 8 total, 8 up, 5 in 2023-06-10T21:50:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:43 smithi152 ceph-mon[156819]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:43 smithi152 ceph-mon[156819]: 14.10 starting backfill to osd.7 from (0'0,0'0] MAX to 488'12839 2023-06-10T21:50:43.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:43 smithi152 ceph-mon[156819]: failed to encode map e491 with expected crc 2023-06-10T21:50:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:43 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3810100095' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:43.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:43 smithi152 ceph-mon[156819]: osdmap e492: 8 total, 8 up, 5 in 2023-06-10T21:50:44.642 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:44.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:44.667 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-10T21:50:44.750 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:44.782 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:44.797 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:44.829 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:44.834 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e490 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e491 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.835 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: pgmap v1773: 99 pgs: 99 active+clean; 13 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 201 op/s 2023-06-10T21:50:44.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-06-10T21:50:44.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-06-10T21:50:44.836 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:44 smithi152 ceph-mon[156819]: osdmap e493: 8 total, 8 up, 5 in 2023-06-10T21:50:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:44.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:44.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e490 with expected crc 2023-06-10T21:50:44.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e491 with expected crc 2023-06-10T21:50:44.869 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: pgmap v1773: 99 pgs: 99 active+clean; 13 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 201 op/s 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-06-10T21:50:44.870 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[186706]: osdmap e493: 8 total, 8 up, 5 in 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e490 with expected crc 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e491 with expected crc 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.871 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: failed to encode map e492 with expected crc 2023-06-10T21:50:44.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: pgmap v1773: 99 pgs: 99 active+clean; 13 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 201 op/s 2023-06-10T21:50:44.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-06-10T21:50:44.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "19"}]': finished 2023-06-10T21:50:44.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:44 smithi064 ceph-mon[180297]: osdmap e493: 8 total, 8 up, 5 in 2023-06-10T21:50:44.878 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [] live_osds: [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:50:44.879 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:50:44.879 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [1, 2, 4, 6, 7, 0, 3, 5] 2023-06-10T21:50:44.879 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-06-10T21:50:44.879 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4 2023-06-10T21:50:44.938 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:44.971 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:45.021 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:45.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:45.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:45.127 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:45.165 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:45.173 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:50:44 smithi152 systemd[1]: Stopping Ceph osd.4 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:50:45.220 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:45.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:45.324 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:45.351 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:45.395 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:45.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:45.443 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:50:45 smithi152 conmon[103728]: debug 2023-06-10T21:50:45.169+0000 7f1f45102700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:50:45.443 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:50:45 smithi152 conmon[103728]: debug 2023-06-10T21:50:45.169+0000 7f1f45102700 -1 osd.4 493 *** Got signal Terminated *** 2023-06-10T21:50:45.443 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:50:45 smithi152 conmon[103728]: debug 2023-06-10T21:50:45.169+0000 7f1f45102700 -1 osd.4 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:50:45.453 INFO:teuthology.orchestra.run.smithi064.stderr:pool 'unique_pool_9' removed 2023-06-10T21:50:45.499 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:45.529 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:45.613 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:45.627 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:45.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:45.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:45.765 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:45.807 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:50:45 smithi152 bash[369069]: 8e3deb2c92b2cc35ba5a411da67f4a74c3d16673e7b99642e2f629d221657c0d 2023-06-10T21:50:45.808 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.808 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.808 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.808 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.809 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: osd.4 marked itself down and dead 2023-06-10T21:50:45.809 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:50:45.809 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:50:45.809 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:45 smithi152 ceph-mon[156819]: osdmap e494: 8 total, 7 up, 5 in 2023-06-10T21:50:45.828 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:45.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:45.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: osd.4 marked itself down and dead 2023-06-10T21:50:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:50:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:50:45.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[180297]: osdmap e494: 8 total, 7 up, 5 in 2023-06-10T21:50:45.843 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: failed to encode map e493 with expected crc 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: osd.4 marked itself down and dead 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2283209814' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-10T21:50:45.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:45 smithi064 ceph-mon[186706]: osdmap e494: 8 total, 7 up, 5 in 2023-06-10T21:50:45.934 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:45.961 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:46.004 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:46.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:46.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:46.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:46.134 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:46.175 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:46.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:46.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:46.279 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:46.310 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:46.512 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:46.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:46.531 INFO:teuthology.task.sequential:In sequential, running task print... 2023-06-10T21:50:46.531 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-06-10T21:50:46.532 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-06-10T21:50:46.536 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-06-10T21:50:46.537 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-06-10T21:50:46.537 DEBUG:teuthology.orchestra.run.smithi064:> stat -- /home/ubuntu/cephtest/mnt.0 2023-06-10T21:50:46.600 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[180297]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[180297]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.601 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[180297]: pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:46.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T21:50:46.607 INFO:teuthology.orchestra.run.smithi064.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-06-10T21:50:46.607 DEBUG:teuthology.orchestra.run.smithi064:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-06-10T21:50:46.609 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:50:46.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:46.645 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:46.699 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-06-10T21:50:46.700 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-06-10T21:50:46.756 INFO:tasks.workunit:timeout=3h 2023-06-10T21:50:46.756 INFO:tasks.workunit:cleanup=True 2023-06-10T21:50:46.757 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-06-10T21:50:46.792 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:46.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:46.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:46.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[186706]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[186706]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.865 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:46 smithi064 ceph-mon[186706]: pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:46.872 INFO:tasks.workunit.client.0.smithi064.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-06-10T21:50:46.896 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:46.925 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:46.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:46 smithi152 ceph-mon[156819]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:46 smithi152 ceph-mon[156819]: failed to encode map e494 with expected crc 2023-06-10T21:50:46.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:46 smithi152 ceph-mon[156819]: pgmap v1776: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:47.226 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:47.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:47.330 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:47.358 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:47.403 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:47.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:47.507 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:47.534 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:47.613 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:50:47.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[186706]: failed to encode map e494 with expected crc 2023-06-10T21:50:47.857 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1453311225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:47.858 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:47.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[180297]: failed to encode map e494 with expected crc 2023-06-10T21:50:47.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1453311225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:47.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:47 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:47.913 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:47 smithi152 ceph-mon[156819]: failed to encode map e494 with expected crc 2023-06-10T21:50:47.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1453311225' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:47.914 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:47 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:50:48.332 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:48.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:48.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:48.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4.service' 2023-06-10T21:50:48.413 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:50:48.413 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-06-10T21:50:48.437 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:48.466 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:48.516 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:48.621 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:48.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: failed to encode map e495 with expected crc 2023-06-10T21:50:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: osdmap e495: 8 total, 7 up, 5 in 2023-06-10T21:50:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:48.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:48 smithi152 ceph-mon[156819]: pgmap v1778: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:48.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:48.760 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:48.808 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:48.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:48.817 INFO:teuthology.orchestra.run.smithi064.stdout:ERROR: (22) Invalid argument 2023-06-10T21:50:48.818 INFO:teuthology.orchestra.run.smithi064.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:50:48.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:48.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: failed to encode map e495 with expected crc 2023-06-10T21:50:48.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:48.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: osdmap e495: 8 total, 7 up, 5 in 2023-06-10T21:50:48.826 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[180297]: pgmap v1778: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: failed to encode map e495 with expected crc 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: osdmap e495: 8 total, 7 up, 5 in 2023-06-10T21:50:48.827 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:50:48.828 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:48 smithi064 ceph-mon[186706]: pgmap v1778: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:48.914 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:48.945 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:49.494 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:49.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:49.599 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:49.629 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:49 smithi064 ceph-mon[186706]: failed to encode map e495 with expected crc 2023-06-10T21:50:49.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:49 smithi064 ceph-mon[180297]: failed to encode map e495 with expected crc 2023-06-10T21:50:49.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:49.678 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:49.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:49.782 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:49.810 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:49.884 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-10T21:50:49.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:49 smithi152 ceph-mon[156819]: failed to encode map e495 with expected crc 2023-06-10T21:50:49.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:50:50.157 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:50.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:50.265 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:50.297 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:50.346 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:50.349 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:50.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:50.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:50.410 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:50:50.452 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:50.482 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:50.600 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:50.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:50.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:50.705 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:50.733 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:50 smithi064 ceph-mon[180297]: pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:50.733 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:50 smithi064 ceph-mon[186706]: pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:50.736 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:50.844 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:50.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:50.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:50 smithi152 ceph-mon[156819]: pgmap v1779: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:50.949 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:50.978 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:51.013 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:50:51.024 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:51.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:51.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:51.033 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:50:51.053 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:50:51.074 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:50:51.095 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:50:51.113 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:50:51.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:51.131 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:50:51.131 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.131 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.132 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:50:51.176 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:51.218 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:51.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:51.324 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:51.354 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:51.401 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:51.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:51.506 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:51.535 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:51.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:51 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1673819397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:51.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:51 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1673819397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:51.937 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:51.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:51.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:51 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1673819397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:50:52.042 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:52.114 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:52.436 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:52.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:52.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:52.518 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:50:52.519 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:50:52.540 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:50:52.562 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:52.571 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:52.630 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:50:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:52.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:50:52.735 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:52.769 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:52.852 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:52 smithi064 ceph-mon[186706]: pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:52.853 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:52 smithi064 ceph-mon[180297]: pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:52.871 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:52.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:52.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:52 smithi152 ceph-mon[156819]: pgmap v1780: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:52.977 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:53.007 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:53.114 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:53.115 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:50:53.115 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:50:53.115 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:50:53.115 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113503 14.d 14:b7d16f24:::benchmark_data_smithi064_559916_object5647:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.334382+0000", 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 200.77430639799999, 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2427955160000002, 2023-06-10T21:50:53.116 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113503 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.117 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334382+0000", 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.118 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334382+0000", 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334380+0000", 2023-06-10T21:50:53.119 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334387+0000", 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3839999999999999e-06 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.120 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334398+0000", 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0766e-05 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.121 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334406+0000", 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1519999999999992e-06 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334450+0000", 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3788999999999997e-05 2023-06-10T21:50:53.122 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.334460+0000", 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0349999999999999e-05 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.123 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.335766+0000", 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.001305857 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.335781+0000", 2023-06-10T21:50:53.124 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4464000000000001e-05 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.470198+0000", 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.134417633 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.125 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.470354+0000", 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000155267 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:50:53.126 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.470492+0000", 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013782900000000001 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.480314+0000", 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0098220979999999996 2023-06-10T21:50:53.127 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.480577+0000", 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026369700000000002 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.128 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.485815+0000", 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052378640000000001 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.577019+0000", 2023-06-10T21:50:53.129 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.091203984000000002 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.577068+0000", 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9082000000000002e-05 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.130 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.577177+0000", 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109054 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.131 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113504 14.d 14:b483ff03:::benchmark_data_smithi064_559916_object5648:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.374447+0000", 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 200.73424153400001, 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2207317930000001, 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.132 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113504 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.133 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374447+0000", 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.134 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374447+0000", 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374444+0000", 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999971 2023-06-10T21:50:53.135 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374454+0000", 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6979999999999993e-06 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.136 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374466+0000", 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2476e-05 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.374476+0000", 2023-06-10T21:50:53.137 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0370000000000001e-05 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.380635+0000", 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0061584320000000001 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.138 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.380648+0000", 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.366e-05 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.139 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.336258+0000", 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95560962699999996 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.336269+0000", 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1352000000000001e-05 2023-06-10T21:50:53.140 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.473816+0000", 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.137546306 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.141 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.473898+0000", 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2404999999999995e-05 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.473973+0000", 2023-06-10T21:50:53.142 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4578000000000003e-05 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.480334+0000", 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0063614810000000004 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.143 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.480624+0000", 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000289838 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.485824+0000", 2023-06-10T21:50:53.144 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052004039999999996 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595036+0000", 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10921154199999999 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.145 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595094+0000", 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8038999999999998e-05 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.146 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595178+0000", 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4380000000000002e-05 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.147 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113508 14.d 14:bed607f1:::benchmark_data_smithi064_559916_object5652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.399493+0000", 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 200.70919554299999, 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.1957976170000002, 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.148 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113508 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.149 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399493+0000", 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399493+0000", 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.150 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399491+0000", 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.151 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399498+0000", 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1790000000000002e-06 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399510+0000", 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2037e-05 2023-06-10T21:50:53.152 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399519+0000", 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9199999999999993e-06 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.153 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399570+0000", 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1242000000000002e-05 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399582+0000", 2023-06-10T21:50:53.154 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1548e-05 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.336305+0000", 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93672320799999997 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.155 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.336315+0000", 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8239999999999995e-06 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.156 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.476931+0000", 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.140616136 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.477053+0000", 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122342 2023-06-10T21:50:53.157 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.477194+0000", 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000140963 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.158 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.487432+0000", 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010237302 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.487438+0000", 2023-06-10T21:50:53.159 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.195e-06 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.487695+0000", 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025702500000000003 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.160 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595204+0000", 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.10750918499999999 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.161 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595232+0000", 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.7920999999999999e-05 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.595290+0000", 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8162000000000003e-05 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.162 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212282 14.0 14:01440542:::benchmark_data_smithi064_559916_object104426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:32.572289+0000", 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.536399163999999, 2023-06-10T21:50:53.163 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88391816499999998, 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212282 2023-06-10T21:50:53.164 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572289+0000", 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.165 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572289+0000", 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.166 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572289+0000", 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572292+0000", 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2839999999999998e-06 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.167 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572299+0000", 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.148e-06 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.168 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572306+0000", 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6919999999999996e-06 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572369+0000", 2023-06-10T21:50:53.169 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2874999999999997e-05 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.572379+0000", 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0122999999999999e-05 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.170 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.442430+0000", 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.870051401 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.171 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.442549+0000", 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011906299999999999 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.443030+0000", 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000481063 2023-06-10T21:50:53.172 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.448391+0000", 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0053603990000000001 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.173 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.453850+0000", 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054593699999999998 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456140+0000", 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022897460000000001 2023-06-10T21:50:53.174 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456169+0000", 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9641e-05 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.175 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456207+0000", 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7821999999999998e-05 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.176 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212314 14.0 14:06f371b1:::benchmark_data_smithi064_559916_object104458:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:32.597661+0000", 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.511027646999999, 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86804856100000005, 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.177 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212314 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.178 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597661+0000", 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597661+0000", 2023-06-10T21:50:53.179 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597660+0000", 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.180 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597663+0000", 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4450000000000001e-06 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.181 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597670+0000", 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0879999999999997e-06 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597674+0000", 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8820000000000003e-06 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.182 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597705+0000", 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0926000000000002e-05 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.183 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.597711+0000", 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.789e-06 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.448417+0000", 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.85070601800000001 2023-06-10T21:50:53.184 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.448550+0000", 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013281799999999999 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.185 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.449052+0000", 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050216599999999996 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456111+0000", 2023-06-10T21:50:53.186 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0070591810000000003 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.460547+0000", 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044358649999999998 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.187 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465625+0000", 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050782659999999997 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.188 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465669+0000", 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4045000000000003e-05 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465709+0000", 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9641999999999998e-05 2023-06-10T21:50:53.189 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212338 14.0 14:0647c6ff:::benchmark_data_smithi064_559916_object104482:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:32.621484+0000", 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.487203858000001, 2023-06-10T21:50:53.190 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84432549499999998, 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212338 2023-06-10T21:50:53.191 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621484+0000", 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.192 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621484+0000", 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.193 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621483+0000", 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621489+0000", 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6540000000000004e-06 2023-06-10T21:50:53.194 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621504+0000", 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5252999999999999e-05 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.195 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621512+0000", 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6320000000000008e-06 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621569+0000", 2023-06-10T21:50:53.196 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7286e-05 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.621579+0000", 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9140000000000003e-06 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.197 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.451226+0000", 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.82964697600000004 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.198 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.451380+0000", 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015463200000000001 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.451853+0000", 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000473031 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.199 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456219+0000", 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043658799999999999 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.200 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.462505+0000", 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062858330000000002 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465744+0000", 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003238856 2023-06-10T21:50:53.201 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465778+0000", 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4113999999999999e-05 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.202 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465810+0000", 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1699999999999998e-05 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.203 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212377 14.0 14:036548c1:::benchmark_data_smithi064_559916_object104521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:32.665284+0000", 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.443404112, 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.81648863500000002, 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.204 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212377 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.205 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665284+0000", 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665284+0000", 2023-06-10T21:50:53.206 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665284+0000", 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.207 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665286+0000", 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5809999999999999e-06 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665295+0000", 2023-06-10T21:50:53.208 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4020000000000002e-06 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665300+0000", 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6960000000000002e-06 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.209 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665374+0000", 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3289999999999998e-05 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.210 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.665385+0000", 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1351e-05 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.453981+0000", 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78859624500000003 2023-06-10T21:50:53.211 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.454107+0000", 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012550800000000001 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.212 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.454506+0000", 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000399434 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.462533+0000", 2023-06-10T21:50:53.213 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0080271660000000005 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463575+0000", 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010418440000000001 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.214 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481627+0000", 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018052353 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481730+0000", 2023-06-10T21:50:53.215 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000102322 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481773+0000", 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2889000000000002e-05 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.216 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212381 14.0 14:00c748d9:::benchmark_data_smithi064_559916_object104525:head [delete] snapc 0=[] ondisk+write+known_if_redirected e483)", 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:32.670492+0000", 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 20.438196268999999, 2023-06-10T21:50:53.217 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.811420162, 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212381 2023-06-10T21:50:53.218 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670492+0000", 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.219 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670492+0000", 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.220 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670491+0000", 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670497+0000", 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4980000000000003e-06 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.221 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670513+0000", 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6200000000000001e-05 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.222 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670522+0000", 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9160000000000007e-06 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670581+0000", 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8953999999999998e-05 2023-06-10T21:50:53.223 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:32.670590+0000", 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0459999999999994e-06 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.224 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456242+0000", 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.78565133200000004 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456329+0000", 2023-06-10T21:50:53.225 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7366000000000003e-05 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456629+0000", 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00030040299999999998 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.226 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.462559+0000", 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0059291400000000003 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.227 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463583+0000", 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010239470000000001 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481835+0000", 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.018252397 2023-06-10T21:50:53.228 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481878+0000", 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3169e-05 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.229 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481912+0000", 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3989000000000003e-05 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.230 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212449 14.10 14:08ef9f03:::benchmark_data_smithi064_559916_object104593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.463606+0000", 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 19.645081683000001, 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0015856519999999, 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.231 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212449 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.232 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463606+0000", 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463606+0000", 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.233 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463605+0000", 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.234 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463613+0000", 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1030000000000002e-06 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463626+0000", 2023-06-10T21:50:53.235 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3174e-05 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.463636+0000", 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0046e-05 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.236 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465873+0000", 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0022363859999999999 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.465880+0000", 2023-06-10T21:50:53.237 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.943e-06 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455372+0000", 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98949197099999997 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.238 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455499+0000", 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000127064 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.239 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455911+0000", 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00041143400000000002 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462333+0000", 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064225760000000002 2023-06-10T21:50:53.240 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462493+0000", 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159586 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.241 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465106+0000", 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026132600000000001 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465150+0000", 2023-06-10T21:50:53.242 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4097999999999997e-05 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465192+0000", 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1705000000000001e-05 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.243 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212455 14.10 14:0bffc565:::benchmark_data_smithi064_559916_object104599:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.468281+0000", 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 19.640407436, 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99710298100000005, 2023-06-10T21:50:53.244 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212455 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.245 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468281+0000", 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.246 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468281+0000", 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468280+0000", 2023-06-10T21:50:53.247 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468283+0000", 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.4889999999999998e-06 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.248 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468288+0000", 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5690000000000004e-06 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.249 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.468293+0000", 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8690000000000003e-06 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.470804+0000", 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025102869999999999 2023-06-10T21:50:53.250 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.470812+0000", 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4079999999999998e-06 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.251 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.457990+0000", 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98717800200000005 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.458092+0000", 2023-06-10T21:50:53.252 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010176200000000001 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.458472+0000", 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00038040399999999999 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.253 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.464687+0000", 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062150360000000002 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465247+0000", 2023-06-10T21:50:53.254 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00055966900000000005 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465289+0000", 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2546999999999998e-05 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.255 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465327+0000", 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7849999999999998e-05 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.256 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465384+0000", 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6455000000000001e-05 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.257 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212514 14.10 14:08e01c74:::benchmark_data_smithi064_559916_object104658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.555258+0000", 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 19.553430648999999, 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92074914600000002, 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.258 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212514 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555258+0000", 2023-06-10T21:50:53.259 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555258+0000", 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.260 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555256+0000", 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.261 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555262+0000", 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1829999999999999e-06 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555274+0000", 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1514e-05 2023-06-10T21:50:53.262 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555280+0000", 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.015e-06 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.263 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555341+0000", 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1226e-05 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.555350+0000", 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6000000000000007e-06 2023-06-10T21:50:53.264 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460268+0000", 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90491782700000001 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.265 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460379+0000", 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011181299999999999 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460696+0000", 2023-06-10T21:50:53.266 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00031613600000000001 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.464695+0000", 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039991649999999998 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.267 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465462+0000", 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00076719899999999998 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.268 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475928+0000", 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010465842 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475967+0000", 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9561000000000003e-05 2023-06-10T21:50:53.269 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.476007+0000", 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9375999999999997e-05 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.270 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212552 14.10 14:0d74b291:::benchmark_data_smithi064_559916_object104696:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.617743+0000", 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 19.490944902999999, 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86088211400000003, 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.271 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212552 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.272 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617743+0000", 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.273 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617743+0000", 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617742+0000", 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:50:53.274 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617746+0000", 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8550000000000004e-06 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.275 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617753+0000", 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0219999999999999e-06 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617762+0000", 2023-06-10T21:50:53.276 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.636e-06 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617803+0000", 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1054999999999999e-05 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.277 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.617812+0000", 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1300000000000007e-06 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462539+0000", 2023-06-10T21:50:53.278 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.84472664900000005 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462637+0000", 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8703999999999996e-05 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.279 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462964+0000", 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00032636299999999998 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.280 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478501+0000", 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01553758 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478509+0000", 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8290000000000008e-06 2023-06-10T21:50:53.281 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478543+0000", 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4408e-05 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.282 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478586+0000", 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2876000000000001e-05 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478625+0000", 2023-06-10T21:50:53.283 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9100999999999997e-05 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212564 14.e 14:7b5c8f88:::benchmark_data_smithi064_559916_object104708:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.284 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.462206+0000", 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.646482335000002, 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0019112029999999, 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.285 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212564 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462206+0000", 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.286 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462206+0000", 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.287 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462205+0000", 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462209+0000", 2023-06-10T21:50:53.288 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5480000000000003e-06 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462219+0000", 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5619999999999993e-06 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.289 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462225+0000", 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6290000000000004e-06 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462280+0000", 2023-06-10T21:50:53.290 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4292000000000001e-05 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462290+0000", 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0088999999999999e-05 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.291 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.454815+0000", 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99252488800000005 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.292 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.454914+0000", 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9284999999999996e-05 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.454983+0000", 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8788000000000006e-05 2023-06-10T21:50:53.293 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464001+0000", 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090182349999999994 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.294 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464033+0000", 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1974000000000003e-05 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464042+0000", 2023-06-10T21:50:53.295 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0769999999999997e-06 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464073+0000", 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0552999999999997e-05 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.296 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464117+0000", 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4483000000000001e-05 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.297 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.298 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212573 14.e 14:743e6dac:::benchmark_data_smithi064_559916_object104717:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.475747+0000", 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.632940782999999, 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98873259099999999, 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.301 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212573 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.302 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475747+0000", 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475747+0000", 2023-06-10T21:50:53.303 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475747+0000", 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.304 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475750+0000", 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0699999999999998e-06 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.305 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475760+0000", 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5429999999999999e-06 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475765+0000", 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5600000000000001e-06 2023-06-10T21:50:53.306 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475866+0000", 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000100886 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.307 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.475876+0000", 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8239999999999995e-06 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.458768+0000", 2023-06-10T21:50:53.308 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98289268100000005 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.458881+0000", 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011214600000000001 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.309 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.458998+0000", 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011745 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464165+0000", 2023-06-10T21:50:53.310 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0051664379999999998 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464286+0000", 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121551 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.311 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464383+0000", 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6464000000000003e-05 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.312 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464442+0000", 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9855000000000002e-05 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464480+0000", 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7589999999999998e-05 2023-06-10T21:50:53.313 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212587 14.e 14:7527e6ee:::benchmark_data_smithi064_559916_object104731:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.488470+0000", 2023-06-10T21:50:53.314 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.620218051999998, 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97938323400000005, 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.315 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212587 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488470+0000", 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.316 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488470+0000", 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.317 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488468+0000", 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488476+0000", 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2899999999999997e-06 2023-06-10T21:50:53.318 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488493+0000", 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7299000000000002e-05 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.319 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.488501+0000", 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1219999999999995e-06 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493345+0000", 2023-06-10T21:50:53.320 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048441029999999998 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493353+0000", 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1680000000000004e-06 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.321 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.461558+0000", 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.968204387 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.322 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.461679+0000", 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121092 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.461748+0000", 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9552000000000001e-05 2023-06-10T21:50:53.323 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464328+0000", 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025800659999999998 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.324 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464341+0000", 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2830000000000001e-05 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.467758+0000", 2023-06-10T21:50:53.325 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0034164199999999999 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.467816+0000", 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8223000000000001e-05 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.326 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.467853+0000", 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7400000000000001e-05 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.327 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212612 14.e 14:7b079954:::benchmark_data_smithi064_559916_object104756:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.524754+0000", 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.583934557999999, 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95691751800000002, 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.328 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212612 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.329 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524754+0000", 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524754+0000", 2023-06-10T21:50:53.330 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524752+0000", 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.331 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524761+0000", 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4110000000000002e-06 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.332 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524775+0000", 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3747000000000001e-05 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524785+0000", 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7059999999999999e-06 2023-06-10T21:50:53.333 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524844+0000", 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9120000000000003e-05 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.334 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.524855+0000", 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1358999999999999e-05 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464520+0000", 2023-06-10T21:50:53.335 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93966495999999999 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464634+0000", 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011373499999999999 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.336 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.464749+0000", 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114917 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.337 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.467893+0000", 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003144048 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474183+0000", 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0062899139999999997 2023-06-10T21:50:53.338 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481577+0000", 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00739436 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.339 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481629+0000", 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2151000000000003e-05 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481671+0000", 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1721000000000003e-05 2023-06-10T21:50:53.340 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212616 14.e 14:7180b0af:::benchmark_data_smithi064_559916_object104760:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.525641+0000", 2023-06-10T21:50:53.341 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.58304759, 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.95615622499999997, 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.342 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212616 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525641+0000", 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.343 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525641+0000", 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.344 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525639+0000", 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525645+0000", 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3129999999999998e-06 2023-06-10T21:50:53.345 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525654+0000", 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0860000000000008e-06 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.346 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525661+0000", 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6199999999999999e-06 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525709+0000", 2023-06-10T21:50:53.347 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7744000000000002e-05 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.525717+0000", 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2709999999999992e-06 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.348 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.467916+0000", 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94219849200000005 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.349 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.468013+0000", 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6620000000000007e-05 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.468137+0000", 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012434100000000001 2023-06-10T21:50:53.350 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.477851+0000", 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0097143750000000008 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.351 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481535+0000", 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036839849999999999 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481715+0000", 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018024600000000001 2023-06-10T21:50:53.352 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481764+0000", 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8340000000000001e-05 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.353 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481797+0000", 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2977e-05 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.354 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212621 14.e 14:721a2f23:::benchmark_data_smithi064_559916_object104765:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.532049+0000", 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.576638698, 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96475328500000002, 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.355 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212621 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.356 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532049+0000", 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532049+0000", 2023-06-10T21:50:53.357 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532048+0000", 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.358 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532055+0000", 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8090000000000004e-06 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532072+0000", 2023-06-10T21:50:53.359 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6739e-05 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532080+0000", 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6300000000000004e-06 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.360 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532142+0000", 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1580000000000003e-05 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.361 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.532151+0000", 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8640000000000002e-06 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.470853+0000", 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93870222199999998 2023-06-10T21:50:53.362 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.470983+0000", 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012990000000000001 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.363 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.471149+0000", 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00016567200000000001 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481494+0000", 2023-06-10T21:50:53.364 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010345211999999999 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.482988+0000", 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0014944789999999999 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.365 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496685+0000", 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013696335 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496745+0000", 2023-06-10T21:50:53.366 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0823999999999999e-05 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496803+0000", 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7274000000000001e-05 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.367 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212658 14.e 14:7a31c0d6:::benchmark_data_smithi064_559916_object104802:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.571306+0000", 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.537381698000001, 2023-06-10T21:50:53.368 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92564441500000005, 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212658 2023-06-10T21:50:53.369 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571306+0000", 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.370 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571306+0000", 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.371 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571305+0000", 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571311+0000", 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3380000000000004e-06 2023-06-10T21:50:53.372 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571321+0000", 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0243e-05 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.373 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571327+0000", 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3589999999999996e-06 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571385+0000", 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7720000000000003e-05 2023-06-10T21:50:53.374 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.571407+0000", 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2432000000000001e-05 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.375 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474212+0000", 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.90280462500000003 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474334+0000", 2023-06-10T21:50:53.376 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000122259 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474473+0000", 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000138226 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.377 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.481543+0000", 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007070565 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.378 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.484225+0000", 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0026820609999999999 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496851+0000", 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012625499 2023-06-10T21:50:53.379 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496903+0000", 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2280000000000001e-05 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.380 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496951+0000", 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7956e-05 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.381 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212662 14.e 14:70e17eac:::benchmark_data_smithi064_559916_object104806:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.583145+0000", 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 18.525543171999999, 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91388716699999994, 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:50:53.382 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212662 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:50:53.383 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583145+0000", 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583145+0000", 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:50:53.384 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583143+0000", 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.385 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583152+0000", 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.7819999999999999e-06 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583171+0000", 2023-06-10T21:50:53.386 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9419000000000002e-05 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583184+0000", 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2544000000000001e-05 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.387 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583245+0000", 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1111999999999994e-05 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.583256+0000", 2023-06-10T21:50:53.388 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1163e-05 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.477884+0000", 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.89462788000000004 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.389 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.478024+0000", 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013970700000000001 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:50:53.390 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.478160+0000", 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013636599999999999 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.484786+0000", 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0066256129999999998 2023-06-10T21:50:53.391 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.488296+0000", 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0035098299999999998 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.392 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.496990+0000", 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086940760000000002 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497012+0000", 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2069000000000002e-05 2023-06-10T21:50:53.393 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497032+0000", 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0497999999999999e-05 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.394 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:50:53.395 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:50:53.395 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:50:53.395 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:50:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:50:52] "GET /metrics HTTP/1.1" 200 221801 "" "Prometheus/2.33.4" 2023-06-10T21:50:53.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:53.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:53.397 INFO:tasks.workunit.client.0.smithi064.stderr:Updating files: 95% (10598/11140) Updating files: 96% (10695/11140) Updating files: 97% (10806/11140) Updating files: 98% (10918/11140) Updating files: 99% (11029/11140) Updating files: 100% (11140/11140) Updating files: 100% (11140/11140), done. 2023-06-10T21:50:53.409 DEBUG:teuthology.orchestra.run.smithi064:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-06-10T21:50:53.414 INFO:tasks.thrashosds.thrasher:in_osds: [3, 7, 2, 0, 6] out_osds: [4, 1, 5] dead_osds: [4] live_osds: [1, 2, 6, 7, 0, 3, 5] 2023-06-10T21:50:53.415 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-10T21:50:53.415 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-10T21:50:53.415 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.554824 2023-06-10T21:50:53.415 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 3 0.5548240388950701 2023-06-10T21:50:53.497 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:53.525 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:53.602 INFO:tasks.workunit.client.0.smithi064.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-06-10T21:50:53.681 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-06-10T21:50:53.682 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-06-10T21:50:53.724 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:50:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:53.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:50:53.828 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:53.855 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:53.900 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:53.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:54.004 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:54.036 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.085 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:54.193 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:54.221 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.266 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:54.371 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:54.397 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.442 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:54.520 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-10T21:50:54.548 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:50:54.574 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.616 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:50:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:50:54.719 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:50:54.745 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.786 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:50:54.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:50:54.894 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:54.919 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:54.960 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:54.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:55.066 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:55.096 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:55.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:55 smithi064 ceph-mon[186706]: pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:55.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:55 smithi064 ceph-mon[180297]: pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:55.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:55 smithi152 ceph-mon[156819]: pgmap v1781: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:57.203 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-10T21:50:57.282 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:57.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:57.318 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-06-10T21:50:57.386 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:50:57.396 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-06-10T21:50:57.415 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:57.436 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-06-10T21:50:57.438 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-06-10T21:50:57.438 INFO:tasks.workunit.client.0.smithi064.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-06-10T21:50:57.459 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:57.462 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:50:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:57.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:50:57.495 INFO:tasks.workunit.client.0.smithi064.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-06-10T21:50:57.549 DEBUG:teuthology.orchestra.run.smithi064:> set -ex 2023-06-10T21:50:57.549 DEBUG:teuthology.orchestra.run.smithi064:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-06-10T21:50:57.559 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:57 smithi064 ceph-mon[180297]: pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:57.560 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:57 smithi064 ceph-mon[186706]: pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:57.562 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:50:57.569 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:57.607 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:57.617 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-06-10T21:50:57.618 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-06-10T21:50:57.618 DEBUG:teuthology.orchestra.run.smithi064:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-06-10T21:50:57.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:57 smithi152 ceph-mon[156819]: pgmap v1782: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:57.728 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:50:57.754 INFO:tasks.workunit.client.0.smithi064.stdout:Running main() from gmock_main.cc 2023-06-10T21:50:57.755 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] Running 70 tests from 1 test suite. 2023-06-10T21:50:57.755 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment set-up. 2023-06-10T21:50:57.755 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd 2023-06-10T21:50:57.859 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:57.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:57.965 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:50:57.999 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:58.113 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:50:58.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:58.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:50:58.219 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:50:58.250 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:58.431 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:50:58.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/411067616' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:58.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:58 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/411067616' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:58.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:58 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:58.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/411067616' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:58.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:58 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:50:59.038 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:50:59.038 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:50:59.039 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:50:59.039 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:50:59.439 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_all_features 2023-06-10T21:50:59.451 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_all_features (12 ms) 2023-06-10T21:50:59.451 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.copyup 2023-06-10T21:50:59.506 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:50:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:50:59.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:50:59.611 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:50:59.640 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:50:59.642 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.copyup (191 ms) 2023-06-10T21:50:59.642 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-06-10T21:50:59.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:59.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:59.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:50:59.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.644 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.645 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:59.645 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: osdmap e496: 8 total, 7 up, 5 in 2023-06-10T21:50:59.645 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2475202670' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.645 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:50:59 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.659 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2023-06-10T21:50:59.659 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-06-10T21:50:59.661 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-10T21:50:59.668 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-06-10T21:50:59.668 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-06-10T21:50:59.699 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.add_remove_child (31 ms) 2023-06-10T21:50:59.699 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.directory_methods 2023-06-10T21:50:59.727 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2023-06-10T21:50:59.728 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.create 2023-06-10T21:50:59.742 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.create (15 ms) 2023-06-10T21:50:59.742 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_features 2023-06-10T21:50:59.743 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-06-10T21:50:59.744 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-06-10T21:50:59.745 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2023-06-10T21:50:59.745 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-06-10T21:50:59.746 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-06-10T21:50:59.747 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-06-10T21:50:59.748 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-06-10T21:50:59.748 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-06-10T21:50:59.749 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-06-10T21:50:59.749 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-06-10T21:50:59.753 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-06-10T21:50:59.753 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.get_size 2023-06-10T21:50:59.757 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-06-10T21:50:59.757 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_size 2023-06-10T21:50:59.763 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-06-10T21:50:59.763 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.protection_status 2023-06-10T21:50:59.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:59.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:59.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:50:59.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.768 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: osdmap e496: 8 total, 7 up, 5 in 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2475202670' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: pgmap v1783: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:50:59.769 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: failed to encode map e496 with expected crc 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: osdmap e496: 8 total, 7 up, 5 in 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2475202670' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.770 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:50:59 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-10T21:50:59.782 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.protection_status (20 ms) 2023-06-10T21:50:59.783 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-06-10T21:50:59.791 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-06-10T21:50:59.791 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v1 2023-06-10T21:50:59.840 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v1 (49 ms) 2023-06-10T21:50:59.840 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.parents_v2 2023-06-10T21:50:59.869 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-06-10T21:50:59.870 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapshots 2023-06-10T21:50:59.891 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-06-10T21:50:59.892 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.snapid_race 2023-06-10T21:50:59.898 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2023-06-10T21:50:59.898 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.stripingv2 2023-06-10T21:50:59.907 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2023-06-10T21:50:59.907 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_save 2023-06-10T21:50:59.910 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2023-06-10T21:50:59.911 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-06-10T21:50:59.915 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-06-10T21:50:59.916 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_update 2023-06-10T21:50:59.925 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_update (10 ms) 2023-06-10T21:50:59.926 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-06-10T21:50:59.926 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-06-10T21:50:59.926 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-06-10T21:50:59.931 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-06-10T21:50:59.931 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-06-10T21:50:59.939 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-06-10T21:50:59.939 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.flags 2023-06-10T21:50:59.945 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-06-10T21:50:59.945 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.metadata 2023-06-10T21:50:59.972 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.metadata (28 ms) 2023-06-10T21:50:59.973 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.set_features 2023-06-10T21:50:59.980 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-06-10T21:50:59.981 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror 2023-06-10T21:51:00.018 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror (37 ms) 2023-06-10T21:51:00.018 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image 2023-06-10T21:51:00.024 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:00.034 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image (17 ms) 2023-06-10T21:51:00.035 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-06-10T21:51:00.152 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:51:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:00.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:51:00.261 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:51:00.301 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:00.365 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:51:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:00.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:51:00.469 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:51:00.508 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:00.552 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-10T21:51:00.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: osdmap e497: 8 total, 7 up, 5 in 2023-06-10T21:51:00.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.553 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:00 smithi152 ceph-mon[156819]: pgmap v1786: 105 pgs: 18 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:00.621 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:51:00.627 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:51:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:00.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:51:00.653 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:51:00.677 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:51:00.706 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:51:00.732 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:51:00.733 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:51:00.757 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:51:00.761 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:00.782 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:51:00.782 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.782 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.783 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-10T21:51:00.795 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-10T21:51:00.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: osdmap e497: 8 total, 7 up, 5 in 2023-06-10T21:51:00.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[186706]: pgmap v1786: 105 pgs: 18 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi064-622489-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: osdmap e497: 8 total, 7 up, 5 in 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: failed to encode map e497 with expected crc 2023-06-10T21:51:00.797 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:00 smithi064 ceph-mon[180297]: pgmap v1786: 105 pgs: 18 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:01.054 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:51:01.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:01.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:51:01.158 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:51:01.195 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:01.261 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:51:01.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:01.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:51:01.365 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:51:01.394 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:01.447 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:51:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.449 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: osdmap e498: 8 total, 7 up, 5 in 2023-06-10T21:51:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3557759304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2013099668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:01.450 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:01.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:51:01.462 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.3 primary-affinity to 0.554824 (8363602) 2023-06-10T21:51:01.534 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:51:01.534 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:51:01.552 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:51:01.576 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: osdmap e498: 8 total, 7 up, 5 in 2023-06-10T21:51:01.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3557759304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:01.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2013099668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:01.577 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:01 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:01.579 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:01.725 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:51:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:01.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:51:01.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:51:01.855 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:01.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: failed to encode map e498 with expected crc 2023-06-10T21:51:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: osdmap e498: 8 total, 7 up, 5 in 2023-06-10T21:51:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3557759304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2013099668' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:01.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:01 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]: dispatch 2023-06-10T21:51:02.139 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:51:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:02.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:51:02.151 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd set noscrub 2023-06-10T21:51:02.191 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-10T21:51:02.243 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:51:02.270 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:51:02.307 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:51:02.311 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:51:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:02.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:51:02.419 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:51:02.452 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:02.511 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:51:02.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:02.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:51:02.557 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:02.614 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:51:02.642 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:02.735 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.736 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.736 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.736 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]': finished 2023-06-10T21:51:02.736 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: osdmap e499: 8 total, 7 up, 5 in 2023-06-10T21:51:02.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.737 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:02 smithi152 ceph-mon[156819]: pgmap v1789: 105 pgs: 3 peering, 18 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 127 KiB/s rd, 34 KiB/s wr, 178 op/s 2023-06-10T21:51:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]': finished 2023-06-10T21:51:02.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: osdmap e499: 8 total, 7 up, 5 in 2023-06-10T21:51:02.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[180297]: pgmap v1789: 105 pgs: 3 peering, 18 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 127 KiB/s rd, 34 KiB/s wr, 178 op/s 2023-06-10T21:51:02.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.5548240388950701}]': finished 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: osdmap e499: 8 total, 7 up, 5 in 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: failed to encode map e499 with expected crc 2023-06-10T21:51:02.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:02 smithi064 ceph-mon[186706]: pgmap v1789: 105 pgs: 3 peering, 18 creating+peering, 3 unknown, 81 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 127 KiB/s rd, 34 KiB/s wr, 178 op/s 2023-06-10T21:51:03.001 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:51:03.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:03.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:51:03.106 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:51:03.144 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:02] "GET /metrics HTTP/1.1" 200 221801 "" "Prometheus/2.33.4" 2023-06-10T21:51:03.148 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:03.190 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_status (3155 ms) 2023-06-10T21:51:03.190 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-06-10T21:51:03.206 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:03.212 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:51:03.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:03.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:51:03.310 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_image_map (120 ms) 2023-06-10T21:51:03.310 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-06-10T21:51:03.317 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-06-10T21:51:03.318 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-06-10T21:51:03.321 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:51:03.324 INFO:tasks.workunit.client.0.smithi064.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/16.2.13-257-gd8c5d349/rpm/el8/BUILD/ceph-16.2.13-257-gd8c5d349/src/test/cls_rbd/test_cls_rbd.cc:2279: Failure 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout:Expected equality of these values: 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout: -85 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout: mirror_image_snapshot_unlink_peer(&ioctx, oid, 1, "peer2") 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout: Which is: 0 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] TestClsRbd.mirror_snapshot (6 ms) 2023-06-10T21:51:03.325 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-06-10T21:51:03.326 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-06-10T21:51:03.326 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-06-10T21:51:03.328 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-06-10T21:51:03.329 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-06-10T21:51:03.332 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-06-10T21:51:03.332 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-06-10T21:51:03.338 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-06-10T21:51:03.339 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-06-10T21:51:03.342 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-06-10T21:51:03.342 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-06-10T21:51:03.346 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-06-10T21:51:03.346 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_add 2023-06-10T21:51:03.348 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-06-10T21:51:03.348 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-06-10T21:51:03.351 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-06-10T21:51:03.352 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_list 2023-06-10T21:51:03.352 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:03.355 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-06-10T21:51:03.355 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-06-10T21:51:03.359 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-06-10T21:51:03.359 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_add 2023-06-10T21:51:03.362 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-06-10T21:51:03.362 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-06-10T21:51:03.365 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-06-10T21:51:03.365 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.image_group_get 2023-06-10T21:51:03.368 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-06-10T21:51:03.368 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-06-10T21:51:03.370 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-06-10T21:51:03.370 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-06-10T21:51:03.372 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (1 ms) 2023-06-10T21:51:03.372 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-06-10T21:51:03.375 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-06-10T21:51:03.375 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-06-10T21:51:03.378 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-06-10T21:51:03.379 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-06-10T21:51:03.380 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-06-10T21:51:03.380 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-06-10T21:51:03.383 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-06-10T21:51:03.383 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-06-10T21:51:03.401 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (17 ms) 2023-06-10T21:51:03.401 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-06-10T21:51:03.435 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:51:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:03.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:51:03.539 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:51:03.569 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:03.617 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:51:03.618 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:51:03.618 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: osdmap e500: 8 total, 7 up, 5 in 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:03.619 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:03.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:51:03.631 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (231 ms) 2023-06-10T21:51:03.632 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-06-10T21:51:03.634 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-06-10T21:51:03.635 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-06-10T21:51:03.637 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2023-06-10T21:51:03.637 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.trash_methods 2023-06-10T21:51:03.644 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-06-10T21:51:03.645 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.op_features 2023-06-10T21:51:03.652 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-06-10T21:51:03.652 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent 2023-06-10T21:51:03.674 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent (22 ms) 2023-06-10T21:51:03.674 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-06-10T21:51:03.683 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_parent_ns (10 ms) 2023-06-10T21:51:03.684 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.clone_child 2023-06-10T21:51:03.696 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-06-10T21:51:03.697 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-06-10T21:51:03.704 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.namespace_methods (8 ms) 2023-06-10T21:51:03.705 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration 2023-06-10T21:51:03.715 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-06-10T21:51:03.715 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.migration_v1 2023-06-10T21:51:03.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:51:03.724 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2023-06-10T21:51:03.724 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-06-10T21:51:03.751 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:03.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:51:03.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.780 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: osdmap e500: 8 total, 7 up, 5 in 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:03.781 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: osdmap e500: 8 total, 7 up, 5 in 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: failed to encode map e500 with expected crc 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:03.782 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:03.895 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:51:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:03.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:51:03.907 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:51:03.907 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:51:03.999 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 2023-06-10T21:51:04.029 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:04.075 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1 found: no such container 2023-06-10T21:51:04.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:04.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.1' 2023-06-10T21:51:04.178 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:51:04.206 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:04.308 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-10T21:51:04.502 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:51:04.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:04.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:51:04.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:51:04.639 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:04.656 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-10T21:51:04.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:04 smithi064 ceph-mon[186706]: pgmap v1791: 105 pgs: 3 peering, 4 creating+peering, 1 unknown, 97 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.90k op/s 2023-06-10T21:51:04.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:04 smithi064 ceph-mon[180297]: pgmap v1791: 105 pgs: 3 peering, 4 creating+peering, 1 unknown, 97 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.90k op/s 2023-06-10T21:51:04.685 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:51:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:04.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:51:04.723 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:04.790 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:51:04.813 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:04 smithi152 ceph-mon[156819]: pgmap v1791: 105 pgs: 3 peering, 4 creating+peering, 1 unknown, 97 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.90k op/s 2023-06-10T21:51:04.816 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:04.860 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:51:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:04.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:51:04.965 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:51:04.993 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:05.040 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:51:05.042 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:05.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:51:05.145 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 2023-06-10T21:51:05.178 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:05.277 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2 found: no such container 2023-06-10T21:51:05.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:05.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.2' 2023-06-10T21:51:05.380 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 2023-06-10T21:51:05.409 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:05.469 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1746 ms) 2023-06-10T21:51:05.470 INFO:tasks.workunit.client.0.smithi064.stdout:[ RUN ] TestClsRbd.sparsify 2023-06-10T21:51:05.475 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is set 2023-06-10T21:51:05.511 INFO:tasks.workunit.client.0.smithi064.stdout:[ OK ] TestClsRbd.sparsify (41 ms) 2023-06-10T21:51:05.518 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: osdmap e501: 8 total, 7 up, 5 in 2023-06-10T21:51:05.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3276929656' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:05.519 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:05 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: osdmap e501: 8 total, 7 up, 5 in 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3276929656' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: osdmap e501: 8 total, 7 up, 5 in 2023-06-10T21:51:05.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: failed to encode map e501 with expected crc 2023-06-10T21:51:05.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3276929656' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:05.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:05 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-10T21:51:06.057 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3 found: no such container 2023-06-10T21:51:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:06.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.3' 2023-06-10T21:51:06.161 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 2023-06-10T21:51:06.191 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:06.238 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7 found: no such container 2023-06-10T21:51:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:06.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.7' 2023-06-10T21:51:06.344 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 2023-06-10T21:51:06.372 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:06.420 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6 found: no such container 2023-06-10T21:51:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:06.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.6' 2023-06-10T21:51:06.440 INFO:teuthology.orchestra.run.smithi152.stdout:ERROR: (22) Invalid argument 2023-06-10T21:51:06.440 INFO:teuthology.orchestra.run.smithi152.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-10T21:51:06.479 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] 70 tests from TestClsRbd (6072 ms total) 2023-06-10T21:51:06.479 INFO:tasks.workunit.client.0.smithi064.stdout: 2023-06-10T21:51:06.479 INFO:tasks.workunit.client.0.smithi064.stdout:[----------] Global test environment tear-down 2023-06-10T21:51:06.479 INFO:tasks.workunit.client.0.smithi064.stdout:[==========] 70 tests from 1 test suite ran. (8724 ms total) 2023-06-10T21:51:06.480 INFO:tasks.workunit.client.0.smithi064.stdout:[ PASSED ] 69 tests. 2023-06-10T21:51:06.480 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] 1 test, listed below: 2023-06-10T21:51:06.480 INFO:tasks.workunit.client.0.smithi064.stdout:[ FAILED ] TestClsRbd.mirror_snapshot 2023-06-10T21:51:06.480 INFO:tasks.workunit.client.0.smithi064.stdout: 2023-06-10T21:51:06.480 INFO:tasks.workunit.client.0.smithi064.stdout: 1 FAILED TEST 2023-06-10T21:51:06.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T21:51:06.481 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-06-10T21:51:06.482 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-06-10T21:51:06.524 DEBUG:teuthology.orchestra.run.smithi152:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 2023-06-10T21:51:06.551 INFO:teuthology.orchestra.run.smithi152.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:06.648 INFO:teuthology.orchestra.run.smithi152.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5 found: no such container 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: osdmap e502: 8 total, 7 up, 5 in 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.650 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.651 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:06 smithi152 ceph-mon[156819]: pgmap v1794: 105 pgs: 3 peering, 102 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.91k op/s 2023-06-10T21:51:06.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:06.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi152 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.5' 2023-06-10T21:51:06.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 2023-06-10T21:51:06.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:51:06.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.753 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:51:06.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: osdmap e502: 8 total, 7 up, 5 in 2023-06-10T21:51:06.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[186706]: pgmap v1794: 105 pgs: 3 peering, 102 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.91k op/s 2023-06-10T21:51:06.754 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: osdmap e502: 8 total, 7 up, 5 in 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: failed to encode map e502 with expected crc 2023-06-10T21:51:06.755 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:06 smithi064 ceph-mon[180297]: pgmap v1794: 105 pgs: 3 peering, 102 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 2.91k op/s 2023-06-10T21:51:06.808 INFO:tasks.thrashosds:joining thrashosds 2023-06-10T21:51:06.811 INFO:teuthology.orchestra.run.smithi064.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-06-10T21:51:06.960 INFO:teuthology.orchestra.run.smithi064.stderr:Error: no container with name or ID ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0 found: no such container 2023-06-10T21:51:06.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-06-10T21:51:06.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi064 with status 125: 'sudo docker kill -s 1 ceph-b618768e-07d0-11ee-9b1e-001a4aab830c-osd.0' 2023-06-10T21:51:06.982 INFO:teuthology.orchestra.run.smithi064.stdout:osd.0: {} 2023-06-10T21:51:07.000 INFO:teuthology.orchestra.run.smithi064.stdout:osd.1: {} 2023-06-10T21:51:07.018 INFO:teuthology.orchestra.run.smithi064.stdout:osd.2: {} 2023-06-10T21:51:07.037 INFO:teuthology.orchestra.run.smithi064.stdout:osd.3: {} 2023-06-10T21:51:07.055 INFO:teuthology.orchestra.run.smithi064.stdout:osd.5: {} 2023-06-10T21:51:07.073 INFO:teuthology.orchestra.run.smithi064.stdout:osd.6: {} 2023-06-10T21:51:07.091 INFO:teuthology.orchestra.run.smithi064.stdout:osd.7: {} 2023-06-10T21:51:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.091 INFO:teuthology.orchestra.run.smithi064.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-10T21:51:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.092 INFO:teuthology.orchestra.run.smithi064.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-10T21:51:07.142 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-10T21:51:07.493 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:07.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:07 smithi152 ceph-mon[156819]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:07 smithi152 ceph-mon[156819]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:07 smithi152 ceph-mon[156819]: osdmap e503: 8 total, 7 up, 5 in 2023-06-10T21:51:07.733 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/447159191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:07.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[180297]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[180297]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[180297]: osdmap e503: 8 total, 7 up, 5 in 2023-06-10T21:51:07.872 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/447159191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:07.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[186706]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[186706]: failed to encode map e503 with expected crc 2023-06-10T21:51:07.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[186706]: osdmap e503: 8 total, 7 up, 5 in 2023-06-10T21:51:07.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/447159191' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-10T21:51:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-10T21:51:07.887 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-10T21:51:08.155 INFO:tasks.thrashosds.thrasher:reviving osd 2023-06-10T21:51:08.155 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-06-10T21:51:08.155 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-06-10T21:51:08.156 DEBUG:teuthology.orchestra.run.smithi152:osd.4> sudo journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4.service 2023-06-10T21:51:08.159 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl start ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4 2023-06-10T21:51:08.188 INFO:journalctl@ceph.osd.4.smithi152.stdout:-- Logs begin at Sat 2023-06-10 20:37:59 UTC. -- 2023-06-10T21:51:08.443 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:08 smithi152 systemd[1]: Starting Ceph osd.4 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:51:08.464 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset noscrub 2023-06-10T21:51:08.723 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:08 smithi152 ceph-mon[156819]: failed to encode map e503 with expected crc 2023-06-10T21:51:08.724 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:08 smithi152 ceph-mon[156819]: pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:08.843 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:08 smithi064 ceph-mon[186706]: failed to encode map e503 with expected crc 2023-06-10T21:51:08.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:08 smithi064 ceph-mon[186706]: pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:08.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:08 smithi064 ceph-mon[180297]: failed to encode map e503 with expected crc 2023-06-10T21:51:08.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:08 smithi064 ceph-mon[180297]: pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:08.935 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:08.935 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:08.936 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:51:08.936 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:51:08.936 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:09.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:09 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:51:09.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:09 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:09.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:09 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:51:09.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:09 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:09.686 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-10T21:51:09.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:09 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-06-10T21:51:09.815 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:09 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:10.193 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-10T21:51:10.193 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-10T21:51:10.194 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-10T21:51:10.194 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-10T21:51:10.194 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-06-10T21:51:10.194 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-10T21:51:10.194 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:09 smithi152 bash[373430]: --> ceph-volume raw activate successful for osd ID: 4 2023-06-10T21:51:10.549 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:10.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:10 smithi152 ceph-mon[156819]: pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:10.770 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:10 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:51:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:10 smithi064 ceph-mon[180297]: pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:10.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:10 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:51:10.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:10 smithi064 ceph-mon[186706]: pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:10.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:10 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:51:11.428 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:51:11.454 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:11 smithi152 conmon[374343]: debug 2023-06-10T21:51:11.252+0000 7f95375df700 1 -- 172.21.15.152:0/3760106994 <== mon.1 v2:172.21.15.152:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556f7c5f84e0 con 0x556f7c5bf000 2023-06-10T21:51:11.455 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:11 smithi152 bash[374209]: ffc3bc582f6d2b076d17008e2757a4724c7c04f33ef9b4b6caabb9e373a72514 2023-06-10T21:51:11.455 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:11 smithi152 systemd[1]: Started Ceph osd.4 for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:51:11.494 INFO:teuthology.orchestra.run.smithi064.stderr:noscrub is unset 2023-06-10T21:51:11.749 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2880072438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:11.749 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:11 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:11.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2880072438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:11.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:11 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:11.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2880072438' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:11.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:11 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-10T21:51:12.028 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:11 smithi152 conmon[374343]: debug 2023-06-10T21:51:11.770+0000 7f953f05b200 -1 Falling back to public interface 2023-06-10T21:51:12.099 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:12.199 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:12.199 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:51:12.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:51:12.199 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177346 14.f 14:fe135908:::benchmark_data_smithi064_559916_object69490:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.498427+0000", 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.69586418900001, 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96249369600000001, 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.200 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177346 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498427+0000", 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498427+0000", 2023-06-10T21:51:12.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498425+0000", 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3249999999999996e-06 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.203 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498431+0000", 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2585999999999999e-05 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498444+0000", 2023-06-10T21:51:12.204 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4370000000000004e-06 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498450+0000", 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8995999999999997e-05 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498489+0000", 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.8720000000000008e-06 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498498+0000", 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95737534700000004 2023-06-10T21:51:12.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.455874+0000", 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000119123 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.455993+0000", 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6605000000000001e-05 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.456089+0000", 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0026952030000000002 2023-06-10T21:51:12.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.458785+0000", 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0020479019999999999 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.460833+0000", 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1353e-05 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.460844+0000", 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0036e-05 2023-06-10T21:51:12.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.460894+0000", 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6580999999999998e-05 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.460920+0000", 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96249369600000001 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177349 14.f 14:f245d3a6:::benchmark_data_smithi064_559916_object69493:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.212 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.498478+0000", 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.695813401, 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96504469500000001, 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.213 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177349 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498478+0000", 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498478+0000", 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498477+0000", 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7740000000000001e-06 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498479+0000", 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4829999999999999e-06 2023-06-10T21:51:12.216 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498483+0000", 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.4389999999999999e-06 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498485+0000", 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6439000000000001e-05 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498521+0000", 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2749999999999998e-06 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.498529+0000", 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96029796499999998 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.458827+0000", 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000110479 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.458937+0000", 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010032200000000001 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.459037+0000", 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0042551189999999999 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463293+0000", 2023-06-10T21:51:12.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4808999999999994e-05 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463377+0000", 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4168e-05 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463432+0000", 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7877999999999998e-05 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463479+0000", 2023-06-10T21:51:12.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2803000000000003e-05 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463522+0000", 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96504469500000001 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.224 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177367 14.f 14:f1102030:::benchmark_data_smithi064_559916_object69511:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.519348+0000", 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.674942706, 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95024309500000004, 2023-06-10T21:51:12.225 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177367 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519348+0000", 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519348+0000", 2023-06-10T21:51:12.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519347+0000", 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3739999999999997e-06 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.228 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519352+0000", 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3540000000000006e-06 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519361+0000", 2023-06-10T21:51:12.229 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2329999999999997e-06 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519367+0000", 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4956999999999997e-05 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519412+0000", 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2099999999999999e-06 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.519421+0000", 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94154490999999996 2023-06-10T21:51:12.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.460966+0000", 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6398999999999997e-05 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.461052+0000", 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2638000000000004e-05 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.461135+0000", 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0024474179999999998 2023-06-10T21:51:12.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463582+0000", 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3859000000000001e-05 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463596+0000", 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059063730000000004 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469502+0000", 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3176000000000001e-05 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469546+0000", 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5708999999999999e-05 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469591+0000", 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95024309500000004 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177369 14.f 14:f46c683a:::benchmark_data_smithi064_559916_object69513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.237 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.520555+0000", 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.673735865, 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94918240600000003, 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.238 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177369 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520555+0000", 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520555+0000", 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520554+0000", 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9779999999999996e-06 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520559+0000", 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2027e-05 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520571+0000", 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2089999999999996e-06 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.242 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520576+0000", 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7305999999999997e-05 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520623+0000", 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1640000000000001e-06 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.520632+0000", 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94299078599999997 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463622+0000", 2023-06-10T21:51:12.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114003 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463736+0000", 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00014228299999999999 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.463879+0000", 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0057701560000000002 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469649+0000", 2023-06-10T21:51:12.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3462e-05 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469672+0000", 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9089999999999997e-06 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469678+0000", 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5542e-05 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469704+0000", 2023-06-10T21:51:12.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3859000000000002e-05 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469738+0000", 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94918240600000003 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.249 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177405 14.f 14:ffd91696:::benchmark_data_smithi064_559916_object69549:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.577187+0000", 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.61710439300001, 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90054229500000005, 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.250 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177405 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577187+0000", 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577187+0000", 2023-06-10T21:51:12.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577185+0000", 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1189999999999997e-06 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.253 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577193+0000", 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2476999999999999e-05 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577206+0000", 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6869999999999997e-06 2023-06-10T21:51:12.254 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577212+0000", 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3801000000000002e-05 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577266+0000", 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0013000000000001e-05 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.577276+0000", 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88922541600000005 2023-06-10T21:51:12.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.466501+0000", 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122836 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.466624+0000", 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000154127 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.466778+0000", 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029658710000000001 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469744+0000", 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6489999999999997e-05 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469791+0000", 2023-06-10T21:51:12.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0078477319999999996 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477639+0000", 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7048000000000001e-05 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477686+0000", 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3275999999999997e-05 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477729+0000", 2023-06-10T21:51:12.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.90054229500000005 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177420 14.f 14:fe3b3b08:::benchmark_data_smithi064_559916_object69564:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.594671+0000", 2023-06-10T21:51:12.262 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.599620149, 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88325085800000003, 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.263 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177420 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594671+0000", 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594671+0000", 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594670+0000", 2023-06-10T21:51:12.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.232e-06 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594673+0000", 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2499999999999997e-06 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594678+0000", 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3979999999999999e-06 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594681+0000", 2023-06-10T21:51:12.267 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7042999999999997e-05 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594728+0000", 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7980000000000002e-06 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.594734+0000", 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87508423400000002 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469818+0000", 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9974000000000005e-05 2023-06-10T21:51:12.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469888+0000", 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3644000000000001e-05 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.469972+0000", 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077723280000000002 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477744+0000", 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3728e-05 2023-06-10T21:51:12.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477778+0000", 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7478999999999999e-05 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477806+0000", 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8328e-05 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477874+0000", 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7783e-05 2023-06-10T21:51:12.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477922+0000", 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.88325085800000003 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177428 14.f 14:f7a7ad52:::benchmark_data_smithi064_559916_object69572:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.608377+0000", 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.585913803, 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87523302599999997, 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.275 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177428 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.276 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608377+0000", 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608377+0000", 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.277 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608376+0000", 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3600000000000004e-06 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608381+0000", 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0669999999999999e-05 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608392+0000", 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8760000000000004e-06 2023-06-10T21:51:12.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608397+0000", 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2197999999999998e-05 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608439+0000", 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2749999999999995e-06 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.608447+0000", 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86424288800000004 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.472690+0000", 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012641 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.472817+0000", 2023-06-10T21:51:12.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000167601 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.472984+0000", 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047664409999999997 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477751+0000", 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0056954800000000002 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483446+0000", 2023-06-10T21:51:12.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5759000000000004e-05 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483522+0000", 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8114e-05 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483570+0000", 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0259000000000002e-05 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483610+0000", 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.87523302599999997 2023-06-10T21:51:12.286 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177437 14.f 14:fe3b0f8f:::benchmark_data_smithi064_559916_object69581:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.616613+0000", 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.577677866, 2023-06-10T21:51:12.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86713079800000004, 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177437 2023-06-10T21:51:12.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616613+0000", 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616613+0000", 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616611+0000", 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2120000000000003e-06 2023-06-10T21:51:12.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616620+0000", 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2194999999999999e-05 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.291 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616632+0000", 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5910000000000003e-06 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616637+0000", 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4863000000000001e-05 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.292 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616682+0000", 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8140000000000003e-06 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.616689+0000", 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.85861392199999997 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.475303+0000", 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010797400000000001 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.475411+0000", 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9753000000000006e-05 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.475501+0000", 2023-06-10T21:51:12.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049926550000000004 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.480493+0000", 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0029926000000000002 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483486+0000", 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000186145 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483672+0000", 2023-06-10T21:51:12.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.8661000000000001e-05 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483711+0000", 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3042e-05 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483744+0000", 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86713079800000004 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.299 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177521 14.f 14:fbdd44c8:::benchmark_data_smithi064_559916_object69665:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:29.838120+0000", 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.356171198, 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.66601885800000005, 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.300 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177521 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838120+0000", 2023-06-10T21:51:12.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838120+0000", 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838118+0000", 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.525e-06 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838126+0000", 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3056e-05 2023-06-10T21:51:12.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838139+0000", 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.455e-06 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.304 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838145+0000", 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4616999999999995e-05 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838210+0000", 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4509999999999992e-06 2023-06-10T21:51:12.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:29.838218+0000", 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.63973985300000002 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.477958+0000", 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114839 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.478073+0000", 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013832800000000001 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.478211+0000", 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0052827320000000001 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483494+0000", 2023-06-10T21:51:12.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000147644 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483642+0000", 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.020414317000000001 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504056+0000", 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.3519999999999997e-05 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504099+0000", 2023-06-10T21:51:12.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9249999999999999e-05 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504139+0000", 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.66601885800000005 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.311 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177577 14.f 14:fbeeab3d:::benchmark_data_smithi064_559916_object69721:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:30.096493+0000", 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.09779838599999, 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.40775320900000001, 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.312 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177577 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096493+0000", 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096493+0000", 2023-06-10T21:51:12.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096491+0000", 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6930000000000007e-06 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096499+0000", 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7697e-05 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096517+0000", 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9800000000000001e-06 2023-06-10T21:51:12.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096524+0000", 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5629000000000001e-05 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.317 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096559+0000", 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0090000000000007e-06 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.096567+0000", 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.38395512199999998 2023-06-10T21:51:12.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.480522+0000", 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011264000000000001 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.480635+0000", 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012919599999999999 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.480764+0000", 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0028830420000000002 2023-06-10T21:51:12.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.483647+0000", 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0086171279999999999 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.492264+0000", 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011913378000000001 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504178+0000", 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5664000000000003e-05 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504213+0000", 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.2475000000000003e-05 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504246+0000", 2023-06-10T21:51:12.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.40775320900000001 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:177581 14.f 14:f63b8158:::benchmark_data_smithi064_559916_object69725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e464)", 2023-06-10T21:51:12.324 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:49:30.173585+0000", 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 102.020705785, 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33085298800000001, 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.325 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 177581 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173585+0000", 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173585+0000", 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173583+0000", 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0168e-05 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173593+0000", 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8134e-05 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173611+0000", 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0150000000000003e-06 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173619+0000", 2023-06-10T21:51:12.329 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1144999999999999e-05 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173671+0000", 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.6360000000000006e-06 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.330 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.173680+0000", 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.31033122899999999 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.484011+0000", 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104052 2023-06-10T21:51:12.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.484115+0000", 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000170824 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.484286+0000", 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0080053550000000005 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.492292+0000", 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0519999999999998e-06 2023-06-10T21:51:12.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.492298+0000", 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012052495 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504350+0000", 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6575e-05 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504397+0000", 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.142e-05 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:49:30.504438+0000", 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.33085298800000001 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:199780 14.f 14:fc1712bc:::benchmark_data_smithi064_559916_object91924:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.003622+0000", 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.190668764999998, 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25222569099999997, 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.337 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199780 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003622+0000", 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003622+0000", 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003621+0000", 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5840000000000002e-06 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003627+0000", 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1477e-05 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003638+0000", 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2729999999999999e-06 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003644+0000", 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4768000000000002e-05 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003689+0000", 2023-06-10T21:51:12.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105779 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003795+0000", 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120412 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003915+0000", 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010405770999999999 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014321+0000", 2023-06-10T21:51:12.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021003630000000001 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.016421+0000", 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23932532400000001 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.345 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255747+0000", 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0022000000000003e-05 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255797+0000", 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1174000000000002e-05 2023-06-10T21:51:12.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255848+0000", 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25222569099999997 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.347 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:199781 14.f 14:f091c53f:::benchmark_data_smithi064_559916_object91925:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.003657+0000", 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.190634408999998, 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25238070200000001, 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.348 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199781 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003657+0000", 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003657+0000", 2023-06-10T21:51:12.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003656+0000", 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6929999999999999e-06 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003659+0000", 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.677e-06 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003664+0000", 2023-06-10T21:51:12.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5410000000000001e-06 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003668+0000", 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048771320000000002 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.008545+0000", 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012460499999999999 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.008670+0000", 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015006399999999999 2023-06-10T21:51:12.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.008820+0000", 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24684029599999999 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255660+0000", 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5213000000000002e-05 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255705+0000", 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00023801000000000001 2023-06-10T21:51:12.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255943+0000", 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4302999999999998e-05 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.357 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255998+0000", 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9637e-05 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256037+0000", 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25238070200000001 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.358 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:199784 14.f 14:ff532426:::benchmark_data_smithi064_559916_object91928:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.003684+0000", 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.190606610000003, 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25252151299999998, 2023-06-10T21:51:12.359 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199784 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.360 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003684+0000", 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003684+0000", 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003684+0000", 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5390000000000002e-06 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003686+0000", 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4149999999999998e-06 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003692+0000", 2023-06-10T21:51:12.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.2400000000000001e-06 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.003696+0000", 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077784760000000003 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.011475+0000", 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000117142 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.011592+0000", 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013681300000000001 2023-06-10T21:51:12.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.011729+0000", 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24394178799999999 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255670+0000", 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00023593099999999999 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255906+0000", 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022112600000000001 2023-06-10T21:51:12.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256127+0000", 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1579000000000003e-05 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256169+0000", 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6859000000000001e-05 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256206+0000", 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.25252151299999998 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.369 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:199786 14.d e474/460 14:b8108ed3:::benchmark_data_smithi064_559916_object91930:head v 474'12550, mlcod=474'12547)", 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.010045+0000", 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.184245705000002, 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24299239, 2023-06-10T21:51:12.370 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010045+0000", 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010045+0000", 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010045+0000", 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6160000000000004e-06 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010050+0000", 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1530000000000002e-06 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010056+0000", 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7900000000000001e-06 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010060+0000", 2023-06-10T21:51:12.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0025081230000000001 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.012568+0000", 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.6652000000000001e-05 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.012604+0000", 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24041193799999999 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.253016+0000", 2023-06-10T21:51:12.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1432000000000001e-05 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.253038+0000", 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24299239 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.377 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:199787 14.d e474/460 14:bf36fa21:::benchmark_data_smithi064_559916_object91931:head v 474'12551, mlcod=474'12547)", 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.010855+0000", 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.183435996999997, 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24219569499999999, 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.378 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010855+0000", 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010855+0000", 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010854+0000", 2023-06-10T21:51:12.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6119999999999998e-06 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010860+0000", 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7419999999999997e-06 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.381 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010868+0000", 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.3079999999999998e-06 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.010873+0000", 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0049493920000000004 2023-06-10T21:51:12.382 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015822+0000", 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0750999999999998e-05 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015843+0000", 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23719796000000001 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.253041+0000", 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4630000000000007e-06 2023-06-10T21:51:12.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.253051+0000", 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24219569499999999 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:199792 14.7 e474/471 14:e4460b42:::benchmark_data_smithi064_559916_object91936:head v 474'12253, mlcod=474'12250)", 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.014388+0000", 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.179903381999999, 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24131409500000001, 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.386 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014388+0000", 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014388+0000", 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014386+0000", 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4602e-05 2023-06-10T21:51:12.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014401+0000", 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1372000000000001e-05 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014412+0000", 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4559999999999997e-06 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014419+0000", 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.9619999999999998e-05 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014518+0000", 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1032000000000001e-05 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014539+0000", 2023-06-10T21:51:12.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24113562499999999 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255675+0000", 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.6724e-05 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.255702+0000", 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24131409500000001 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:199793 14.f 14:f9a870b8:::benchmark_data_smithi064_559916_object91937:head [delete] snapc 0=[] ondisk+write+known_if_redirected e474)", 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.014468+0000", 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.179823138000003, 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24771062899999999, 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.394 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 199793 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014468+0000", 2023-06-10T21:51:12.395 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014468+0000", 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014467+0000", 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2039999999999996e-06 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014472+0000", 2023-06-10T21:51:12.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3400000000000004e-06 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014481+0000", 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9789999999999998e-06 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.014487+0000", 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0019561740000000002 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.016443+0000", 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116376 2023-06-10T21:51:12.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.016560+0000", 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013876000000000001 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.016698+0000", 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.239352066 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256050+0000", 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0389000000000002e-05 2023-06-10T21:51:12.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.256091+0000", 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0059961670000000002 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.262087+0000", 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0615000000000002e-05 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.262138+0000", 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0833e-05 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.262178+0000", 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.24771062899999999 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_repop(client.47150.0:199790 14.3 e474/471 14:c7766e94:::benchmark_data_smithi064_559916_object91934:head v 474'12373, mlcod=474'12371)", 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:12.015046+0000", 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 60.179244777999997, 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23780943600000001, 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.405 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015046+0000", 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.406 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015046+0000", 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015046+0000", 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6609999999999997e-06 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.407 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015051+0000", 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0770000000000002e-06 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015059+0000", 2023-06-10T21:51:12.408 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0319999999999997e-06 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015063+0000", 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5547000000000001e-05 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.409 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015128+0000", 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7781000000000001e-05 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.015146+0000", 2023-06-10T21:51:12.410 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0016419780000000001 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.016788+0000", 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.236067744 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.411 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:12.252856+0000", 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.23780943600000001 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.412 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.47150.0:213986 14.f 14:f0b379cb:::benchmark_data_smithi064_559916_object106130:head [delete] snapc 0=[] ondisk+write+known_if_redirected e486)", 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:50:36.792114+0000", 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 35.402176593, 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.032433322000000001, 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.47150", 2023-06-10T21:51:12.413 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 213986 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792114+0000", 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:12.414 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792114+0000", 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.415 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792113+0000", 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3359999999999997e-06 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792120+0000", 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3390999999999999e-05 2023-06-10T21:51:12.416 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792134+0000", 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1170000000000003e-06 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:12.417 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792139+0000", 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.0577000000000001e-05 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792179+0000", 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7762000000000004e-05 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.418 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792267+0000", 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4031999999999998e-05 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,6", 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.792351+0000", 2023-06-10T21:51:12.419 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0023756699999999999 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.794727+0000", 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000136866 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.420 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.794864+0000", 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.029579781999999999 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.824443+0000", 2023-06-10T21:51:12.421 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.8363999999999999e-05 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.824502+0000", 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5899999999999998e-05 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:12.422 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:50:36.824548+0000", 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.032433322000000001 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:12.423 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:12.424 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:12.424 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-10T21:51:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-06-10T21:51:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:51:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: osdmap e504: 8 total, 7 up, 5 in 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[186706]: pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:51:12.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:51:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: osdmap e504: 8 total, 7 up, 5 in 2023-06-10T21:51:12.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:12 smithi064 ceph-mon[180297]: pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:12.882 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: failed to encode map e504 with expected crc 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: osdmap e504: 8 total, 7 up, 5 in 2023-06-10T21:51:12.883 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:12 smithi152 ceph-mon[156819]: pgmap v1799: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:12.925 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:13.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:12] "GET /metrics HTTP/1.1" 200 224277 "" "Prometheus/2.33.4" 2023-06-10T21:51:13.608 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-10T21:51:14.382 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:0 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:1 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:2 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2023-06-10T21:51:14.383 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2023-06-10T21:51:14.384 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2023-06-10T21:51:14.384 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore bluefs device info [] 2023-06-10T21:51:14.384 INFO:teuthology.orchestra.run.smithi152.stderr:config diff 2023-06-10T21:51:14.384 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2023-06-10T21:51:14.529 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:51:15.118 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-06-10T21:51:15.479 INFO:teuthology.orchestra.run.smithi064.stderr:nodeep-scrub is unset 2023-06-10T21:51:15.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:15 smithi152 ceph-mon[156819]: pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:15.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:15 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1223921756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:15.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:15 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:15.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[180297]: pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:15.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1223921756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:15.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[186706]: pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1223921756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:15.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:15 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-10T21:51:16.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[180297]: failed to encode map e505 with expected crc 2023-06-10T21:51:16.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:51:16.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[180297]: osdmap e505: 8 total, 7 up, 5 in 2023-06-10T21:51:16.479 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[180297]: pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:16.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[186706]: failed to encode map e505 with expected crc 2023-06-10T21:51:16.872 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:51:16.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[186706]: osdmap e505: 8 total, 7 up, 5 in 2023-06-10T21:51:16.873 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:16 smithi064 ceph-mon[186706]: pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:16 smithi152 ceph-mon[156819]: failed to encode map e505 with expected crc 2023-06-10T21:51:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:16 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-10T21:51:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:16 smithi152 ceph-mon[156819]: osdmap e505: 8 total, 7 up, 5 in 2023-06-10T21:51:16.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:16 smithi152 ceph-mon[156819]: pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:17.108 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:17.108 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:51:17.108 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:51:17.109 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:51:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:51:17.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:51:17.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:51:17.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.937 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-10T21:51:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:51:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:51:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:18.303 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:19.621 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:19.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:19.622 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[180297]: pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:19.622 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:19.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:19.623 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:19 smithi064 ceph-mon[186706]: pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:19 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:19.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:19 smithi152 ceph-mon[156819]: pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:19.853 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:19.854 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:51:19.854 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:51:19.854 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:51:19.854 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:20.119 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:51:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:20.593 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-10T21:51:20.958 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:21 smithi152 ceph-mon[156819]: pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:21.694 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:21 smithi152 conmon[374343]: debug 2023-06-10T21:51:21.544+0000 7f953f05b200 -1 osd.4 493 log_to_monitors {default=true} 2023-06-10T21:51:21.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:21 smithi064 ceph-mon[180297]: pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:21.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:21 smithi064 ceph-mon[186706]: pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:22.100 INFO:teuthology.orchestra.run.smithi152.stderr:no valid command found; 10 closest matches: 2023-06-10T21:51:22.100 INFO:teuthology.orchestra.run.smithi152.stderr:0 2023-06-10T21:51:22.100 INFO:teuthology.orchestra.run.smithi152.stderr:1 2023-06-10T21:51:22.100 INFO:teuthology.orchestra.run.smithi152.stderr:2 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:abort 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:assert 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs debug_inject_read_zeros 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs files list 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:bluefs stats 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator dump block 2023-06-10T21:51:22.101 INFO:teuthology.orchestra.run.smithi152.stderr:bluestore allocator fragmentation block 2023-06-10T21:51:22.102 INFO:teuthology.orchestra.run.smithi152.stderr:admin_socket: invalid command 2023-06-10T21:51:22.579 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:22.579 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:51:22.579 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152134 14.3 14:cd82057e:::benchmark_data_smithi064_559916_object44278:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e451)", 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 163.00440751400001, 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.031321833, 2023-06-10T21:51:22.580 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152134 2023-06-10T21:51:22.581 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.582 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570118+0000", 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570116+0000", 2023-06-10T21:51:22.583 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570124+0000", 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5170000000000002e-06 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.584 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.585 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570137+0000", 2023-06-10T21:51:22.585 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3825000000000001e-05 2023-06-10T21:51:22.585 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.585 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.586 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.586 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.570147+0000", 2023-06-10T21:51:22.586 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.2900000000000008e-06 2023-06-10T21:51:22.586 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.587 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.587 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.587 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.572192+0000", 2023-06-10T21:51:22.587 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002045135 2023-06-10T21:51:22.587 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.572201+0000", 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4639999999999995e-06 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.588 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601383+0000", 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0291812229999999 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601440+0000", 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7219000000000002e-05 2023-06-10T21:51:22.589 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:152156 14.3 14:cc7f4242:::benchmark_data_smithi064_559916_object44300:head [delete] snapc 0=[] ondisk+write+known_if_redirected e451)", 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:51:22.590 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 162.98439138699999, 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.032985703, 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.591 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 152156 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.592 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590134+0000", 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.593 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590132+0000", 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590142+0000", 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4730000000000006e-06 2023-06-10T21:51:22.594 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590161+0000", 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9055e-05 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.595 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590187+0000", 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6055999999999999e-05 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590276+0000", 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.9653000000000004e-05 2023-06-10T21:51:22.596 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:39.590289+0000", 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2323999999999999e-05 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.597 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601787+0000", 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0114977540000001 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.601859+0000", 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2046999999999996e-05 2023-06-10T21:51:22.598 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.602296+0000", 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043782199999999998 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.599 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617514+0000", 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01521788 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.617560+0000", 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5865999999999998e-05 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.600 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623034+0000", 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.005473449 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.601 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623080+0000", 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6686e-05 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:48:40.623120+0000", 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9532e-05 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.602 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212565 14.b 14:d2b89b12:::benchmark_data_smithi064_559916_object104709:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.465559+0000", 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.108966273999997, 2023-06-10T21:51:22.603 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0441391959999999, 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212565 2023-06-10T21:51:22.604 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465559+0000", 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.605 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465559+0000", 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465558+0000", 2023-06-10T21:51:22.606 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465564+0000", 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6969999999999996e-06 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.607 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465580+0000", 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5996999999999999e-05 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.608 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.609 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465588+0000", 2023-06-10T21:51:22.609 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9219999999999997e-06 2023-06-10T21:51:22.609 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.609 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.610 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.610 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465638+0000", 2023-06-10T21:51:22.610 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0061000000000001e-05 2023-06-10T21:51:22.610 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.610 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465649+0000", 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0322e-05 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.611 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.612 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474849+0000", 2023-06-10T21:51:22.612 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0092008539999999 2023-06-10T21:51:22.612 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.613 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.613 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.613 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.474964+0000", 2023-06-10T21:51:22.613 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011505 2023-06-10T21:51:22.613 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.475494+0000", 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00052970800000000005 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.614 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497413+0000", 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.021918387000000001 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.500351+0000", 2023-06-10T21:51:22.615 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029388880000000002 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509607+0000", 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0092555970000000008 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.616 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509672+0000", 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4593999999999999e-05 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.617 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509698+0000", 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6787999999999998e-05 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.618 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.619 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212578 14.b 14:d980ef2a:::benchmark_data_smithi064_559916_object104722:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:51:22.619 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.478786+0000", 2023-06-10T21:51:22.619 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.095739336999998, 2023-06-10T21:51:22.619 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0310544319999999, 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.620 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212578 2023-06-10T21:51:22.621 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.621 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.621 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.621 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.622 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478786+0000", 2023-06-10T21:51:22.622 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.622 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.622 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.623 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.623 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478786+0000", 2023-06-10T21:51:22.623 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.623 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.624 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.624 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.624 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478785+0000", 2023-06-10T21:51:22.624 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.624 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478790+0000", 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4279999999999998e-06 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.625 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478800+0000", 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0281e-05 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478808+0000", 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.5680000000000002e-06 2023-06-10T21:51:22.626 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.481391+0000", 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0025832099999999998 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.627 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.481402+0000", 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1137e-05 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.489144+0000", 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.00774176 2023-06-10T21:51:22.628 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.489272+0000", 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012852 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.629 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.489408+0000", 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013538600000000001 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.500372+0000", 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.010963850000000001 2023-06-10T21:51:22.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.500403+0000", 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1325000000000002e-05 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509756+0000", 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.009353419 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509802+0000", 2023-06-10T21:51:22.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5374000000000003e-05 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509841+0000", 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8863999999999999e-05 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.633 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212592 14.b 14:d0813bf4:::benchmark_data_smithi064_559916_object104736:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.493614+0000", 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.080911614999998, 2023-06-10T21:51:22.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0163337640000001, 2023-06-10T21:51:22.635 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.635 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.635 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.635 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212592 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493614+0000", 2023-06-10T21:51:22.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493614+0000", 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493613+0000", 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493617+0000", 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2030000000000002e-06 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493625+0000", 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4200000000000001e-06 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493631+0000", 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2380000000000002e-06 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493666+0000", 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4681999999999998e-05 2023-06-10T21:51:22.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.493675+0000", 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7070000000000004e-06 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.642 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.491744+0000", 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99806877100000002 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.491861+0000", 2023-06-10T21:51:22.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011738799999999999 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.491989+0000", 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012714200000000001 2023-06-10T21:51:22.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.500438+0000", 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0084494930000000006 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.504849+0000", 2023-06-10T21:51:22.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0044106859999999996 2023-06-10T21:51:22.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509880+0000", 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050316329999999998 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.647 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509914+0000", 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3670000000000001e-05 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509948+0000", 2023-06-10T21:51:22.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3521e-05 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212613 14.b 14:d1877ba1:::benchmark_data_smithi064_559916_object104757:head [delete] snapc 0=[] ondisk+write+known_if_redirected e485)", 2023-06-10T21:51:22.649 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:34.521193+0000", 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 48.053332161, 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99040751000000005, 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.650 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212613 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521193+0000", 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521193+0000", 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521192+0000", 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521199+0000", 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0099999999999998e-06 2023-06-10T21:51:22.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521212+0000", 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2687000000000001e-05 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521221+0000", 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4970000000000007e-06 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521269+0000", 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7460000000000003e-05 2023-06-10T21:51:22.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.656 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.656 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.521281+0000", 2023-06-10T21:51:22.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2663000000000001e-05 2023-06-10T21:51:22.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.657 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.657 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.657 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497444+0000", 2023-06-10T21:51:22.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97616284499999995 2023-06-10T21:51:22.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497589+0000", 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014442599999999999 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.497714+0000", 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000125635 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.504854+0000", 2023-06-10T21:51:22.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.007139762 2023-06-10T21:51:22.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.509566+0000", 2023-06-10T21:51:22.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0047124389999999997 2023-06-10T21:51:22.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.511503+0000", 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0019369980000000001 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.511563+0000", 2023-06-10T21:51:22.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9803999999999999e-05 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:35.511601+0000", 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7564000000000002e-05 2023-06-10T21:51:22.664 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214567 14.c 14:39f22383:::benchmark_data_smithi064_559916_object106711:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e487)", 2023-06-10T21:51:22.665 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.551091+0000", 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 45.023434807000001, 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0597545909999999, 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.666 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.667 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214567 2023-06-10T21:51:22.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.667 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551091+0000", 2023-06-10T21:51:22.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551091+0000", 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551090+0000", 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551095+0000", 2023-06-10T21:51:22.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9559999999999996e-06 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551105+0000", 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0692e-05 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551113+0000", 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.7309999999999993e-06 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551373+0000", 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00026000699999999998 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551382+0000", 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.524e-06 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.610738+0000", 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.05935595 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.610845+0000", 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010758 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.675 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214574 14.c 14:38c5b448:::benchmark_data_smithi064_559916_object106718:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e487)", 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.551123+0000", 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 45.023402433999998, 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0860494949999999, 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.676 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214574 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551123+0000", 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551123+0000", 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551123+0000", 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.679 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551125+0000", 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.52e-06 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551131+0000", 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0419999999999999e-06 2023-06-10T21:51:22.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551136+0000", 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.814e-06 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551402+0000", 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000266371 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.551408+0000", 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0410000000000003e-06 2023-06-10T21:51:22.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.611141+0000", 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0597329580000001 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.611277+0000", 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013589900000000001 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.611772+0000", 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00049513900000000002 2023-06-10T21:51:22.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629353+0000", 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.017580287999999999 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629437+0000", 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4530000000000006e-05 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637081+0000", 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0076436949999999998 2023-06-10T21:51:22.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637127+0000", 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6010999999999997e-05 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637173+0000", 2023-06-10T21:51:22.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5519999999999998e-05 2023-06-10T21:51:22.688 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.688 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.688 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.689 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214605 14.c 14:30d139a8:::benchmark_data_smithi064_559916_object106749:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.689 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.583669+0000", 2023-06-10T21:51:22.689 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.990856413000003, 2023-06-10T21:51:22.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.053599237, 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214605 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.690 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583669+0000", 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.691 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583669+0000", 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583668+0000", 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:22.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583674+0000", 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1020000000000002e-06 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583686+0000", 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2077e-05 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583694+0000", 2023-06-10T21:51:22.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1089999999999998e-06 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583739+0000", 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5011e-05 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583749+0000", 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0287e-05 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.624663+0000", 2023-06-10T21:51:22.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0409138630000001 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.624782+0000", 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118664 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.624887+0000", 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105222 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629407+0000", 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0045201019999999998 2023-06-10T21:51:22.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629484+0000", 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6648000000000002e-05 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637184+0000", 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0077000180000000003 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637235+0000", 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.1038000000000002e-05 2023-06-10T21:51:22.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637268+0000", 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3386000000000002e-05 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214607 14.c 14:3d189983:::benchmark_data_smithi064_559916_object106751:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.583719+0000", 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.990806730000003, 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0537903749999999, 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.702 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214607 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583719+0000", 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.704 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583719+0000", 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583718+0000", 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583721+0000", 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.224e-06 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583728+0000", 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9709999999999998e-06 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583734+0000", 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4840000000000003e-06 2023-06-10T21:51:22.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583771+0000", 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6860000000000003e-05 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583778+0000", 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.7410000000000003e-06 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.627094+0000", 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0433168100000001 2023-06-10T21:51:22.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.627234+0000", 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013979699999999999 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.627390+0000", 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015603700000000001 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629466+0000", 2023-06-10T21:51:22.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020757589999999999 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637281+0000", 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0078147900000000003 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637417+0000", 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013644300000000001 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637465+0000", 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7763999999999999e-05 2023-06-10T21:51:22.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637509+0000", 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4200999999999999e-05 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.714 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214608 14.c 14:316ea556:::benchmark_data_smithi064_559916_object106752:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.583739+0000", 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.990786946, 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0539943709999999, 2023-06-10T21:51:22.715 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214608 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583739+0000", 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583739+0000", 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583738+0000", 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583740+0000", 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9640000000000002e-06 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583745+0000", 2023-06-10T21:51:22.719 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2220000000000001e-06 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583750+0000", 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5430000000000004e-06 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583797+0000", 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7182999999999999e-05 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.583804+0000", 2023-06-10T21:51:22.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5400000000000001e-06 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629496+0000", 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.045692442 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629583+0000", 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6725999999999996e-05 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.629669+0000", 2023-06-10T21:51:22.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6520999999999994e-05 2023-06-10T21:51:22.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.631669+0000", 2023-06-10T21:51:22.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001999982 2023-06-10T21:51:22.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637525+0000", 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058555780000000002 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637643+0000", 2023-06-10T21:51:22.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118329 2023-06-10T21:51:22.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637690+0000", 2023-06-10T21:51:22.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6980000000000001e-05 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637733+0000", 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2654999999999997e-05 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.728 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214613 14.c 14:3da1cbc6:::benchmark_data_smithi064_559916_object106757:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.589920+0000", 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.984605371999997, 2023-06-10T21:51:22.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.053720443, 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214613 2023-06-10T21:51:22.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589920+0000", 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589920+0000", 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589919+0000", 2023-06-10T21:51:22.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589925+0000", 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2210000000000002e-06 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.733 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589937+0000", 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1694e-05 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589944+0000", 2023-06-10T21:51:22.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2640000000000002e-06 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.589991+0000", 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6461999999999997e-05 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590000+0000", 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.713e-06 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.631838+0000", 2023-06-10T21:51:22.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.041837313 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.631971+0000", 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013307099999999999 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.632116+0000", 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144735 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637341+0000", 2023-06-10T21:51:22.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0052251779999999996 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637535+0000", 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019389599999999999 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643545+0000", 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0060099209999999997 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643595+0000", 2023-06-10T21:51:22.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0531000000000001e-05 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643641+0000", 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5479000000000003e-05 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.741 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214616 14.c 14:31cf1d01:::benchmark_data_smithi064_559916_object106760:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.590001+0000", 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.984524209, 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0597061139999999, 2023-06-10T21:51:22.742 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214616 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590001+0000", 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590001+0000", 2023-06-10T21:51:22.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590001+0000", 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590005+0000", 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.4789999999999998e-06 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590017+0000", 2023-06-10T21:51:22.746 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2320999999999999e-05 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590031+0000", 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4139e-05 2023-06-10T21:51:22.747 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590100+0000", 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8245999999999997e-05 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.590109+0000", 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3409999999999992e-06 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.634216+0000", 2023-06-10T21:51:22.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0441073300000001 2023-06-10T21:51:22.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.634437+0000", 2023-06-10T21:51:22.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000220845 2023-06-10T21:51:22.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.634570+0000", 2023-06-10T21:51:22.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013264599999999999 2023-06-10T21:51:22.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637577+0000", 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030068669999999999 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637588+0000", 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1095e-05 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649625+0000", 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.012037152000000001 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649671+0000", 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5899999999999998e-05 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649707+0000", 2023-06-10T21:51:22.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6507000000000001e-05 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214647 14.c 14:30dceb51:::benchmark_data_smithi064_559916_object106791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.756 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.617952+0000", 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.956573214000002, 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.040398266, 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.757 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214647 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617952+0000", 2023-06-10T21:51:22.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617952+0000", 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617952+0000", 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617955+0000", 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.19e-06 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617962+0000", 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4610000000000002e-06 2023-06-10T21:51:22.761 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.617968+0000", 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6829999999999996e-06 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.618043+0000", 2023-06-10T21:51:22.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4924000000000003e-05 2023-06-10T21:51:22.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.618053+0000", 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7359999999999996e-06 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637771+0000", 2023-06-10T21:51:22.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0197180539999999 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.637919+0000", 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014844399999999999 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.638064+0000", 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144995 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649591+0000", 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.011526507 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649773+0000", 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00018260100000000001 2023-06-10T21:51:22.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658281+0000", 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085074009999999995 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658314+0000", 2023-06-10T21:51:22.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3621999999999998e-05 2023-06-10T21:51:22.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658351+0000", 2023-06-10T21:51:22.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6229000000000002e-05 2023-06-10T21:51:22.770 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214652 14.c 14:3e1c6f3c:::benchmark_data_smithi064_559916_object106796:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.771 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.624287+0000", 2023-06-10T21:51:22.772 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.950238016, 2023-06-10T21:51:22.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.034152779, 2023-06-10T21:51:22.772 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.772 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.772 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214652 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624287+0000", 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624287+0000", 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.774 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624287+0000", 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624289+0000", 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.7540000000000001e-06 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624294+0000", 2023-06-10T21:51:22.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2680000000000002e-06 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624297+0000", 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9380000000000001e-06 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624333+0000", 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6261999999999999e-05 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.624340+0000", 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1410000000000002e-06 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.640837+0000", 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.016497703 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.640943+0000", 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010533 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.641051+0000", 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010849999999999999 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649744+0000", 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086925400000000003 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658273+0000", 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0085297889999999994 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658367+0000", 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3769000000000003e-05 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658404+0000", 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6353000000000002e-05 2023-06-10T21:51:22.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658440+0000", 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6721999999999998e-05 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214673 14.c 14:35f46b43:::benchmark_data_smithi064_559916_object106817:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.644478+0000", 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.930047146, 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.014102136, 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214673 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644478+0000", 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644478+0000", 2023-06-10T21:51:22.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644477+0000", 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:22.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644485+0000", 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8520000000000004e-06 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644505+0000", 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9981000000000001e-05 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644514+0000", 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1339999999999993e-06 2023-06-10T21:51:22.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644556+0000", 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2240999999999998e-05 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644566+0000", 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8339999999999994e-06 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643679+0000", 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99911299399999998 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643809+0000", 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000129987 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.643944+0000", 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013512999999999999 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649802+0000", 2023-06-10T21:51:22.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058576469999999997 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658451+0000", 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086498949999999995 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658502+0000", 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0284999999999997e-05 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658544+0000", 2023-06-10T21:51:22.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1924e-05 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658580+0000", 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.684e-05 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.797 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214675 14.c 14:3f741c15:::benchmark_data_smithi064_559916_object106819:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.644944+0000", 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.929581392999999, 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0137883839999999, 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.798 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214675 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644944+0000", 2023-06-10T21:51:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644944+0000", 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644943+0000", 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644949+0000", 2023-06-10T21:51:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8039999999999998e-06 2023-06-10T21:51:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644965+0000", 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5673000000000001e-05 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.644973+0000", 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.0870000000000007e-06 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.646213+0000", 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001240503 2023-06-10T21:51:22.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.646223+0000", 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.3899999999999999e-06 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.646709+0000", 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0004865709999999 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.646828+0000", 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000118727 2023-06-10T21:51:22.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.646949+0000", 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121575 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649833+0000", 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002883184 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658457+0000", 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086241159999999994 2023-06-10T21:51:22.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658657+0000", 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000200182 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.810 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658699+0000", 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2302999999999998e-05 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658732+0000", 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3275000000000002e-05 2023-06-10T21:51:22.811 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214697 14.c 14:343b7752:::benchmark_data_smithi064_559916_object106841:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.669104+0000", 2023-06-10T21:51:22.812 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.905421269000001, 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98984434899999996, 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.813 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214697 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669104+0000", 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669104+0000", 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669103+0000", 2023-06-10T21:51:22.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669109+0000", 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6749999999999999e-06 2023-06-10T21:51:22.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669123+0000", 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.394e-05 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669130+0000", 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.4420000000000001e-06 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669186+0000", 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.5971000000000003e-05 2023-06-10T21:51:22.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.669196+0000", 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0359e-05 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649854+0000", 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98065752399999995 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.649954+0000", 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010057500000000001 2023-06-10T21:51:22.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.650059+0000", 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010437699999999999 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658191+0000", 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0081325410000000001 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658594+0000", 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00040282700000000002 2023-06-10T21:51:22.822 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658848+0000", 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025340999999999997 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658901+0000", 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.3776999999999997e-05 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658949+0000", 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7169000000000002e-05 2023-06-10T21:51:22.824 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214702 14.c 14:3c16d60b:::benchmark_data_smithi064_559916_object106846:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.673268+0000", 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.901257141000002, 2023-06-10T21:51:22.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99340995600000004, 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214702 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.826 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673268+0000", 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673268+0000", 2023-06-10T21:51:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673268+0000", 2023-06-10T21:51:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673271+0000", 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8500000000000004e-06 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673280+0000", 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.6719999999999993e-06 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673287+0000", 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.8920000000000003e-06 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673334+0000", 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6542999999999998e-05 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.673341+0000", 2023-06-10T21:51:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.763e-06 2023-06-10T21:51:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.652547+0000", 2023-06-10T21:51:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97920587299999995 2023-06-10T21:51:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.834 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.652674+0000", 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012679400000000001 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.652765+0000", 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.1421000000000004e-05 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658237+0000", 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054717480000000002 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658606+0000", 2023-06-10T21:51:22.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00036863800000000002 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.666594+0000", 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0079883190000000007 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.666640+0000", 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5516000000000003e-05 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.666678+0000", 2023-06-10T21:51:22.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8652000000000002e-05 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:214704 14.c 14:358bc307:::benchmark_data_smithi064_559916_object106848:head [delete] snapc 0=[] ondisk+write+known_if_redirected e487)", 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:37.675720+0000", 2023-06-10T21:51:22.839 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 44.898805312, 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99641574799999999, 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 214704 2023-06-10T21:51:22.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675720+0000", 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675720+0000", 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:22.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675719+0000", 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675725+0000", 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4240000000000001e-06 2023-06-10T21:51:22.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675736+0000", 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0927999999999999e-05 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675743+0000", 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3880000000000002e-06 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675797+0000", 2023-06-10T21:51:22.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4051999999999997e-05 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:37.675806+0000", 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4499999999999993e-06 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.846 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.655232+0000", 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97942517299999998 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.655350+0000", 2023-06-10T21:51:22.847 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011846899999999999 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.655747+0000", 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00039684799999999999 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658610+0000", 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0028635240000000001 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.658792+0000", 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000181182 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.672044+0000", 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013252163000000001 2023-06-10T21:51:22.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.672094+0000", 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9725999999999998e-05 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:22.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:38.672136+0000", 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.2370999999999998e-05 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:22.852 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:22.853 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:51:22 smithi152 conmon[374343]: debug 2023-06-10T21:51:22.424+0000 7f952fd54700 -1 osd.4 493 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-10T21:51:22.853 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:22 smithi064 ceph-mon[186706]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.854 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:22 smithi064 ceph-mon[186706]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.854 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:22 smithi152 ceph-mon[156819]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.854 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:22 smithi152 ceph-mon[156819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-10T21:51:22.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:22 smithi064 ceph-mon[180297]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.855 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:22 smithi064 ceph-mon[180297]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-10T21:51:22.855 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-06-10T21:51:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:22] "GET /metrics HTTP/1.1" 200 222017 "" "Prometheus/2.33.4" 2023-06-10T21:51:23.311 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: osdmap e506: 8 total, 7 up, 5 in 2023-06-10T21:51:23.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[180297]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T21:51:23.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: osdmap e506: 8 total, 7 up, 5 in 2023-06-10T21:51:23.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:23.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:23 smithi064 ceph-mon[186706]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:23.684 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: failed to encode map e506 with expected crc 2023-06-10T21:51:23.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-10T21:51:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: osdmap e506: 8 total, 7 up, 5 in 2023-06-10T21:51:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: from='osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:23.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:23 smithi152 ceph-mon[156819]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2023-06-10T21:51:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[180297]: osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690] boot 2023-06-10T21:51:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[180297]: osdmap e507: 8 total, 8 up, 5 in 2023-06-10T21:51:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:51:24.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[180297]: pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[186706]: osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690] boot 2023-06-10T21:51:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[186706]: osdmap e507: 8 total, 8 up, 5 in 2023-06-10T21:51:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:51:24.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:24 smithi064 ceph-mon[186706]: pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:24 smithi152 ceph-mon[156819]: osd.4 [v2:172.21.15.152:6800/3531161690,v1:172.21.15.152:6801/3531161690] boot 2023-06-10T21:51:24.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:24 smithi152 ceph-mon[156819]: osdmap e507: 8 total, 8 up, 5 in 2023-06-10T21:51:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:24 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-10T21:51:24.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:24 smithi152 ceph-mon[156819]: pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:25.294 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:51:25.295 INFO:teuthology.orchestra.run.smithi064.stdout: "num_ops": 0 2023-06-10T21:51:25.295 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:25.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e494 with expected crc 2023-06-10T21:51:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[186706]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e494 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:25 smithi064 ceph-mon[180297]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e494 with expected crc 2023-06-10T21:51:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:25.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:25 smithi152 ceph-mon[156819]: failed to encode map e507 with expected crc 2023-06-10T21:51:26.052 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-10T21:51:26.420 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:27.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:27 smithi152 ceph-mon[156819]: pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:27.771 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:27 smithi064 ceph-mon[180297]: pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:27.772 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:27 smithi064 ceph-mon[186706]: pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:27.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:51:28.011 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [], 2023-06-10T21:51:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "complaint_time": 30, 2023-06-10T21:51:28.011 INFO:teuthology.orchestra.run.smithi064.stdout: "num_blocked_ops": 0 2023-06-10T21:51:28.011 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:28.219 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:28.752 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-10T21:51:29.119 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:29.626 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:29 smithi064 ceph-mon[186706]: pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:29.626 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:29 smithi064 ceph-mon[180297]: pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:29.664 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:29.665 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:29.665 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:51:29.665 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:29.684 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:29 smithi152 ceph-mon[156819]: pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:30.364 INFO:tasks.thrashosds.thrasher:inning osd 2023-06-10T21:51:30.364 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-06-10T21:51:30.365 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 5 2023-06-10T21:51:30.627 INFO:teuthology.orchestra.run.smithi064.stdout:{ 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: "size": 20, 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 600, 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: "ops": [ 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106691 13.3 13:cd7d9036:::benchmark_data_smithi064_501823_object106690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:51:30.628 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 510.10275806800001, 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.983989778, 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:51:30.629 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106691 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.630 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520099+0000", 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.631 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520098+0000", 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520145+0000", 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6650999999999998e-05 2023-06-10T21:51:30.632 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.633 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.633 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.633 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520153+0000", 2023-06-10T21:51:30.633 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6559999999999992e-06 2023-06-10T21:51:30.633 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520159+0000", 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4359999999999999e-06 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.634 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520202+0000", 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3389000000000001e-05 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520324+0000", 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000121074 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.635 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 7", 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.520472+0000", 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000148715 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.636 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.526115+0000", 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0056431629999999997 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547106+0000", 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.020990175 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.637 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547117+0000", 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.182e-05 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189400+0000", 2023-06-10T21:51:30.638 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64228294200000002 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189417+0000", 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6770999999999999e-05 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.639 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.503987+0000", 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.314569638 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.504089+0000", 2023-06-10T21:51:30.640 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000101869 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106703 13.3 13:caa34abe:::benchmark_data_smithi064_501823_object106702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e389)", 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:51:30.641 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 510.08367338599999, 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.981275578, 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106703 2023-06-10T21:51:30.642 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.643 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539184+0000", 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539182+0000", 2023-06-10T21:51:30.644 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.539209+0000", 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.6724999999999999e-05 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.645 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542176+0000", 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002966991 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.542190+0000", 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4246999999999999e-05 2023-06-10T21:51:30.646 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.647 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547147+0000", 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049566059999999997 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.648 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:00.547155+0000", 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.3969999999999994e-06 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189447+0000", 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.64229122500000002 2023-06-10T21:51:30.649 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.189456+0000", 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4609999999999997e-06 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.650 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505480+0000", 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.316023876 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505680+0000", 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000200366 2023-06-10T21:51:30.651 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 5,7", 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.505915+0000", 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000234653 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.652 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.508394+0000", 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024786389999999999 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.517040+0000", 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0086467249999999992 2023-06-10T21:51:30.653 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.518965+0000", 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001924469 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.654 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519013+0000", 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.8300000000000002e-05 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.520459+0000", 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001446073 2023-06-10T21:51:30.655 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.46829.0:106693 13.2 13:481d9832:::benchmark_data_smithi064_501823_object106692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e391)", 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 509.43195879299998, 2023-06-10T21:51:30.656 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3284778340000001, 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.46829", 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 106693 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.657 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.658 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190898+0000", 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190896+0000", 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.659 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190926+0000", 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9950999999999998e-05 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.660 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190941+0000", 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5157000000000001e-05 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.190954+0000", 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2877999999999999e-05 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.661 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193380+0000", 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0024257100000000002 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.662 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:01.193393+0000", 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2578000000000001e-05 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519272+0000", 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.325879748 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.663 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:43:02.519376+0000", 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010365300000000001 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.664 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113489 14.2 14:4e77c190:::benchmark_data_smithi064_559916_object5633:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.30738322100001, 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0400237479999999, 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.665 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113489 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:51:30.666 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315474+0000", 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.667 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315472+0000", 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315476+0000", 2023-06-10T21:51:30.668 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.8809999999999998e-06 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315484+0000", 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.8690000000000004e-06 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.669 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315490+0000", 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6720000000000001e-06 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315593+0000", 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000103514 2023-06-10T21:51:30.670 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315724+0000", 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013101899999999999 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:51:30.671 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.315831+0000", 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106902 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.318758+0000", 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002926508 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.672 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320772+0000", 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0020147619999999998 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.673 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321248+0000", 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00047574200000000003 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.361815+0000", 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.040567060000000002 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.674 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.361827+0000", 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1287000000000001e-05 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332544+0000", 2023-06-10T21:51:30.675 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97071757299999994 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332558+0000", 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3631999999999999e-05 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.676 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355409+0000", 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.022850839 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355497+0000", 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.8868000000000004e-05 2023-06-10T21:51:30.677 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113492 14.2 14:4e2dedac:::benchmark_data_smithi064_559916_object5636:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:51:30.678 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.301930527, 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0489783180000001, 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113492 2023-06-10T21:51:30.679 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.680 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320926+0000", 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320925+0000", 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.681 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320931+0000", 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3489999999999997e-06 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.682 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320944+0000", 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2369e-05 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.320952+0000", 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.5750000000000001e-06 2023-06-10T21:51:30.683 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321274+0000", 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000322009 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.684 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321413+0000", 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013840599999999999 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.321538+0000", 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000124993 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.685 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.384946+0000", 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.063408185000000006 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.384958+0000", 2023-06-10T21:51:30.686 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2075e-05 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332592+0000", 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94763364299999997 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.687 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332600+0000", 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1370000000000002e-06 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355799+0000", 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0231989180000001 2023-06-10T21:51:30.688 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for missing object", 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355949+0000", 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014996000000000001 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.689 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369808+0000", 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013859274 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369905+0000", 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6813999999999998e-05 2023-06-10T21:51:30.690 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113495 14.1 14:85da5a60:::benchmark_data_smithi064_559916_object5639:head [delete] snapc 0=[] ondisk+write+known_if_redirected e442)", 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.30028318399999, 2023-06-10T21:51:30.691 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.126787014, 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "reached pg", 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113495 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.692 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.693 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322574+0000", 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322573+0000", 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:30.694 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322579+0000", 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.8270000000000003e-06 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.695 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322590+0000", 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1576000000000001e-05 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322598+0000", 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6040000000000004e-06 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.696 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322655+0000", 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7009000000000003e-05 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322767+0000", 2023-06-10T21:51:30.697 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112612 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,7", 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.322867+0000", 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9697000000000004e-05 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.698 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.327963+0000", 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050958289999999996 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.396532+0000", 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.068569243000000002 2023-06-10T21:51:30.699 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.396544+0000", 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1412e-05 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.700 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334604+0000", 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93806067599999998 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334618+0000", 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3395e-05 2023-06-10T21:51:30.701 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.449264+0000", 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.114646585 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.702 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.449361+0000", 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6520000000000004e-05 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113506 14.2 14:437c8228:::benchmark_data_smithi064_559916_object5650:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:51:30.703 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.23322321500001, 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.9829840920000001, 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.704 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113506 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.705 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389634+0000", 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.706 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389632+0000", 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389639+0000", 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.9890000000000003e-06 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.707 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389651+0000", 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1640999999999999e-05 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.708 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389660+0000", 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0599999999999997e-06 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389719+0000", 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9182999999999999e-05 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.709 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389729+0000", 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.7180000000000008e-06 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332652+0000", 2023-06-10T21:51:30.710 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.942923069 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332659+0000", 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.2389999999999996e-06 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.711 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.355991+0000", 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0233322730000001 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.356154+0000", 2023-06-10T21:51:30.712 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000162709 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.356326+0000", 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00017248599999999999 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.713 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.365479+0000", 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0091523160000000006 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368469+0000", 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029906360000000001 2023-06-10T21:51:30.714 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369468+0000", 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00099876499999999998 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.715 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372462+0000", 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029935249999999999 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372521+0000", 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9165e-05 2023-06-10T21:51:30.716 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372618+0000", 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.6997999999999995e-05 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.717 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113507 14.2 14:4867687b:::benchmark_data_smithi064_559916_object5651:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.23318432600001, 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.983135852, 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.718 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113507 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.719 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389673+0000", 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.720 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389672+0000", 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.721 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389675+0000", 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.5040000000000001e-06 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389680+0000", 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7849999999999997e-06 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.722 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389685+0000", 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9180000000000002e-06 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389748+0000", 2023-06-10T21:51:30.723 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.2556000000000002e-05 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.389754+0000", 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0560000000000001e-06 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.724 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332683+0000", 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.94292939200000003 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.332690+0000", 2023-06-10T21:51:30.725 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0119999999999999e-06 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359119+0000", 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0264282819999999 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.726 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359263+0000", 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000144466 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.359414+0000", 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015090899999999999 2023-06-10T21:51:30.727 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368481+0000", 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0090673170000000001 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.728 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369506+0000", 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0010247920000000001 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.369942+0000", 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00043549000000000002 2023-06-10T21:51:30.729 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372655+0000", 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002713783 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.730 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372710+0000", 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.4558999999999997e-05 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372808+0000", 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8418000000000005e-05 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.731 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113510 14.2 14:41be09f6:::benchmark_data_smithi064_559916_object5654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.22333315099999, 2023-06-10T21:51:30.732 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.973454032, 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113510 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.733 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.734 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399524+0000", 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399522+0000", 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.735 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399528+0000", 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2519999999999999e-06 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.736 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399540+0000", 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.2697999999999999e-05 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399548+0000", 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.9829999999999996e-06 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.737 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399590+0000", 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1582000000000003e-05 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.399599+0000", 2023-06-10T21:51:30.738 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5289999999999997e-06 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.333991+0000", 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93439153399999997 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.739 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334002+0000", 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0492000000000001e-05 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362370+0000", 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0283688769999999 2023-06-10T21:51:30.740 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362530+0000", 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000159383 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,5,7", 2023-06-10T21:51:30.741 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.362686+0000", 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00015642100000000001 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.368494+0000", 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0058078749999999997 2023-06-10T21:51:30.742 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.371560+0000", 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030656170000000001 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.743 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372046+0000", 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00048609899999999998 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372850+0000", 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00080377499999999995 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.744 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372896+0000", 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.6505000000000003e-05 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.372978+0000", 2023-06-10T21:51:30.745 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1731999999999996e-05 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113514 14.1 14:8819266d:::benchmark_data_smithi064_559916_object5658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e443)", 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:51:30.746 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 238.216559797, 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.0529779540000002, 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.747 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113514 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.748 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406297+0000", 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406296+0000", 2023-06-10T21:51:30.749 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.999999 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406302+0000", 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.93e-06 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.750 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406316+0000", 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3675999999999999e-05 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406322+0000", 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.4699999999999999e-06 2023-06-10T21:51:30.751 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406382+0000", 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.9517e-05 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.752 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:32.406393+0000", 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1188e-05 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334656+0000", 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92826324699999996 2023-06-10T21:51:30.753 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.334666+0000", 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8989999999999996e-06 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.754 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453295+0000", 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1186290750000001 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453431+0000", 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013597699999999999 2023-06-10T21:51:30.755 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.453547+0000", 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000116291 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.756 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459016+0000", 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0054681790000000001 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459027+0000", 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1032999999999999e-05 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.757 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459070+0000", 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.3897999999999999e-05 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459116+0000", 2023-06-10T21:51:30.758 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5872999999999999e-05 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459167+0000", 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0689999999999997e-05 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.759 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.459275+0000", 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000108057 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.760 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:113526 14.1 14:8a61b658:::benchmark_data_smithi064_559916_object5670:head [delete] snapc 0=[] ondisk+write+known_if_redirected e444)", 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 237.18071528999999, 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.033151127, 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.761 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 113526 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:51:30.762 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442142+0000", 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.763 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442140+0000", 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442146+0000", 2023-06-10T21:51:30.764 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.579e-06 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442163+0000", 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6396999999999999e-05 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.765 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442172+0000", 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.4679999999999998e-06 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442233+0000", 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.0779000000000002e-05 2023-06-10T21:51:30.766 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:33.442250+0000", 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6546999999999999e-05 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.767 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456413+0000", 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0141633699999999 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456538+0000", 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012505200000000001 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.768 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,2,7", 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.456651+0000", 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112871 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.769 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467864+0000", 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.01121276 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467974+0000", 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011081 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.770 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.467990+0000", 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5242000000000001e-05 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475090+0000", 2023-06-10T21:51:30.771 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0071004320000000003 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475156+0000", 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.6271000000000003e-05 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.772 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:47:34.475293+0000", 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00013633100000000001 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.773 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:193393 14.a 14:5c3440d0:::benchmark_data_smithi064_559916_object85537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e469)", 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:49:58.836569+0000", 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 91.786287544000004, 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.37343404899999999, 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.774 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 193393 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836569+0000", 2023-06-10T21:51:30.775 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836569+0000", 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.776 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836569+0000", 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836571+0000", 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.2400000000000002e-06 2023-06-10T21:51:30.777 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836577+0000", 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.6640000000000003e-06 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.778 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.836582+0000", 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0649999999999998e-06 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.840558+0000", 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0039762039999999997 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.779 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.840671+0000", 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000112289 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 6,7", 2023-06-10T21:51:30.780 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.840778+0000", 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010791800000000001 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.849146+0000", 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0083680720000000007 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.781 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:58.851475+0000", 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.002328756 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:59.209928+0000", 2023-06-10T21:51:30.782 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.35845263500000002 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:59.209969+0000", 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.1106999999999998e-05 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.783 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:49:59.210003+0000", 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4431999999999998e-05 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.784 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212436 14.11 14:887d2294:::benchmark_data_smithi064_559916_object104580:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.436482+0000", 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.186375241999997, 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.024118992, 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.785 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212436 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436482+0000", 2023-06-10T21:51:30.786 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436482+0000", 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.787 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436480+0000", 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436487+0000", 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0940000000000002e-06 2023-06-10T21:51:30.788 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436503+0000", 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.596e-05 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.789 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436517+0000", 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.4136e-05 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436587+0000", 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.0042999999999993e-05 2023-06-10T21:51:30.790 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.436604+0000", 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.6143000000000001e-05 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.791 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.451239+0000", 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0146352199999999 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.451387+0000", 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014826999999999999 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.792 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.451717+0000", 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00033035600000000002 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.456766+0000", 2023-06-10T21:51:30.793 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0050487120000000003 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.456802+0000", 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6325000000000002e-05 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.794 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460535+0000", 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0037329559999999999 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460572+0000", 2023-06-10T21:51:30.795 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.6705999999999997e-05 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460601+0000", 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.8572999999999999e-05 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.796 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212446 14.11 14:8c9f233d:::benchmark_data_smithi064_559916_object104590:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.456854+0000", 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.166002476999999, 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.00856415, 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.797 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212446 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.798 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456854+0000", 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456854+0000", 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.799 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456853+0000", 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.800 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456861+0000", 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.2689999999999999e-06 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456877+0000", 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.5650000000000001e-05 2023-06-10T21:51:30.801 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456885+0000", 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.4470000000000006e-06 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.802 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456938+0000", 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.2961999999999999e-05 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.456950+0000", 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1508999999999999e-05 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.803 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.453558+0000", 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.99660854899999995 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.453677+0000", 2023-06-10T21:51:30.804 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00011864 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.453913+0000", 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023633500000000001 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.805 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.456843+0000", 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029296999999999999 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460527+0000", 2023-06-10T21:51:30.806 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0036842649999999999 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465339+0000", 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0048124259999999999 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.807 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465385+0000", 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5247999999999998e-05 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465419+0000", 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3917000000000001e-05 2023-06-10T21:51:30.808 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212462 14.11 14:8abc0c2f:::benchmark_data_smithi064_559916_object104606:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.481572+0000", 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.141284976000001, 2023-06-10T21:51:30.809 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.98405804200000002, 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212462 2023-06-10T21:51:30.810 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481572+0000", 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.811 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481572+0000", 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481571+0000", 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:30.812 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481575+0000", 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.5599999999999998e-06 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.813 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481582+0000", 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.1450000000000002e-06 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.481588+0000", 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1970000000000001e-06 2023-06-10T21:51:30.814 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.482089+0000", 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00050091299999999997 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.815 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.482100+0000", 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0781999999999999e-05 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455283+0000", 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97318317300000001 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.816 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455408+0000", 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00012496 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.455645+0000", 2023-06-10T21:51:30.817 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00023681700000000001 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462124+0000", 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0064792699999999996 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.818 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465426+0000", 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003301713 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465569+0000", 2023-06-10T21:51:30.819 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00014262000000000001 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465600+0000", 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.1869999999999998e-05 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.820 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465630+0000", 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 2.9553999999999999e-05 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.821 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212474 14.19 14:9f6232b6:::benchmark_data_smithi064_559916_object104618:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.492764+0000", 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.130092951999998, 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.97277418000000004, 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.822 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212474 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492764+0000", 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.823 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492764+0000", 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.824 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492762+0000", 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492769+0000", 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5060000000000001e-06 2023-06-10T21:51:30.825 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492782+0000", 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3284e-05 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.826 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492792+0000", 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5089999999999999e-06 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492854+0000", 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.1948999999999999e-05 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.827 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.492864+0000", 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0570999999999999e-05 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.828 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460618+0000", 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.96775332300000005 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460732+0000", 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000114664 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.829 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.460975+0000", 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00024270100000000001 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465332+0000", 2023-06-10T21:51:30.830 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043566840000000004 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465441+0000", 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000109274 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.831 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465475+0000", 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.3720999999999998e-05 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465507+0000", 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2802999999999997e-05 2023-06-10T21:51:30.832 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465538+0000", 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.0735000000000003e-05 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.833 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212506 14.19 14:9c623ce4:::benchmark_data_smithi064_559916_object104650:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.544553+0000", 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.078304103000001, 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.92542990400000003, 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.834 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212506 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.835 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544553+0000", 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544553+0000", 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.836 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544551+0000", 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.837 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544556+0000", 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.604e-06 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544567+0000", 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.0345e-05 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.838 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544574+0000", 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3860000000000001e-06 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544650+0000", 2023-06-10T21:51:30.839 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.6042000000000001e-05 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.544663+0000", 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3176999999999999e-05 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.840 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462211+0000", 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.91754750699999998 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462309+0000", 2023-06-10T21:51:30.841 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.8086999999999998e-05 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462507+0000", 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000197956 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.842 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465446+0000", 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0029387879999999999 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.468544+0000", 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0030980280000000001 2023-06-10T21:51:30.843 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.469889+0000", 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0013452259999999999 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.844 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.469937+0000", 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.7904000000000001e-05 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.469983+0000", 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.5800999999999997e-05 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.845 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212510 14.11 14:8e1bf71d:::benchmark_data_smithi064_559916_object104654:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.547829+0000", 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.075027562000002, 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.93107748800000001, 2023-06-10T21:51:30.846 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212510 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.847 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547829+0000", 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.848 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547829+0000", 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547828+0000", 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.849 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547835+0000", 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.3119999999999996e-06 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547848+0000", 2023-06-10T21:51:30.850 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3478999999999999e-05 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547857+0000", 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.0589999999999992e-06 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.851 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547908+0000", 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0902000000000002e-05 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.547918+0000", 2023-06-10T21:51:30.852 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.5189999999999998e-06 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.456866+0000", 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.908948478 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.853 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.456971+0000", 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000105172 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.457167+0000", 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00019536600000000001 2023-06-10T21:51:30.854 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462155+0000", 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0049881580000000003 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.855 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465431+0000", 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.003275905 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478820+0000", 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013388938 2023-06-10T21:51:30.856 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478869+0000", 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4.9490000000000002e-05 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.857 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478907+0000", 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7512999999999998e-05 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212537 14.19 14:99a7692a:::benchmark_data_smithi064_559916_object104681:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.858 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.593195+0000", 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.029661943999997, 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87689406700000005, 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.859 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212537 2023-06-10T21:51:30.860 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.860 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.860 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.860 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.860 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593195+0000", 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593195+0000", 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.861 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593193+0000", 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.862 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593199+0000", 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.0599999999999998e-06 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593209+0000", 2023-06-10T21:51:30.863 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 9.9879999999999999e-06 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593215+0000", 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.5420000000000002e-06 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.864 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593296+0000", 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 8.1173000000000004e-05 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.593307+0000", 2023-06-10T21:51:30.865 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1218e-05 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.463699+0000", 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.87039114200000001 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.866 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.463800+0000", 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00010136699999999999 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.867 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 0,6", 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.464010+0000", 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000209621 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.868 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.468326+0000", 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0043164010000000001 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.468577+0000", 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00025105499999999997 2023-06-10T21:51:30.869 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.470020+0000", 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.001442711 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.870 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.470057+0000", 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.7249000000000002e-05 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.871 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.470089+0000", 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.2054999999999997e-05 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.872 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.873 INFO:teuthology.orchestra.run.smithi064.stdout: "description": "osd_op(client.47150.0:212540 14.11 14:8eb44695:::benchmark_data_smithi064_559916_object104684:head [delete] snapc 0=[] ondisk+write+known_if_redirected e484)", 2023-06-10T21:51:30.873 INFO:teuthology.orchestra.run.smithi064.stdout: "initiated_at": "2023-06-10T21:50:33.596831+0000", 2023-06-10T21:51:30.873 INFO:teuthology.orchestra.run.smithi064.stdout: "age": 57.026026184000003, 2023-06-10T21:51:30.873 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.88218476400000001, 2023-06-10T21:51:30.873 INFO:teuthology.orchestra.run.smithi064.stdout: "type_data": { 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_info": { 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client": "client.47150", 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "client_addr": "172.21.15.64:0/4292810566", 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "tid": 212540 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.874 INFO:teuthology.orchestra.run.smithi064.stdout: "events": [ 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "initiated", 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596831+0000", 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.875 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "throttled", 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596831+0000", 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.876 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "header_read", 2023-06-10T21:51:30.877 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596829+0000", 2023-06-10T21:51:30.877 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:30.877 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.877 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.877 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "all_read", 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596836+0000", 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 6.3899999999999998e-06 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "dispatched", 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596850+0000", 2023-06-10T21:51:30.878 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.3848e-05 2023-06-10T21:51:30.879 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.879 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.879 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "queued_for_pg", 2023-06-10T21:51:30.879 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596857+0000", 2023-06-10T21:51:30.879 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 7.605e-06 2023-06-10T21:51:30.880 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.880 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.880 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.880 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596915+0000", 2023-06-10T21:51:30.880 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 5.7343999999999998e-05 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for peered", 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:33.596926+0000", 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 1.1209000000000001e-05 2023-06-10T21:51:30.881 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "reached_pg", 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.458519+0000", 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.86159322400000005 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.882 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "started", 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.458626+0000", 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.000106828 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "waiting for subops from 2,6", 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.458847+0000", 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.00022099399999999999 2023-06-10T21:51:30.883 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.462183+0000", 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0033366099999999998 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.884 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "op_commit", 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.465436+0000", 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.0032525240000000001 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478942+0000", 2023-06-10T21:51:30.885 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 0.013505977000000001 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "commit_sent", 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.478976+0000", 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.4421999999999997e-05 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: }, 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: { 2023-06-10T21:51:30.886 INFO:teuthology.orchestra.run.smithi064.stdout: "event": "done", 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: "time": "2023-06-10T21:50:34.479015+0000", 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: "duration": 3.9147000000000003e-05 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: } 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout: ] 2023-06-10T21:51:30.887 INFO:teuthology.orchestra.run.smithi064.stdout:} 2023-06-10T21:51:31.380 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:31.397 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-10T21:51:31.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:31 smithi064 ceph-mon[186706]: pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:31.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:31 smithi064 ceph-mon[180297]: pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:31.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:31 smithi152 ceph-mon[156819]: pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:31.759 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:32.553 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:32 smithi064 ceph-mon[180297]: pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:32.553 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:32 smithi064 ceph-mon[186706]: pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:32.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:32 smithi152 ceph-mon[156819]: pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:33.156 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:32 smithi064 conmon[180272]: 2023-06-10T21:51:32.957+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e507 definitely_dead 0 2023-06-10T21:51:33.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:32] "GET /metrics HTTP/1.1" 200 222017 "" "Prometheus/2.33.4" 2023-06-10T21:51:33.268 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:33.268 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:33.268 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:51:33.268 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:33.498 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.5. 2023-06-10T21:51:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:33 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:51:33.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:33.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:33 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:51:33.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:33 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-06-10T21:51:33.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:33.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-10T21:51:34.275 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-06-10T21:51:34.275 INFO:tasks.thrashosds.thrasher:inning osd 2023-06-10T21:51:34.275 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-06-10T21:51:34.275 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 4 2023-06-10T21:51:34.279 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:34.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: osdmap e508: 8 total, 8 up, 6 in 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]: dispatch 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[186706]: pgmap v1814: 73 pgs: 18 peering, 55 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:34.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: osdmap e508: 8 total, 8 up, 6 in 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]: dispatch 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:34 smithi064 ceph-mon[180297]: pgmap v1814: 73 pgs: 18 peering, 55 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:34.678 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:34.755 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: failed to encode map e508 with expected crc 2023-06-10T21:51:34.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3305043396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-06-10T21:51:34.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: osdmap e508: 8 total, 8 up, 6 in 2023-06-10T21:51:34.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.756 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]: dispatch 2023-06-10T21:51:34.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]: dispatch 2023-06-10T21:51:34.757 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:34 smithi152 ceph-mon[156819]: pgmap v1814: 73 pgs: 18 peering, 55 active+clean; 582 KiB data, 3.5 GiB used, 444 GiB / 447 GiB avail 2023-06-10T21:51:35.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.598 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]': finished 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]': finished 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]': finished 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: osdmap e509: 8 total, 8 up, 6 in 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.599 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.600 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:35 smithi152 ceph-mon[156819]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.761 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:35.761 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:35.762 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:51:35.762 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:51:35.762 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:35.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]': finished 2023-06-10T21:51:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]': finished 2023-06-10T21:51:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]': finished 2023-06-10T21:51:35.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: osdmap e509: 8 total, 8 up, 6 in 2023-06-10T21:51:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[186706]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.12", "id": [3, 6]}]': finished 2023-06-10T21:51:35.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.17", "id": [5, 2]}]': finished 2023-06-10T21:51:35.910 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 6]}]': finished 2023-06-10T21:51:35.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: osdmap e509: 8 total, 8 up, 6 in 2023-06-10T21:51:35.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: failed to encode map e509 with expected crc 2023-06-10T21:51:35.911 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:35 smithi064 ceph-mon[180297]: failed to encode map e509 with expected crc 2023-06-10T21:51:36.370 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-10T21:51:36.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: osdmap e510: 8 total, 8 up, 6 in 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: pgmap v1817: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-06-10T21:51:36.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:36 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-06-10T21:51:36.718 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:36.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 conmon[180272]: 2023-06-10T21:51:36.432+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e510 definitely_dead 0 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: osdmap e510: 8 total, 8 up, 6 in 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: pgmap v1817: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: osdmap e510: 8 total, 8 up, 6 in 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: failed to encode map e510 with expected crc 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: pgmap v1817: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 532 GiB / 536 GiB avail 2023-06-10T21:51:36.910 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:36 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-06-10T21:51:37.428 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.4. 2023-06-10T21:51:38.158 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:38.158 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166840 13.5 13:ab2851c6:::benchmark_data_smithi064_501823_object56838:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.339258+0000", 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.81422121499998, 2023-06-10T21:51:38.159 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.018799998, 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166840 2023-06-10T21:51:38.160 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339258+0000", 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.161 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339258+0000", 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339257+0000", 2023-06-10T21:51:38.162 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2260000000000005e-06 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339265+0000", 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1489e-05 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.163 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339277+0000", 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.6720000000000003e-06 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339281+0000", 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3250000000000006e-05 2023-06-10T21:51:38.164 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339345+0000", 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0154e-05 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.165 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339355+0000", 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0185985239999999 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.357953+0000", 2023-06-10T21:51:38.166 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000104961 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358058+0000", 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.018799998 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.167 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166847 13.5 13:a0f47406:::benchmark_data_smithi064_501823_object56845:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e406)", 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.339309+0000", 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.81417009299997, 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.059631829, 2023-06-10T21:51:38.168 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166847 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.169 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339309+0000", 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.170 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339309+0000", 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339309+0000", 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7999999999999999e-06 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.171 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339312+0000", 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0650000000000004e-06 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.172 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339318+0000", 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5020000000000003e-06 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339322+0000", 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00045938499999999998 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.173 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339782+0000", 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1899999999999997e-06 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.339786+0000", 2023-06-10T21:51:38.174 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0182890410000001 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358075+0000", 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1086000000000003e-05 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.175 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358146+0000", 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.040711719 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.398858+0000", 2023-06-10T21:51:38.176 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3547000000000005e-05 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.398941+0000", 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.059631829 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.177 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166860 13.5 13:a8407c9b:::benchmark_data_smithi064_501823_object56858:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.340667+0000", 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.81281259799999, 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0435402760000001, 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.178 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166860 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.179 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340667+0000", 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340667+0000", 2023-06-10T21:51:38.180 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340666+0000", 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0719999999999999e-06 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.181 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340671+0000", 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2738999999999999e-05 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340684+0000", 2023-06-10T21:51:38.182 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7150000000000003e-06 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340689+0000", 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.5241000000000003e-05 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.183 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340735+0000", 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3999999999999997e-06 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340741+0000", 2023-06-10T21:51:38.184 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0174409259999999 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358182+0000", 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012555300000000001 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.185 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358308+0000", 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041888899999999998 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.358726+0000", 2023-06-10T21:51:38.186 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.013489851000000001 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.372216+0000", 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011804683999999999 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.187 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384021+0000", 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3195000000000006e-05 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384104+0000", 2023-06-10T21:51:38.188 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9473e-05 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384144+0000", 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3378000000000002e-05 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.189 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384207+0000", 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0435402760000001 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.190 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166862 13.5 13:aef2972d:::benchmark_data_smithi064_501823_object56860:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.340699+0000", 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.81278067699998, 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.043800861, 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.191 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166862 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340699+0000", 2023-06-10T21:51:38.192 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340699+0000", 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.193 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340698+0000", 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.965e-06 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340701+0000", 2023-06-10T21:51:38.194 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0179999999999998e-06 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340707+0000", 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1189999999999999e-06 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.195 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.340711+0000", 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00046419300000000001 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.341175+0000", 2023-06-10T21:51:38.196 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.5129999999999999e-06 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.341183+0000", 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0244400810000001 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.197 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.365623+0000", 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015464999999999999 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.365778+0000", 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000185708 2023-06-10T21:51:38.198 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.365963+0000", 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0062582610000000002 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.199 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.372222+0000", 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.011849643999999999 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384071+0000", 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026145500000000001 2023-06-10T21:51:38.200 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384333+0000", 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5769000000000001e-05 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.201 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384389+0000", 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011098 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384500+0000", 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.043800861 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.202 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166876 13.5 13:a4eda0d2:::benchmark_data_smithi064_501823_object56874:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.368077+0000", 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.78540269600001, 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.01652721, 2023-06-10T21:51:38.203 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166876 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.204 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368077+0000", 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.205 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368077+0000", 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368075+0000", 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1289999999999996e-06 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.206 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368083+0000", 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3825000000000001e-05 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.207 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368097+0000", 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6499999999999996e-06 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368105+0000", 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.7019000000000003e-05 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.208 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368172+0000", 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3297e-05 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.209 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.368185+0000", 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.99940087799999999 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.367586+0000", 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9098999999999995e-05 2023-06-10T21:51:38.210 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.367675+0000", 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.1910999999999995e-05 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.211 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.367767+0000", 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0069176250000000002 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.374684+0000", 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095725519999999998 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.212 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384257+0000", 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026546600000000001 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.213 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384522+0000", 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.8958999999999998e-05 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384551+0000", 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.2574000000000002e-05 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.214 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384604+0000", 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.01652721 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.215 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166932 13.5 13:adf9a0f4:::benchmark_data_smithi064_501823_object56930:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.432068+0000", 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.72141156200001, 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95273398099999995, 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.216 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166932 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.217 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432068+0000", 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432068+0000", 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.218 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432066+0000", 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.008e-05 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432076+0000", 2023-06-10T21:51:38.219 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2539e-05 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432089+0000", 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4320000000000002e-06 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.220 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432096+0000", 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1499000000000003e-05 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432147+0000", 2023-06-10T21:51:38.221 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0778000000000001e-05 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.432158+0000", 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93672868600000003 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.222 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.368887+0000", 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012782599999999999 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.369015+0000", 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00012022 2023-06-10T21:51:38.223 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.369135+0000", 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0055580550000000001 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.224 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.374693+0000", 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0095971500000000005 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384290+0000", 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000361758 2023-06-10T21:51:38.225 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384652+0000", 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.9839999999999997e-05 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.226 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384702+0000", 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000100018 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384802+0000", 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.95273398099999995 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.227 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166973 13.5 13:aa809b62:::benchmark_data_smithi064_501823_object56971:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.524715+0000", 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.62876466199998, 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86039613999999998, 2023-06-10T21:51:38.228 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166973 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.229 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524715+0000", 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.230 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524715+0000", 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524713+0000", 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.153e-05 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.231 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524724+0000", 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0584000000000001e-05 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.232 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524745+0000", 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.038e-06 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.524753+0000", 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00028867999999999998 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.233 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.525041+0000", 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.401e-05 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.234 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.525055+0000", 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84500488799999995 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.370060+0000", 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000114152 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.235 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.370175+0000", 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000164477 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.370339+0000", 2023-06-10T21:51:38.236 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0043613849999999997 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.374700+0000", 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010126381 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.237 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384827+0000", 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4567000000000004e-05 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384921+0000", 2023-06-10T21:51:38.238 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6636000000000004e-05 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384988+0000", 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000122891 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.239 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.240 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.240 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.385111+0000", 2023-06-10T21:51:38.240 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.86039613999999998 2023-06-10T21:51:38.240 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.240 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166975 13.5 13:a7d731c9:::benchmark_data_smithi064_501823_object56973:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.526300+0000", 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.62717893400003, 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.85901286200000004, 2023-06-10T21:51:38.241 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166975 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.242 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526300+0000", 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.243 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526300+0000", 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526299+0000", 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0029999999999997e-06 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.244 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526306+0000", 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2096000000000001e-05 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.245 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526318+0000", 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2129999999999996e-06 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526324+0000", 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6168000000000002e-05 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.246 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526380+0000", 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4399999999999994e-06 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.526390+0000", 2023-06-10T21:51:38.247 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.845868022 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.372258+0000", 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8287999999999994e-05 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.248 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.372326+0000", 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0214000000000001e-05 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.372406+0000", 2023-06-10T21:51:38.249 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0047407389999999999 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.377147+0000", 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077213630000000002 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.250 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.384868+0000", 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000294713 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.385163+0000", 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.8244000000000001e-05 2023-06-10T21:51:38.251 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.385211+0000", 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00010207 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.252 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.385313+0000", 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.85901286200000004 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.253 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:166989 13.5 13:a0cf5f7c:::benchmark_data_smithi064_501823_object56987:head [delete] snapc 0=[] ondisk+write+known_if_redirected e406)", 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:22.540757+0000", 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 435.61272243600001, 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84706883600000005, 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.254 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 166989 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540757+0000", 2023-06-10T21:51:38.255 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540757+0000", 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.256 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540755+0000", 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9699999999999999e-06 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540763+0000", 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5658000000000002e-05 2023-06-10T21:51:38.257 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540779+0000", 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4400000000000002e-06 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.258 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540785+0000", 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.596e-05 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540841+0000", 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.2130000000000008e-06 2023-06-10T21:51:38.259 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:22.540850+0000", 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.83262279800000005 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.260 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.373472+0000", 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7131000000000003e-05 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.373559+0000", 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000107679 2023-06-10T21:51:38.261 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 2,3", 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.373667+0000", 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005593822 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.262 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.379261+0000", 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0071752589999999998 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.386436+0000", 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.001214702 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.263 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.387651+0000", 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.6648999999999999e-05 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.264 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.387708+0000", 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000118212 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:23.387826+0000", 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.84706883600000005 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.265 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184499 13.2 13:4bc9d582:::benchmark_data_smithi064_501823_object74497:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e410)", 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.329906+0000", 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.82357330500002, 2023-06-10T21:51:38.266 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2613112289999999, 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "reached pg", 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184499 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.267 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329906+0000", 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.268 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329906+0000", 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329905+0000", 2023-06-10T21:51:38.269 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3119999999999999e-06 2023-06-10T21:51:38.270 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.270 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.270 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.270 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329911+0000", 2023-06-10T21:51:38.270 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.154e-05 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329923+0000", 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4269999999999999e-06 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.271 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329928+0000", 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4696999999999994e-05 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.272 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.329993+0000", 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2312e-05 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.330005+0000", 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2361479980000001 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.273 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.566153+0000", 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00041076300000000002 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for missing object", 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.566564+0000", 2023-06-10T21:51:38.274 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.024578836999999999 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.591143+0000", 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4705999999999999e-05 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.275 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.591217+0000", 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2613112289999999 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.276 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184518 13.2 13:4b9f23eb:::benchmark_data_smithi064_501823_object74516:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.357058+0000", 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.79642104800001, 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2318183089999999, 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.277 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184518 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357058+0000", 2023-06-10T21:51:38.278 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357058+0000", 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.279 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357056+0000", 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5640000000000002e-06 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357066+0000", 2023-06-10T21:51:38.280 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7486000000000002e-05 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357083+0000", 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6109999999999996e-06 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.281 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357091+0000", 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5129000000000006e-05 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.282 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357156+0000", 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3441999999999999e-05 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.357169+0000", 2023-06-10T21:51:38.283 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2094487330000001 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.566618+0000", 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000123576 2023-06-10T21:51:38.284 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.566742+0000", 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016254099999999999 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.285 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.286 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.286 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.566904+0000", 2023-06-10T21:51:38.286 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0096324210000000004 2023-06-10T21:51:38.286 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.286 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576537+0000", 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012175257 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.287 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588712+0000", 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7082999999999994e-05 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.288 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588789+0000", 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.3383999999999997e-05 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588822+0000", 2023-06-10T21:51:38.289 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.4271999999999998e-05 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588877+0000", 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2318183089999999 2023-06-10T21:51:38.290 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184529 13.2 13:4d14406f:::benchmark_data_smithi064_501823_object74527:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.291 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.372582+0000", 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.780897676, 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.216419892, 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.292 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184529 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372582+0000", 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.293 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372582+0000", 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.294 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372580+0000", 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.0749999999999998e-06 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372588+0000", 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4165000000000001e-05 2023-06-10T21:51:38.295 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372602+0000", 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0330000000000003e-06 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.296 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.372609+0000", 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0021225620000000001 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.374732+0000", 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1946999999999999e-05 2023-06-10T21:51:38.297 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.374744+0000", 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1938691669999999 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.298 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.568613+0000", 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.543e-05 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.568698+0000", 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000105431 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.299 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.568804+0000", 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077369600000000002 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576541+0000", 2023-06-10T21:51:38.300 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012199531 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588740+0000", 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000167944 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.301 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588908+0000", 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.1448000000000001e-05 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588940+0000", 2023-06-10T21:51:38.302 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2028999999999999e-05 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589002+0000", 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.216419892 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.303 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184557 13.2 13:4f264bd7:::benchmark_data_smithi064_501823_object74555:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.419251+0000", 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.73422878999997, 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1698929309999999, 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.304 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184557 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.305 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419251+0000", 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419251+0000", 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:38.306 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419250+0000", 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5709999999999997e-06 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.307 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419256+0000", 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1279999999999996e-06 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419262+0000", 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7659999999999998e-06 2023-06-10T21:51:38.308 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419267+0000", 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4213000000000003e-05 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.309 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419331+0000", 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2218999999999999e-05 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.419343+0000", 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.15092401 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.310 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.570267+0000", 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5315999999999993e-05 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.311 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.570362+0000", 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.3584999999999993e-05 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.570446+0000", 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006098834 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.312 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576545+0000", 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012219895999999999 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.588765+0000", 2023-06-10T21:51:38.313 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00029008000000000001 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589055+0000", 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7079000000000002e-05 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.314 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589092+0000", 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.1753999999999999e-05 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589144+0000", 2023-06-10T21:51:38.315 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1698929309999999 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184570 13.2 13:4dd56e99:::benchmark_data_smithi064_501823_object74568:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.437167+0000", 2023-06-10T21:51:38.316 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.71631195100002, 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.152154267, 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.317 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184570 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437167+0000", 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.318 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437167+0000", 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999986 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437166+0000", 2023-06-10T21:51:38.319 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7700000000000001e-06 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437174+0000", 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2396e-05 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.320 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437186+0000", 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0149999999999998e-06 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437193+0000", 2023-06-10T21:51:38.321 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4410000000000002e-05 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437257+0000", 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1148999999999999e-05 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.322 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.437269+0000", 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1343157150000001 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.571584+0000", 2023-06-10T21:51:38.323 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1259000000000003e-05 2023-06-10T21:51:38.324 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.324 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.324 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.324 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.571655+0000", 2023-06-10T21:51:38.324 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6088999999999996e-05 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.571732+0000", 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0048169989999999998 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.325 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576549+0000", 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.012632921 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589182+0000", 2023-06-10T21:51:38.326 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.9601999999999999e-05 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589211+0000", 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.5618000000000001e-05 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.327 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589237+0000", 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4963000000000006e-05 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589322+0000", 2023-06-10T21:51:38.328 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.152154267 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184598 13.2 13:43f8b83a:::benchmark_data_smithi064_501823_object74596:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.472303+0000", 2023-06-10T21:51:38.329 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.68117610899998, 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1173167580000001, 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184598 2023-06-10T21:51:38.330 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472303+0000", 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.331 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472303+0000", 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472301+0000", 2023-06-10T21:51:38.332 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1910000000000003e-06 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472309+0000", 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3515e-05 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.333 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472322+0000", 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0260000000000002e-06 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472329+0000", 2023-06-10T21:51:38.334 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.1241000000000005e-05 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472390+0000", 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2714999999999999e-05 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.335 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.472403+0000", 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1013985239999999 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.336 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.573802+0000", 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000113725 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.573915+0000", 2023-06-10T21:51:38.337 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.9151000000000001e-05 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.574005+0000", 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0050931600000000002 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.338 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.339 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.339 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.579098+0000", 2023-06-10T21:51:38.339 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.010273414999999999 2023-06-10T21:51:38.339 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.339 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589371+0000", 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.1303000000000004e-05 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.340 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589452+0000", 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5668000000000003e-05 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.341 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589508+0000", 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000111956 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589620+0000", 2023-06-10T21:51:38.342 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1173167580000001 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.343 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184600 13.2 13:4305fbd2:::benchmark_data_smithi064_501823_object74598:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.477643+0000", 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.67583652600001, 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1122451739999999, 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.344 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184600 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.345 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.346 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.346 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477643+0000", 2023-06-10T21:51:38.346 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.346 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.346 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477643+0000", 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477641+0000", 2023-06-10T21:51:38.347 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4243999999999999e-05 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477655+0000", 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7686999999999999e-05 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.348 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477673+0000", 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.3429999999999998e-06 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477680+0000", 2023-06-10T21:51:38.349 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8157000000000005e-05 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477748+0000", 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1469e-05 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.350 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.477760+0000", 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.097260374 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.575020+0000", 2023-06-10T21:51:38.351 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00011777099999999999 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.575138+0000", 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015163899999999999 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.352 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.575289+0000", 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00689268 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.582182+0000", 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0072286210000000002 2023-06-10T21:51:38.353 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589411+0000", 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027356800000000002 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.354 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589684+0000", 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0418999999999999e-05 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589745+0000", 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000143373 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.355 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589888+0000", 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1122451739999999 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.356 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184614 13.2 13:451b457d:::benchmark_data_smithi064_501823_object74612:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.492545+0000", 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.66093473699999, 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0976371840000001, 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.357 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184614 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.358 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492545+0000", 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492545+0000", 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.359 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492544+0000", 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5897e-05 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.360 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492560+0000", 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.1619000000000001e-05 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492581+0000", 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1534e-05 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.361 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492593+0000", 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.4307000000000005e-05 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492667+0000", 2023-06-10T21:51:38.362 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2814e-05 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.492680+0000", 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.083909386 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.363 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576589+0000", 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000106519 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576696+0000", 2023-06-10T21:51:38.364 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3010999999999994e-05 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.576789+0000", 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054000519999999998 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.365 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.582189+0000", 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0077639889999999998 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.589953+0000", 2023-06-10T21:51:38.366 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0024999999999994e-05 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590023+0000", 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0143999999999999e-05 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.367 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590083+0000", 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.8863999999999994e-05 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590182+0000", 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0976371840000001 2023-06-10T21:51:38.368 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184618 13.2 13:4e02b4f5:::benchmark_data_smithi064_501823_object74616:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.498735+0000", 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.65474415599999, 2023-06-10T21:51:38.369 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0916862439999999, 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184618 2023-06-10T21:51:38.370 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.498735+0000", 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.371 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.498735+0000", 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.498735+0000", 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.4060000000000001e-06 2023-06-10T21:51:38.372 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.498738+0000", 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.002830122 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.373 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.501568+0000", 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0766e-05 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.501579+0000", 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0320000000000004e-05 2023-06-10T21:51:38.374 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.501649+0000", 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1600000000000001e-05 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.375 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.501661+0000", 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.076183903 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.577845+0000", 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000121044 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.376 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.577966+0000", 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.7521000000000004e-05 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.578063+0000", 2023-06-10T21:51:38.377 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0067569209999999999 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.584820+0000", 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0054241410000000004 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.378 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590244+0000", 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.5009999999999999e-05 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590279+0000", 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.7651000000000002e-05 2023-06-10T21:51:38.379 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590327+0000", 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4357000000000005e-05 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.380 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590421+0000", 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0916862439999999 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.381 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184691 13.2 13:426af13d:::benchmark_data_smithi064_501823_object74689:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.629752+0000", 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.523727162, 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96093546100000005, 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.382 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184691 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629752+0000", 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.383 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629752+0000", 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.384 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629750+0000", 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5806000000000001e-05 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629765+0000", 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0109999999999999e-05 2023-06-10T21:51:38.385 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629786+0000", 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.4409999999999993e-06 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.386 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629794+0000", 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1221000000000001e-05 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629865+0000", 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3674e-05 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.387 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.629879+0000", 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94931835600000003 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.388 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.579197+0000", 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000116142 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.579313+0000", 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000142829 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.389 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.579456+0000", 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0053683940000000003 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.584825+0000", 2023-06-10T21:51:38.390 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.005652331 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590477+0000", 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4199e-05 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.391 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590541+0000", 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4922000000000001e-05 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590586+0000", 2023-06-10T21:51:38.392 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000101556 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590688+0000", 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.96093546100000005 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.393 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "osd_op(client.46829.0:184699 13.2 13:4eea3950:::benchmark_data_smithi064_501823_object74697:head [delete] snapc 0=[] ondisk+write+known_if_redirected e410)", 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:44:45.644695+0000", 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 412.50878478300001, 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94676181599999998, 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:38.394 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client_info": { 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client": "client.46829", 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: "client_addr": "172.21.15.64:0/84078936", 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: "tid": 184699 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:38.395 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644695+0000", 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644695+0000", 2023-06-10T21:51:38.396 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644694+0000", 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.5439999999999998e-06 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.397 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644699+0000", 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0193999999999999e-05 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644709+0000", 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0929999999999999e-06 2023-06-10T21:51:38.398 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644716+0000", 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6636000000000004e-05 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.399 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644782+0000", 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1775e-05 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for peered", 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:45.644794+0000", 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.93619650799999998 2023-06-10T21:51:38.400 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.580991+0000", 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0469000000000006e-05 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:38.401 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.581061+0000", 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7850999999999996e-05 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "waiting for subops from 3,7", 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.581149+0000", 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.006110819 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.402 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "op_commit", 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.587260+0000", 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.0034804329999999998 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.403 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.590740+0000", 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00057654600000000005 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "sub_op_commit_rec", 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.591317+0000", 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.4589999999999998e-05 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.404 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "commit_sent", 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.591361+0000", 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.5191000000000004e-05 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:38.405 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:44:46.591456+0000", 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.94676181599999998 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:38.406 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:38.407 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-06-10T21:51:38.408 INFO:tasks.thrashosds.thrasher:inning osd 2023-06-10T21:51:38.408 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-06-10T21:51:38.408 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd in 1 2023-06-10T21:51:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[180297]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[180297]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-06-10T21:51:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[180297]: osdmap e511: 8 total, 8 up, 7 in 2023-06-10T21:51:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[186706]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[186706]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-06-10T21:51:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:38 smithi064 ceph-mon[186706]: osdmap e511: 8 total, 8 up, 7 in 2023-06-10T21:51:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:38 smithi152 ceph-mon[156819]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:38 smithi152 ceph-mon[156819]: failed to encode map e511 with expected crc 2023-06-10T21:51:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:38 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3475191612' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-06-10T21:51:38.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:38 smithi152 ceph-mon[156819]: osdmap e511: 8 total, 8 up, 7 in 2023-06-10T21:51:38.807 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:38.929 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-10T21:51:39.285 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:39.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: pgmap v1819: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-06-10T21:51:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: osdmap e512: 8 total, 8 up, 7 in 2023-06-10T21:51:39.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.661 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[180297]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: pgmap v1819: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-06-10T21:51:39.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.662 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: osdmap e512: 8 total, 8 up, 7 in 2023-06-10T21:51:39.663 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.664 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.665 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:39 smithi064 ceph-mon[186706]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: pgmap v1819: 73 pgs: 23 peering, 50 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-06-10T21:51:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: osdmap e512: 8 total, 8 up, 7 in 2023-06-10T21:51:39.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.696 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:39.697 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:39 smithi152 ceph-mon[156819]: failed to encode map e512 with expected crc 2023-06-10T21:51:40.550 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:40 smithi152 ceph-mon[156819]: pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 925 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:40.610 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:40 smithi064 ceph-mon[180297]: pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 925 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:40.611 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:40 smithi064 ceph-mon[186706]: pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 925 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:40.725 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:40.725 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:40.726 INFO:teuthology.orchestra.run.smithi152.stdout: "num_ops": 0 2023-06-10T21:51:40.726 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:40.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:40 smithi064 conmon[180272]: 2023-06-10T21:51:40.605+0000 7f0f67ab4700 -1 mon.a@0(leader).osd e512 definitely_dead 0 2023-06-10T21:51:41.355 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-10T21:51:41.437 INFO:teuthology.orchestra.run.smithi064.stderr:marked in osd.1. 2023-06-10T21:51:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/861752193' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:41 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/861752193' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:41 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.700 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/861752193' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.701 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:41 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-10T21:51:41.713 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:42.297 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-06-10T21:51:42.297 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-06-10T21:51:42.297 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd pool set device_health_metrics pgp_num 31 2023-06-10T21:51:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[180297]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[180297]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-10T21:51:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[180297]: osdmap e513: 8 total, 8 up, 8 in 2023-06-10T21:51:42.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[180297]: pgmap v1823: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[186706]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[186706]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-10T21:51:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[186706]: osdmap e513: 8 total, 8 up, 8 in 2023-06-10T21:51:42.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:42 smithi064 ceph-mon[186706]: pgmap v1823: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:42 smithi152 ceph-mon[156819]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:42 smithi152 ceph-mon[156819]: failed to encode map e513 with expected crc 2023-06-10T21:51:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:42 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-10T21:51:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:42 smithi152 ceph-mon[156819]: osdmap e513: 8 total, 8 up, 8 in 2023-06-10T21:51:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:42 smithi152 ceph-mon[156819]: pgmap v1823: 73 pgs: 9 peering, 64 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:42.736 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:43.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:42] "GET /metrics HTTP/1.1" 200 221796 "" "Prometheus/2.33.4" 2023-06-10T21:51:43.181 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [], 2023-06-10T21:51:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "complaint_time": 30, 2023-06-10T21:51:43.182 INFO:teuthology.orchestra.run.smithi152.stdout: "num_blocked_ops": 0 2023-06-10T21:51:43.182 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:43.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: osdmap e514: 8 total, 8 up, 8 in 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[180297]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: osdmap e514: 8 total, 8 up, 8 in 2023-06-10T21:51:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:43 smithi064 ceph-mon[186706]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: osdmap e514: 8 total, 8 up, 8 in 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.695 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:43 smithi152 ceph-mon[156819]: failed to encode map e514 with expected crc 2023-06-10T21:51:43.911 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-10T21:51:44.261 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.b/config 2023-06-10T21:51:44.490 INFO:teuthology.orchestra.run.smithi064.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-10T21:51:44.716 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:44 smithi064 ceph-mon[186706]: pgmap v1825: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:44.716 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:44 smithi064 ceph-mon[180297]: pgmap v1825: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:44.795 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:44 smithi152 ceph-mon[156819]: pgmap v1825: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 906 B/s, 1 keys/s, 3 objects/s recovering 2023-06-10T21:51:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-10T21:51:45.244 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-06-10T21:51:45.522 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:45 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2048795060' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.522 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:45 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.615 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:45 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2048795060' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:45 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:45 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2048795060' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:45 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-06-10T21:51:45.714 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2023-06-10T21:51:45.714 INFO:teuthology.orchestra.run.smithi152.stdout: "size": 20, 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 600, 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "ops": [ 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.b 512/511 [PushOp(3:d0630e4c:::mds0_sessionmap:head, version: 60'1, data_included: [], data_size: 0, omap_header_size: 14, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:d0630e4c:::mds0_sessionmap:head@60'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.440675+0000", 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2705779379999997, 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024488299999999999, 2023-06-10T21:51:45.715 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440675+0000", 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.716 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440675+0000", 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.717 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.721 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.721 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440674+0000", 2023-06-10T21:51:45.721 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.5787999999999999e-05 2023-06-10T21:51:45.721 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440690+0000", 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7503000000000002e-05 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.722 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440708+0000", 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1785e-05 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440719+0000", 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0001999999999999e-05 2023-06-10T21:51:45.723 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440779+0000", 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.2329999999999995e-06 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.724 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440789+0000", 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00013108400000000001 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.440920+0000", 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024488299999999999 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.725 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.6 512/511 [PushOp(3:654134d2:::mds0_openfiles.0:head, version: 60'1, data_included: [], data_size: 0, omap_header_size: 42, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:654134d2:::mds0_openfiles.0:head@60'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.450046+0000", 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2612062440000003, 2023-06-10T21:51:45.726 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025967600000000003, 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450046+0000", 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.727 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450046+0000", 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.728 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450046+0000", 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8031000000000001e-05 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450064+0000", 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2832e-05 2023-06-10T21:51:45.729 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450077+0000", 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.9019999999999999e-06 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.730 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450084+0000", 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7924000000000003e-05 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450142+0000", 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7110000000000001e-06 2023-06-10T21:51:45.731 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450151+0000", 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015500900000000001 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.732 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.450306+0000", 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00025967600000000003 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.e 512/511 [PushOp(3:717a0223:::608.00000000:head, version: 60'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:717a0223:::608.00000000:head@60'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.733 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.452339+0000", 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2589133690000001, 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024504799999999999, 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.734 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452339+0000", 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452339+0000", 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:45.735 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452339+0000", 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.3818000000000001e-05 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.736 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452362+0000", 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.9958000000000001e-05 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452382+0000", 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1075e-05 2023-06-10T21:51:45.737 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452393+0000", 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.7877000000000002e-05 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.738 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452451+0000", 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.8960000000000003e-06 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452459+0000", 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000125239 2023-06-10T21:51:45.739 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.452584+0000", 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024504799999999999 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.740 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.1d 512/511 [PushOp(3:b871830b:::605.00000000:head, version: 60'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:b871830b:::605.00000000:head@60'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.454468+0000", 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2567849400000002, 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026069199999999998, 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.741 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454468+0000", 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.742 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454468+0000", 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454467+0000", 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1235e-05 2023-06-10T21:51:45.743 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454478+0000", 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3575e-05 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.744 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454492+0000", 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.2320000000000004e-06 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454499+0000", 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.6796000000000002e-05 2023-06-10T21:51:45.745 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454566+0000", 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.4739999999999994e-06 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.746 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454575+0000", 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015307 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.454728+0000", 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026069199999999998 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.747 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.f 512/511 [PushOp(3:f1f69726:::400.00000000:head, version: 60'1, data_included: [0~22], data_size: 22, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:f1f69726:::400.00000000:head@60'1, size: 22, copy_subset: [0~22], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:22, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.456113+0000", 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2551396549999998, 2023-06-10T21:51:45.748 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038354699999999998, 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456113+0000", 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.749 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456113+0000", 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.750 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456112+0000", 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7266e-05 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456129+0000", 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2623999999999999e-05 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.751 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456152+0000", 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2937e-05 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.752 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456165+0000", 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.7951e-05 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456243+0000", 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 9.3209999999999994e-06 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.753 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456252+0000", 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024428500000000001 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.456497+0000", 2023-06-10T21:51:45.754 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038354699999999998 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.1c 512/511 [PushOp(3:3ed09add:::607.00000000:head, version: 60'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:3ed09add:::607.00000000:head@60'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.458113+0000", 2023-06-10T21:51:45.755 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2531395749999996, 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027835799999999998, 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.756 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458113+0000", 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458113+0000", 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.757 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458112+0000", 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1331999999999999e-05 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458124+0000", 2023-06-10T21:51:45.758 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3777999999999999e-05 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458138+0000", 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0600000000000002e-06 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.759 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458145+0000", 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.9500999999999997e-05 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458214+0000", 2023-06-10T21:51:45.760 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.6910000000000003e-06 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458223+0000", 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016861799999999999 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.761 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.458391+0000", 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00027835799999999998 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.762 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.15 512/511 [PushOp(3:a93a17c2:::604.00000000:head, version: 60'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:a93a17c2:::604.00000000:head@60'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.459657+0000", 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.251595687, 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024693399999999998, 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.763 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459657+0000", 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.764 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459657+0000", 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459656+0000", 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2981e-05 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.765 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459669+0000", 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7039000000000001e-05 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.766 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459686+0000", 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.2266000000000001e-05 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459699+0000", 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.6001000000000006e-05 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.767 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459775+0000", 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 8.7609999999999996e-06 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459783+0000", 2023-06-10T21:51:45.768 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000120536 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.459904+0000", 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024693399999999998 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.769 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.15 512/511 [PushOp(3:aa448500:::500.00000000:head, version: 60'3, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:aa448500:::500.00000000:head@60'3, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.502676+0000", 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.2085771049999998, 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035995599999999998, 2023-06-10T21:51:45.770 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502676+0000", 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.771 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502676+0000", 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502674+0000", 2023-06-10T21:51:45.772 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.8938999999999999e-05 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502693+0000", 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2393000000000001e-05 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.773 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502715+0000", 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0074e-05 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502725+0000", 2023-06-10T21:51:45.774 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.2378000000000005e-05 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502788+0000", 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.1369999999999996e-06 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.775 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.502795+0000", 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024071699999999999 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.503036+0000", 2023-06-10T21:51:45.776 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035995599999999998 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.13 512/511 [PushOp(3:cd5a64a3:::100.00000000:head, version: 60'1, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 10, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:cd5a64a3:::100.00000000:head@60'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.546427+0000", 2023-06-10T21:51:45.777 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.164826025, 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022222999999999999, 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.778 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546427+0000", 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546427+0000", 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.779 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546426+0000", 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.0754000000000002e-05 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546447+0000", 2023-06-10T21:51:45.780 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.1749999999999999e-05 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546459+0000", 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4983999999999999e-05 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.781 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546474+0000", 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4.1312e-05 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546515+0000", 2023-06-10T21:51:45.782 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9800000000000003e-06 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546521+0000", 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000127971 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.783 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.546649+0000", 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022222999999999999 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.784 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.13 512/511 [PushOp(3:c91d4a1d:::mds0_inotable:head, version: 60'2, data_included: [0~34], data_size: 34, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:c91d4a1d:::mds0_inotable:head@60'2, size: 34, copy_subset: [0~34], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:34, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.589440+0000", 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.1218123670000004, 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038079699999999999, 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.785 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589440+0000", 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589440+0000", 2023-06-10T21:51:45.786 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999981 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589438+0000", 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.7839999999999997e-05 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.787 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589476+0000", 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.7027000000000001e-05 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589503+0000", 2023-06-10T21:51:45.788 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.3923999999999999e-05 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589517+0000", 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.0224999999999999e-05 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.789 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589587+0000", 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8820000000000003e-06 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589594+0000", 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00022699600000000001 2023-06-10T21:51:45.790 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.589821+0000", 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00038079699999999999 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.791 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 512/511 [PushOp(3:2f55791f:::606.00000000:head, version: 60'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:2f55791f:::606.00000000:head@60'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.633432+0000", 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.0778209319999998, 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00030658500000000001, 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.792 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633432+0000", 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.793 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633432+0000", 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633431+0000", 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.7892000000000001e-05 2023-06-10T21:51:45.794 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633449+0000", 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4669e-05 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.795 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633464+0000", 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 7.926e-06 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633472+0000", 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.5541000000000001e-05 2023-06-10T21:51:45.796 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633537+0000", 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0913e-05 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.797 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633548+0000", 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.000190323 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.633738+0000", 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00030658500000000001 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.798 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 512/511 [PushOp(3:287034c4:::1.00000000.inode:head, version: 60'4, data_included: [0~554], data_size: 554, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:287034c4:::1.00000000.inode:head@60'4, size: 554, copy_subset: [0~554], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:554, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.676223+0000", 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 7.035029529, 2023-06-10T21:51:45.799 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026525499999999999, 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676223+0000", 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.800 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676223+0000", 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.999999 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.801 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676222+0000", 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0733e-05 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676233+0000", 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4822e-05 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.802 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676248+0000", 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8349999999999998e-06 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.803 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676255+0000", 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.0362000000000002e-05 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676315+0000", 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.8719999999999996e-06 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.804 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676322+0000", 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00016638300000000001 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.676488+0000", 2023-06-10T21:51:45.805 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00026525499999999999 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 512/511 [PushOp(3:292cf221:::200.00000000:head, version: 77'22, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:292cf221:::200.00000000:head@77'22, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.806 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.719221+0000", 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.9920321769999996, 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024879600000000002, 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.807 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719221+0000", 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719221+0000", 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999995 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.808 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719220+0000", 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.099e-05 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719231+0000", 2023-06-10T21:51:45.809 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4919999999999999e-05 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719246+0000", 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.0363000000000001e-05 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.810 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719256+0000", 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.0463000000000001e-05 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719307+0000", 2023-06-10T21:51:45.811 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.3230000000000003e-06 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719313+0000", 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00015633699999999999 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.812 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.719469+0000", 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00024879600000000002 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.813 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "description": "MOSDPGPush(3.14 512/511 [PushOp(3:2e2fa760:::200.00000001:head, version: 77'23, data_included: [0~4072], data_size: 4072, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:2e2fa760:::200.00000001:head@77'23, size: 4072, copy_subset: [0~4072], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4072, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "initiated_at": "2023-06-10T21:51:38.762286+0000", 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "age": 6.9489667390000003, 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035496900000000001, 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "type_data": { 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "flag_point": "started", 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: "events": [ 2023-06-10T21:51:45.814 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "initiated", 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762286+0000", 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "throttled", 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762286+0000", 2023-06-10T21:51:45.815 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 4294967295.9999976 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "header_read", 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762284+0000", 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 3.9372999999999997e-05 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.816 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "all_read", 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762323+0000", 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 2.2702e-05 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "dispatched", 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762346+0000", 2023-06-10T21:51:45.817 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 1.4019000000000001e-05 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "queued_for_pg", 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762360+0000", 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 5.9981000000000001e-05 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.818 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "reached_pg", 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762420+0000", 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 6.4389999999999997e-06 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "started", 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762426+0000", 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00021460500000000001 2023-06-10T21:51:45.819 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: { 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: "event": "done", 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: "time": "2023-06-10T21:51:38.762641+0000", 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: "duration": 0.00035496900000000001 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.820 INFO:teuthology.orchestra.run.smithi152.stdout: } 2023-06-10T21:51:45.821 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2023-06-10T21:51:45.821 INFO:teuthology.orchestra.run.smithi152.stdout:} 2023-06-10T21:51:47.061 INFO:teuthology.orchestra.run.smithi064.stdout:mon.a: {} 2023-06-10T21:51:47.071 INFO:teuthology.orchestra.run.smithi064.stdout:mon.b: {} 2023-06-10T21:51:47.080 INFO:teuthology.orchestra.run.smithi064.stdout:mon.c: {} 2023-06-10T21:51:47.081 INFO:teuthology.orchestra.run.smithi064.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-10T21:51:47.082 INFO:teuthology.orchestra.run.smithi064.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-10T21:51:47.082 INFO:teuthology.orchestra.run.smithi064.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-10T21:51:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:47 smithi152 ceph-mon[156819]: pgmap v1826: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 684 B/s, 1 keys/s, 2 objects/s recovering 2023-06-10T21:51:47.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2875203174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:51:47.810 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:47 smithi064 ceph-mon[186706]: pgmap v1826: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 684 B/s, 1 keys/s, 2 objects/s recovering 2023-06-10T21:51:47.811 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2875203174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:51:47.811 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:47 smithi064 ceph-mon[180297]: pgmap v1826: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 684 B/s, 1 keys/s, 2 objects/s recovering 2023-06-10T21:51:47.811 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2875203174' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-10T21:51:47.824 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 1 1 2023-06-10T21:51:48.188 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:48.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:48.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:48.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:51:49.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:49 smithi064 ceph-mon[180297]: pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:49.517 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:49 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:49.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:49 smithi064 ceph-mon[186706]: pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:49.517 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:49 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:49.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:49 smithi152 ceph-mon[156819]: pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:49.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:49 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:51:50.430 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.1 to 1 (10000) 2023-06-10T21:51:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:50 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1983422577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:50 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:50 smithi152 ceph-mon[156819]: pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:50.906 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1983422577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[186706]: pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:50.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1983422577' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:50.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:50 smithi064 ceph-mon[180297]: pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:51.142 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 1 1 2023-06-10T21:51:51.513 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:51 smithi152 ceph-mon[156819]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:51 smithi152 ceph-mon[156819]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:51 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:51.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:51 smithi152 ceph-mon[156819]: osdmap e515: 8 total, 8 up, 8 in 2023-06-10T21:51:51.698 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[186706]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[186706]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:51.699 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[186706]: osdmap e515: 8 total, 8 up, 8 in 2023-06-10T21:51:51.699 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[180297]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[180297]: failed to encode map e515 with expected crc 2023-06-10T21:51:51.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:51.700 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:51 smithi064 ceph-mon[180297]: osdmap e515: 8 total, 8 up, 8 in 2023-06-10T21:51:52.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:52 smithi152 ceph-mon[156819]: pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:52.867 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:52 smithi064 ceph-mon[180297]: pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:52.867 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:52 smithi064 ceph-mon[186706]: pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:51:53.157 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:51:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:51:52] "GET /metrics HTTP/1.1" 200 221882 "" "Prometheus/2.33.4" 2023-06-10T21:51:53.454 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-06-10T21:51:53.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:53 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/552202942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:53.690 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:53 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:53.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:53 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/552202942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:53.690 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:53 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:53 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/552202942' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:53.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:53 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-10T21:51:54.277 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 2 1 2023-06-10T21:51:54.656 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:54.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: osdmap e516: 8 total, 8 up, 8 in 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[186706]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: osdmap e516: 8 total, 8 up, 8 in 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:54 smithi064 ceph-mon[180297]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:54.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: osdmap e516: 8 total, 8 up, 8 in 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: failed to encode map e516 with expected crc 2023-06-10T21:51:54.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:54 smithi152 ceph-mon[156819]: pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:55.747 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[180297]: failed to encode map e517 with expected crc 2023-06-10T21:51:55.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[180297]: osdmap e517: 8 total, 8 up, 8 in 2023-06-10T21:51:55.748 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[180297]: failed to encode map e517 with expected crc 2023-06-10T21:51:55.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[186706]: failed to encode map e517 with expected crc 2023-06-10T21:51:55.748 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[186706]: osdmap e517: 8 total, 8 up, 8 in 2023-06-10T21:51:55.749 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:55 smithi064 ceph-mon[186706]: failed to encode map e517 with expected crc 2023-06-10T21:51:55.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:55 smithi152 ceph-mon[156819]: failed to encode map e517 with expected crc 2023-06-10T21:51:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:55 smithi152 ceph-mon[156819]: osdmap e517: 8 total, 8 up, 8 in 2023-06-10T21:51:55.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:55 smithi152 ceph-mon[156819]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[186706]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[186706]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[186706]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[180297]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[180297]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:56 smithi064 ceph-mon[180297]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:56.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:56 smithi152 ceph-mon[156819]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:56 smithi152 ceph-mon[156819]: failed to encode map e517 with expected crc 2023-06-10T21:51:56.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:56 smithi152 ceph-mon[156819]: pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:57.470 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.2 to 1 (10000) 2023-06-10T21:51:57.706 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:57 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:51:57.707 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:57 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:51:57.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:57 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:51:58.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 2 1 2023-06-10T21:51:58.637 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:51:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:51:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: osdmap e518: 8 total, 8 up, 8 in 2023-06-10T21:51:58.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[180297]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:51:58.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: osdmap e518: 8 total, 8 up, 8 in 2023-06-10T21:51:58.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:58 smithi064 ceph-mon[186706]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:58.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/671734908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:51:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: osdmap e518: 8 total, 8 up, 8 in 2023-06-10T21:51:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: failed to encode map e518 with expected crc 2023-06-10T21:51:58.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:58 smithi152 ceph-mon[156819]: pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:51:59.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[186706]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.891 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[186706]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[186706]: osdmap e519: 8 total, 8 up, 8 in 2023-06-10T21:51:59.892 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[186706]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[180297]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[180297]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.892 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[180297]: osdmap e519: 8 total, 8 up, 8 in 2023-06-10T21:51:59.893 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:51:59 smithi064 ceph-mon[180297]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:59 smithi152 ceph-mon[156819]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:59 smithi152 ceph-mon[156819]: failed to encode map e519 with expected crc 2023-06-10T21:51:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:59 smithi152 ceph-mon[156819]: osdmap e519: 8 total, 8 up, 8 in 2023-06-10T21:51:59.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:51:59 smithi152 ceph-mon[156819]: failed to encode map e519 with expected crc 2023-06-10T21:52:00.477 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-06-10T21:52:00.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[180297]: failed to encode map e519 with expected crc 2023-06-10T21:52:00.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:52:00.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[180297]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:00.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[186706]: failed to encode map e519 with expected crc 2023-06-10T21:52:00.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:52:00.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:00 smithi064 ceph-mon[186706]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:00.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:00 smithi152 ceph-mon[156819]: failed to encode map e519 with expected crc 2023-06-10T21:52:00.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:00 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-10T21:52:00.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:00 smithi152 ceph-mon[156819]: pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:01.272 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 6 1 2023-06-10T21:52:01.639 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:01.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[186706]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[186706]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:52:01.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[186706]: osdmap e520: 8 total, 8 up, 8 in 2023-06-10T21:52:01.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[180297]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[180297]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:52:01.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:01 smithi064 ceph-mon[180297]: osdmap e520: 8 total, 8 up, 8 in 2023-06-10T21:52:01.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:01 smithi152 ceph-mon[156819]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:01 smithi152 ceph-mon[156819]: failed to encode map e520 with expected crc 2023-06-10T21:52:01.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:01 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3834365150' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-10T21:52:01.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:01 smithi152 ceph-mon[156819]: osdmap e520: 8 total, 8 up, 8 in 2023-06-10T21:52:02.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:02 smithi064 ceph-mon[180297]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:02.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:02 smithi064 ceph-mon[186706]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:02.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:02 smithi152 ceph-mon[156819]: pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:02.991 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:02] "GET /metrics HTTP/1.1" 200 221882 "" "Prometheus/2.33.4" 2023-06-10T21:52:03.488 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.6 to 1 (10000) 2023-06-10T21:52:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1029964014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:03.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:03.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:03.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1029964014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:03.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:03 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:03.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:03.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:03 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1029964014' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:03.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:03 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:04.255 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 6 1 2023-06-10T21:52:04.498 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.500 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:04.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: osdmap e521: 8 total, 8 up, 8 in 2023-06-10T21:52:04.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.501 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.502 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[180297]: pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:04.623 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:04.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:04.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: osdmap e521: 8 total, 8 up, 8 in 2023-06-10T21:52:04.814 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.815 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:04 smithi064 ceph-mon[186706]: pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: osdmap e521: 8 total, 8 up, 8 in 2023-06-10T21:52:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: failed to encode map e521 with expected crc 2023-06-10T21:52:04.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:04 smithi152 ceph-mon[156819]: pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-06-10T21:52:05.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:05 smithi064 ceph-mon[186706]: failed to encode map e521 with expected crc 2023-06-10T21:52:05.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:05 smithi064 ceph-mon[186706]: failed to encode map e521 with expected crc 2023-06-10T21:52:05.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:05 smithi064 ceph-mon[180297]: failed to encode map e521 with expected crc 2023-06-10T21:52:05.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:05 smithi064 ceph-mon[180297]: failed to encode map e521 with expected crc 2023-06-10T21:52:05.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:05 smithi152 ceph-mon[156819]: failed to encode map e521 with expected crc 2023-06-10T21:52:05.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:05 smithi152 ceph-mon[156819]: failed to encode map e521 with expected crc 2023-06-10T21:52:07.429 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-06-10T21:52:07.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:07 smithi152 ceph-mon[156819]: pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 70 B/s, 0 objects/s recovering 2023-06-10T21:52:07.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:07.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:07 smithi064 ceph-mon[180297]: pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 70 B/s, 0 objects/s recovering 2023-06-10T21:52:07.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:07.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:07 smithi064 ceph-mon[186706]: pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 70 B/s, 0 objects/s recovering 2023-06-10T21:52:07.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-10T21:52:08.149 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 7 1 2023-06-10T21:52:08.521 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:08.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:08 smithi152 ceph-mon[156819]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:08 smithi152 ceph-mon[156819]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:08 smithi152 ceph-mon[156819]: osdmap e522: 8 total, 8 up, 8 in 2023-06-10T21:52:08.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:08 smithi152 ceph-mon[156819]: pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:08.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[180297]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[180297]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.713 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[180297]: osdmap e522: 8 total, 8 up, 8 in 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[180297]: pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[186706]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[186706]: failed to encode map e522 with expected crc 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1189063815' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[186706]: osdmap e522: 8 total, 8 up, 8 in 2023-06-10T21:52:08.714 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:08 smithi064 ceph-mon[186706]: pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:10.389 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.7 to 1 (10000) 2023-06-10T21:52:10.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:10 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:10.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:10 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:10.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:10 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:11.112 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 7 1 2023-06-10T21:52:11.495 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: osdmap e523: 8 total, 8 up, 8 in 2023-06-10T21:52:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[186706]: osdmap e524: 8 total, 8 up, 8 in 2023-06-10T21:52:11.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: osdmap e523: 8 total, 8 up, 8 in 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:11 smithi064 ceph-mon[180297]: osdmap e524: 8 total, 8 up, 8 in 2023-06-10T21:52:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1024988315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:11.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: osdmap e523: 8 total, 8 up, 8 in 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: failed to encode map e523 with expected crc 2023-06-10T21:52:11.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:11 smithi152 ceph-mon[156819]: osdmap e524: 8 total, 8 up, 8 in 2023-06-10T21:52:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[180297]: pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.884 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:12 smithi064 ceph-mon[186706]: pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:12.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: failed to encode map e524 with expected crc 2023-06-10T21:52:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:12 smithi152 ceph-mon[156819]: pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:13.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:12] "GET /metrics HTTP/1.1" 200 221852 "" "Prometheus/2.33.4" 2023-06-10T21:52:13.461 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-06-10T21:52:13.703 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2601426834' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:13.704 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:13 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:13.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2601426834' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:13.704 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:13 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:13.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2601426834' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:13.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:13 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-10T21:52:14.306 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 0 1 2023-06-10T21:52:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[180297]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[180297]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[180297]: osdmap e525: 8 total, 8 up, 8 in 2023-06-10T21:52:14.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[180297]: pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[186706]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[186706]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[186706]: osdmap e525: 8 total, 8 up, 8 in 2023-06-10T21:52:14.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:14 smithi064 ceph-mon[186706]: pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:14.675 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:14.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:14 smithi152 ceph-mon[156819]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:14 smithi152 ceph-mon[156819]: failed to encode map e525 with expected crc 2023-06-10T21:52:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:14 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-10T21:52:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:14 smithi152 ceph-mon[156819]: osdmap e525: 8 total, 8 up, 8 in 2023-06-10T21:52:14.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:14 smithi152 ceph-mon[156819]: pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:16.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:16 smithi064 ceph-mon[180297]: pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:16.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:16 smithi064 ceph-mon[186706]: pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:16 smithi152 ceph-mon[156819]: pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:17.487 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.0 to 1 (10000) 2023-06-10T21:52:17.906 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:17 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:17.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:17 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:52:17.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:17 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:17.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:17 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:52:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:17 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:17.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:17 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:52:18.313 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 0 1 2023-06-10T21:52:18.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[180297]: failed to encode map e526 with expected crc 2023-06-10T21:52:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[180297]: osdmap e526: 8 total, 8 up, 8 in 2023-06-10T21:52:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:18.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[180297]: pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:18.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[186706]: failed to encode map e526 with expected crc 2023-06-10T21:52:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[186706]: osdmap e526: 8 total, 8 up, 8 in 2023-06-10T21:52:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:18.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:18 smithi064 ceph-mon[186706]: pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:18.796 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:18 smithi152 ceph-mon[156819]: failed to encode map e526 with expected crc 2023-06-10T21:52:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:18 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3790111369' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:18 smithi152 ceph-mon[156819]: osdmap e526: 8 total, 8 up, 8 in 2023-06-10T21:52:18.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:18.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:18 smithi152 ceph-mon[156819]: pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:19.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:19 smithi064 ceph-mon[186706]: failed to encode map e526 with expected crc 2023-06-10T21:52:19.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:19 smithi064 ceph-mon[180297]: failed to encode map e526 with expected crc 2023-06-10T21:52:19.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:19 smithi152 ceph-mon[156819]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[186706]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[186706]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[186706]: pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[180297]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[180297]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:20 smithi064 ceph-mon[180297]: pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:20.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:20 smithi152 ceph-mon[156819]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:20 smithi152 ceph-mon[156819]: failed to encode map e526 with expected crc 2023-06-10T21:52:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:20 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:20.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:20 smithi152 ceph-mon[156819]: pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:21.503 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-06-10T21:52:21.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:21 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2476235813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:21.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:21 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:21 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2476235813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:21.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:21 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:21.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:21 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2476235813' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:21.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:21 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-10T21:52:22.294 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 3 1 2023-06-10T21:52:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:22.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: osdmap e527: 8 total, 8 up, 8 in 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[180297]: pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:22.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: osdmap e527: 8 total, 8 up, 8 in 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:52:22.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:52:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:22 smithi064 ceph-mon[186706]: pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:22.661 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:22.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: failed to encode map e527 with expected crc 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: osdmap e527: 8 total, 8 up, 8 in 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-10T21:52:22.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:22.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:22 smithi152 ceph-mon[156819]: pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:22] "GET /metrics HTTP/1.1" 200 221818 "" "Prometheus/2.33.4" 2023-06-10T21:52:23.906 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:23 smithi064 ceph-mon[186706]: failed to encode map e527 with expected crc 2023-06-10T21:52:23.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:23 smithi064 ceph-mon[186706]: failed to encode map e527 with expected crc 2023-06-10T21:52:23.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:23 smithi064 ceph-mon[180297]: failed to encode map e527 with expected crc 2023-06-10T21:52:23.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:23 smithi064 ceph-mon[180297]: failed to encode map e527 with expected crc 2023-06-10T21:52:23.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:23 smithi152 ceph-mon[156819]: failed to encode map e527 with expected crc 2023-06-10T21:52:23.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:23 smithi152 ceph-mon[156819]: failed to encode map e527 with expected crc 2023-06-10T21:52:24.524 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.3 to 1 (10000) 2023-06-10T21:52:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:24 smithi064 ceph-mon[186706]: pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:24.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:24 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:24.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:24 smithi064 ceph-mon[180297]: pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:24.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:24 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:24.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:24 smithi152 ceph-mon[156819]: pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:24.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:24 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:25.312 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 3 1 2023-06-10T21:52:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: osdmap e528: 8 total, 8 up, 8 in 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[186706]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: osdmap e528: 8 total, 8 up, 8 in 2023-06-10T21:52:25.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.660 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:25 smithi064 ceph-mon[180297]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.687 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:25.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1330242132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: osdmap e528: 8 total, 8 up, 8 in 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:25.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:25 smithi152 ceph-mon[156819]: failed to encode map e528 with expected crc 2023-06-10T21:52:26.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: osdmap e529: 8 total, 8 up, 8 in 2023-06-10T21:52:26.522 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.523 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[180297]: pgmap v1861: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: osdmap e529: 8 total, 8 up, 8 in 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:26 smithi064 ceph-mon[186706]: pgmap v1861: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:26.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: osdmap e529: 8 total, 8 up, 8 in 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: failed to encode map e529 with expected crc 2023-06-10T21:52:26.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:26 smithi152 ceph-mon[156819]: pgmap v1861: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:27.534 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-06-10T21:52:27.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:27 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:27.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:27 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:27.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:27 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-10T21:52:28.346 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 5 1 2023-06-10T21:52:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: osdmap e530: 8 total, 8 up, 8 in 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[180297]: pgmap v1863: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:28.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: osdmap e530: 8 total, 8 up, 8 in 2023-06-10T21:52:28.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:28 smithi064 ceph-mon[186706]: pgmap v1863: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:28.725 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:28.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3833241017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: osdmap e530: 8 total, 8 up, 8 in 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: failed to encode map e530 with expected crc 2023-06-10T21:52:28.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:28 smithi152 ceph-mon[156819]: pgmap v1863: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:29.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: osdmap e531: 8 total, 8 up, 8 in 2023-06-10T21:52:29.795 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.796 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[180297]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.796 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: osdmap e531: 8 total, 8 up, 8 in 2023-06-10T21:52:29.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.797 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:29 smithi064 ceph-mon[186706]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: osdmap e531: 8 total, 8 up, 8 in 2023-06-10T21:52:29.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: failed to encode map e531 with expected crc 2023-06-10T21:52:29.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:29 smithi152 ceph-mon[156819]: failed to encode map e531 with expected crc 2023-06-10T21:52:30.551 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.5 to 1 (10000) 2023-06-10T21:52:30.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:30 smithi064 ceph-mon[180297]: pgmap v1865: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:30.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:30 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:30.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:30 smithi064 ceph-mon[186706]: pgmap v1865: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:30.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:30 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:30.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:30 smithi152 ceph-mon[156819]: pgmap v1865: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:30.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:30 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:31.360 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 5 1 2023-06-10T21:52:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[186706]: osdmap e532: 8 total, 8 up, 8 in 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:31.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:31 smithi064 ceph-mon[180297]: osdmap e532: 8 total, 8 up, 8 in 2023-06-10T21:52:31.734 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:31.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: failed to encode map e532 with expected crc 2023-06-10T21:52:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3168216430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:31.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:31 smithi152 ceph-mon[156819]: osdmap e532: 8 total, 8 up, 8 in 2023-06-10T21:52:32.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: osdmap e533: 8 total, 8 up, 8 in 2023-06-10T21:52:32.858 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[180297]: pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.859 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: osdmap e533: 8 total, 8 up, 8 in 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.860 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:32 smithi064 ceph-mon[186706]: pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:52:32.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: osdmap e533: 8 total, 8 up, 8 in 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: failed to encode map e533 with expected crc 2023-06-10T21:52:32.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:32 smithi152 ceph-mon[156819]: pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-06-10T21:52:33.120 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:32 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:32] "GET /metrics HTTP/1.1" 200 221818 "" "Prometheus/2.33.4" 2023-06-10T21:52:33.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:33 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:33.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:33 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:33.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:33 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:34.567 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-06-10T21:52:34.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2818556151' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-06-10T21:52:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-10T21:52:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:52:34.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:52:34.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2818556151' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.908 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-06-10T21:52:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-10T21:52:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:52:34.909 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:34 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:52:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2818556151' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-10T21:52:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-06-10T21:52:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-10T21:52:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-06-10T21:52:34.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:34 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-10T21:52:35.390 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd reweight 4 1 2023-06-10T21:52:35.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-10T21:52:35.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:52:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:52:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: osdmap e534: 8 total, 8 up, 8 in 2023-06-10T21:52:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.659 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[180297]: osdmap e535: 8 total, 8 up, 8 in 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:35.660 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-10T21:52:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:52:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:52:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: osdmap e534: 8 total, 8 up, 8 in 2023-06-10T21:52:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.661 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:35 smithi064 ceph-mon[186706]: osdmap e535: 8 total, 8 up, 8 in 2023-06-10T21:52:35.771 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-06-10T21:52:35.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-10T21:52:35.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: osdmap e534: 8 total, 8 up, 8 in 2023-06-10T21:52:35.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: failed to encode map e534 with expected crc 2023-06-10T21:52:35.945 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:35 smithi152 ceph-mon[156819]: osdmap e535: 8 total, 8 up, 8 in 2023-06-10T21:52:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[180297]: pgmap v1872: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 12 B/s, 1 objects/s recovering 2023-06-10T21:52:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:36 smithi064 ceph-mon[186706]: pgmap v1872: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 12 B/s, 1 objects/s recovering 2023-06-10T21:52:36.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: failed to encode map e535 with expected crc 2023-06-10T21:52:36.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:36 smithi152 ceph-mon[156819]: pgmap v1872: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 12 B/s, 1 objects/s recovering 2023-06-10T21:52:37.582 INFO:teuthology.orchestra.run.smithi064.stderr:reweighted osd.4 to 1 (10000) 2023-06-10T21:52:37.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[180297]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-06-10T21:52:37.843 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4121671249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:37.844 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:37.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[186706]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-06-10T21:52:37.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4121671249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:37.844 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:37 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:37.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:37 smithi152 ceph-mon[156819]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-06-10T21:52:37.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:37 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4121671249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:37.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:37 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:38.384 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd primary-affinity 4 1 2023-06-10T21:52:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:38.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: osdmap e536: 8 total, 8 up, 8 in 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[180297]: pgmap v1874: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:38.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: osdmap e536: 8 total, 8 up, 8 in 2023-06-10T21:52:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.659 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:38 smithi064 ceph-mon[186706]: pgmap v1874: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:38.763 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:38.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: osdmap e536: 8 total, 8 up, 8 in 2023-06-10T21:52:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: failed to encode map e536 with expected crc 2023-06-10T21:52:38.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:38 smithi152 ceph-mon[156819]: pgmap v1874: 73 pgs: 10 peering, 63 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:40 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:40.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:40 smithi064 ceph-mon[180297]: failed to encode map e536 with expected crc 2023-06-10T21:52:40.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:40 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:40.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:40 smithi064 ceph-mon[186706]: failed to encode map e536 with expected crc 2023-06-10T21:52:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:40 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:40.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:40 smithi152 ceph-mon[156819]: failed to encode map e536 with expected crc 2023-06-10T21:52:41.198 INFO:teuthology.orchestra.run.smithi064.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-06-10T21:52:41.442 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:41 smithi064 ceph-mon[180297]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:41.443 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:41 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:41.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:41 smithi064 ceph-mon[186706]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:41.443 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:41 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:41 smithi152 ceph-mon[156819]: pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 B/s, 0 objects/s recovering 2023-06-10T21:52:41.444 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:41 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-10T21:52:41.996 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-06-10T21:52:41.996 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-06-10T21:52:41.997 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-06-10T21:52:41.997 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-06-10T21:52:41.997 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-10T21:52:41.998 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T21:52:42.370 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: Cluster is now healthy 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:42.408 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[180297]: osdmap e537: 8 total, 8 up, 8 in 2023-06-10T21:52:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-10T21:52:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: Cluster is now healthy 2023-06-10T21:52:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.408 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:42.409 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:42 smithi064 ceph-mon[186706]: osdmap e537: 8 total, 8 up, 8 in 2023-06-10T21:52:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-10T21:52:42.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: Cluster is now healthy 2023-06-10T21:52:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: failed to encode map e537 with expected crc 2023-06-10T21:52:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3782516389' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-10T21:52:42.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:42 smithi152 ceph-mon[156819]: osdmap e537: 8 total, 8 up, 8 in 2023-06-10T21:52:43.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:42 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:42] "GET /metrics HTTP/1.1" 200 221807 "" "Prometheus/2.33.4" 2023-06-10T21:52:43.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.627 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: osdmap e538: 8 total, 8 up, 8 in 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[186706]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.628 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: osdmap e538: 8 total, 8 up, 8 in 2023-06-10T21:52:43.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.629 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:43 smithi064 ceph-mon[180297]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: osdmap e538: 8 total, 8 up, 8 in 2023-06-10T21:52:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: failed to encode map e538 with expected crc 2023-06-10T21:52:43.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:43 smithi152 ceph-mon[156819]: pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:44.052 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:52:44.053 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":538,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T21:52:42.179485+0000","last_up_change":"2023-06-10T21:51:23.417160+0000","last_in_change":"2023-06-10T21:51:41.423096+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":86,"last_clean_end":231,"up_from":244,"up_thru":528,"down_at":232,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6803","nonce":1109161228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6805","nonce":1109161228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6809","nonce":1109161228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6807","nonce":1109161228}]},"public_addr":"172.21.15.64:6803/1109161228","cluster_addr":"172.21.15.64:6805/1109161228","heartbeat_back_addr":"172.21.15.64:6809/1109161228","heartbeat_front_addr":"172.21.15.64:6807/1109161228","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":101,"up_from":105,"up_thru":532,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6811","nonce":215784888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6813","nonce":215784888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6817","nonce":215784888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6815","nonce":215784888}]},"public_addr":"172.21.15.64:6811/215784888","cluster_addr":"172.21.15.64:6813/215784888","heartbeat_back_addr":"172.21.15.64:6817/215784888","heartbeat_front_addr":"172.21.15.64:6815/215784888","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":534,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6819","nonce":746809109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6821","nonce":746809109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6825","nonce":746809109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6823","nonce":746809109}]},"public_addr":"172.21.15.64:6819/746809109","cluster_addr":"172.21.15.64:6821/746809109","heartbeat_back_addr":"172.21.15.64:6825/746809109","heartbeat_front_addr":"172.21.15.64:6823/746809109","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":125,"last_clean_end":218,"up_from":250,"up_thru":534,"down_at":219,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6827","nonce":235474426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6829","nonce":235474426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6833","nonce":235474426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6831","nonce":235474426}]},"public_addr":"172.21.15.64:6827/235474426","cluster_addr":"172.21.15.64:6829/235474426","heartbeat_back_addr":"172.21.15.64:6833/235474426","heartbeat_front_addr":"172.21.15.64:6831/235474426","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":493,"up_from":507,"up_thru":537,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6801","nonce":3531161690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6803","nonce":3531161690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6807","nonce":3531161690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6805","nonce":3531161690}]},"public_addr":"172.21.15.152:6801/3531161690","cluster_addr":"172.21.15.152:6803/3531161690","heartbeat_back_addr":"172.21.15.152:6807/3531161690","heartbeat_front_addr":"172.21.15.152:6805/3531161690","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":383,"up_from":390,"up_thru":534,"down_at":384,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6809","nonce":41874071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6811","nonce":41874071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6815","nonce":41874071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6813","nonce":41874071}]},"public_addr":"172.21.15.152:6809/41874071","cluster_addr":"172.21.15.152:6811/41874071","heartbeat_back_addr":"172.21.15.152:6815/41874071","heartbeat_front_addr":"172.21.15.152:6813/41874071","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":534,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":532,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":231},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":101},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":218},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":493},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":383},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.12","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":5,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":6}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.11","mappings":[{"from":3,"to":7}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/1656599775":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1184928448":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1551224238":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1338325282":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/3424306201":"2023-06-11T21:05:32.902880+0000","172.21.15.152:6835/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.64:6801/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3850833042":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.152:6834/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.152:0/1248818928":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1955159746":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/703163662":"2023-06-11T21:02:16.546685+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3731648028":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/1618902179":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/790526867":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T21:52:44.407 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:44 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/3179807084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:44.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:44 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/3179807084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:44.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:44 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/3179807084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:44.833 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-10T21:52:44.834 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd dump --format=json 2023-06-10T21:52:45.200 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:45.458 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:45 smithi064 ceph-mon[180297]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:45.459 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:45 smithi064 ceph-mon[186706]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:45.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:45 smithi152 ceph-mon[156819]: pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:46.525 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:46 smithi064 ceph-mon[180297]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:46.526 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:46 smithi064 ceph-mon[186706]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:46.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:46 smithi152 ceph-mon[156819]: pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:46.947 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:52:46.948 INFO:teuthology.orchestra.run.smithi064.stdout:{"epoch":538,"fsid":"b618768e-07d0-11ee-9b1e-001a4aab830c","created":"2023-06-10T20:53:15.981225+0000","modified":"2023-06-10T21:52:42.179485+0000","last_up_change":"2023-06-10T21:51:23.417160+0000","last_in_change":"2023-06-10T21:51:41.423096+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-10T20:56:09.705420+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-10T20:58:33.064642+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-10T20:59:24.301846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-10T20:59:24.831209+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"17854ddb-2190-4089-8177-63bd7a59d79a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":86,"last_clean_end":231,"up_from":244,"up_thru":528,"down_at":232,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6802","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6803","nonce":1109161228}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6804","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6805","nonce":1109161228}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6808","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6809","nonce":1109161228}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6806","nonce":1109161228},{"type":"v1","addr":"172.21.15.64:6807","nonce":1109161228}]},"public_addr":"172.21.15.64:6803/1109161228","cluster_addr":"172.21.15.64:6805/1109161228","heartbeat_back_addr":"172.21.15.64:6809/1109161228","heartbeat_front_addr":"172.21.15.64:6807/1109161228","state":["exists","up"]},{"osd":1,"uuid":"c0f26f57-1eae-44eb-b91a-24db061d2f83","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":101,"up_from":105,"up_thru":532,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6810","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6811","nonce":215784888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6812","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6813","nonce":215784888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6816","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6817","nonce":215784888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6814","nonce":215784888},{"type":"v1","addr":"172.21.15.64:6815","nonce":215784888}]},"public_addr":"172.21.15.64:6811/215784888","cluster_addr":"172.21.15.64:6813/215784888","heartbeat_back_addr":"172.21.15.64:6817/215784888","heartbeat_front_addr":"172.21.15.64:6815/215784888","state":["exists","up"]},{"osd":2,"uuid":"d4a979da-bdc4-46f7-aaa3-e912753f1272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":116,"up_from":120,"up_thru":534,"down_at":117,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6818","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6819","nonce":746809109}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6820","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6821","nonce":746809109}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6824","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6825","nonce":746809109}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6822","nonce":746809109},{"type":"v1","addr":"172.21.15.64:6823","nonce":746809109}]},"public_addr":"172.21.15.64:6819/746809109","cluster_addr":"172.21.15.64:6821/746809109","heartbeat_back_addr":"172.21.15.64:6825/746809109","heartbeat_front_addr":"172.21.15.64:6823/746809109","state":["exists","up"]},{"osd":3,"uuid":"efd43bfa-889e-4c76-997c-0fd3570fbc6e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":125,"last_clean_end":218,"up_from":250,"up_thru":534,"down_at":219,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6826","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6827","nonce":235474426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6828","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6829","nonce":235474426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6832","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6833","nonce":235474426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.64:6830","nonce":235474426},{"type":"v1","addr":"172.21.15.64:6831","nonce":235474426}]},"public_addr":"172.21.15.64:6827/235474426","cluster_addr":"172.21.15.64:6829/235474426","heartbeat_back_addr":"172.21.15.64:6833/235474426","heartbeat_front_addr":"172.21.15.64:6831/235474426","state":["exists","up"]},{"osd":4,"uuid":"9e5f714f-9f72-4bd3-9e4d-97286fbfb88e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":493,"up_from":507,"up_thru":537,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6801","nonce":3531161690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6803","nonce":3531161690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6807","nonce":3531161690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":3531161690},{"type":"v1","addr":"172.21.15.152:6805","nonce":3531161690}]},"public_addr":"172.21.15.152:6801/3531161690","cluster_addr":"172.21.15.152:6803/3531161690","heartbeat_back_addr":"172.21.15.152:6807/3531161690","heartbeat_front_addr":"172.21.15.152:6805/3531161690","state":["exists","up"]},{"osd":5,"uuid":"eb69283f-ccc4-4166-8c9a-05522cd8fd5c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":383,"up_from":390,"up_thru":534,"down_at":384,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6809","nonce":41874071}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6811","nonce":41874071}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6815","nonce":41874071}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":41874071},{"type":"v1","addr":"172.21.15.152:6813","nonce":41874071}]},"public_addr":"172.21.15.152:6809/41874071","cluster_addr":"172.21.15.152:6811/41874071","heartbeat_back_addr":"172.21.15.152:6815/41874071","heartbeat_front_addr":"172.21.15.152:6813/41874071","state":["exists","up"]},{"osd":6,"uuid":"834d3ada-5c5e-40f0-8520-ad53ff0648ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":534,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6817","nonce":3361850000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6819","nonce":3361850000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6823","nonce":3361850000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":3361850000},{"type":"v1","addr":"172.21.15.152:6821","nonce":3361850000}]},"public_addr":"172.21.15.152:6817/3361850000","cluster_addr":"172.21.15.152:6819/3361850000","heartbeat_back_addr":"172.21.15.152:6823/3361850000","heartbeat_front_addr":"172.21.15.152:6821/3361850000","state":["exists","up"]},{"osd":7,"uuid":"8c45d0aa-0cfc-475f-bbc1-c7ad9e224c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":532,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6825","nonce":769883648}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6827","nonce":769883648}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6830","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6831","nonce":769883648}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":769883648},{"type":"v1","addr":"172.21.15.152:6829","nonce":769883648}]},"public_addr":"172.21.15.152:6825/769883648","cluster_addr":"172.21.15.152:6827/769883648","heartbeat_back_addr":"172.21.15.152:6831/769883648","heartbeat_front_addr":"172.21.15.152:6829/769883648","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:21.935452+0000","dead_epoch":231},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:55:44.193131+0000","dead_epoch":101},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:07.400345+0000","dead_epoch":116},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:29.868489+0000","dead_epoch":218},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:56:50.531497+0000","dead_epoch":493},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:11.376596+0000","dead_epoch":383},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:32.266881+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-10T20:57:53.711439+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.12","mappings":[{"from":3,"to":6}]},{"pgid":"3.17","mappings":[{"from":5,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":6}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.11","mappings":[{"from":3,"to":7}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.64:0/1656599775":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1184928448":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6800/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1551224238":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/1338325282":"2023-06-11T21:05:32.902880+0000","172.21.15.64:0/3424306201":"2023-06-11T21:05:32.902880+0000","172.21.15.152:6835/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.64:6801/1079194554":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6801/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3850833042":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3566875446":"2023-06-11T20:58:13.320243+0000","172.21.15.152:6834/3978239224":"2023-06-11T21:04:32.578278+0000","172.21.15.152:0/1248818928":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3632777501":"2023-06-11T20:53:43.436289+0000","172.21.15.64:6800/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/1955159746":"2023-06-11T21:02:16.546685+0000","172.21.15.64:0/703163662":"2023-06-11T21:02:16.546685+0000","172.21.15.64:6800/1928672739":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/3731648028":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/3307901273":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/3450946694":"2023-06-11T20:58:13.320243+0000","172.21.15.64:0/1719072918":"2023-06-11T20:54:08.425389+0000","172.21.15.152:0/1618902179":"2023-06-11T21:04:32.578278+0000","172.21.15.64:0/790526867":"2023-06-11T21:05:32.902880+0000","172.21.15.64:6801/3618131890":"2023-06-11T21:02:16.546685+0000","172.21.15.64:6801/2180524494":"2023-06-11T20:58:13.320243+0000","172.21.15.64:6800/530031084":"2023-06-11T20:53:43.436289+0000","172.21.15.64:0/2757723192":"2023-06-11T20:54:08.425389+0000","172.21.15.64:6800/2180524494":"2023-06-11T20:58:13.320243+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-10T21:52:47.727 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-10T21:52:47.728 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-10T21:52:47.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-10T21:52:47.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-10T21:52:47.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-10T21:52:47.730 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-10T21:52:47.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-10T21:52:47.731 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-10T21:52:47.778 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:47 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4052023068' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:47.779 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:47 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4052023068' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:47.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:47 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4052023068' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-10T21:52:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:48 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:48.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:48 smithi064 ceph-mon[186706]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:48.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:48 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:48.908 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:48 smithi064 ceph-mon[180297]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:48.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:48 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:52:48.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:48 smithi152 ceph-mon[156819]: pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:49.010 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:49.264 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:49.314 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:49.475 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:50.009 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:50.010 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:50.406 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:50 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:50.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:50 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:50.443 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:50 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' 2023-06-10T21:52:51.173 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:51.175 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:52:51.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:51 smithi064 ceph-mon[186706]: pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:51.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:51 smithi064 ceph-mon[180297]: pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:51.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:51 smithi152 ceph-mon[156819]: pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:52.882 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:52 smithi064 ceph-mon[180297]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:52.883 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:52 smithi064 ceph-mon[186706]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:52.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:52 smithi152 ceph-mon[156819]: pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:53.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:52:52 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:52:52] "GET /metrics HTTP/1.1" 200 221798 "" "Prometheus/2.33.4" 2023-06-10T21:52:54.482 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:54 smithi064 ceph-mon[180297]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:54.483 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:54 smithi064 ceph-mon[186706]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:54.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:54 smithi152 ceph-mon[156819]: pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:56.832 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:56 smithi064 ceph-mon[180297]: pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:56.832 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:56 smithi064 ceph-mon[186706]: pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:56.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:56 smithi152 ceph-mon[156819]: pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:58.294 INFO:teuthology.orchestra.run.smithi064.stdout:1675037245561 2023-06-10T21:52:58.295 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-10T21:52:58.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:52:58 smithi064 ceph-mon[180297]: pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:58.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:52:58 smithi064 ceph-mon[186706]: pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:58.728 INFO:teuthology.orchestra.run.smithi064.stdout:450971566541 2023-06-10T21:52:58.729 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-10T21:52:58.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:52:58 smithi152 ceph-mon[156819]: pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:52:59.661 INFO:teuthology.orchestra.run.smithi064.stdout:2177548419093 2023-06-10T21:52:59.662 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-10T21:53:00.356 INFO:teuthology.orchestra.run.smithi064.stdout:180388627099 2023-06-10T21:53:00.356 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-10T21:53:00.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:00 smithi064 ceph-mon[186706]: pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:00.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:00 smithi064 ceph-mon[180297]: pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:00.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:00 smithi152 ceph-mon[156819]: pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:01.955 INFO:teuthology.orchestra.run.smithi064.stdout:515396075930 2023-06-10T21:53:01.955 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-10T21:53:02.009 INFO:teuthology.orchestra.run.smithi064.stdout:1047972020519 2023-06-10T21:53:02.010 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-10T21:53:02.567 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:02 smithi064 ceph-mon[186706]: pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:02.567 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:02 smithi064 ceph-mon[180297]: pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:02.588 INFO:teuthology.orchestra.run.smithi064.stdout:1073741824290 2023-06-10T21:53:02.588 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-10T21:53:02.846 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:02.905 INFO:teuthology.orchestra.run.smithi064.stdout:206158430872 2023-06-10T21:53:02.906 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-10T21:53:02.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:02 smithi152 ceph-mon[156819]: pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:02.996 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:03.026 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:03.128 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:02 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:53:02] "GET /metrics HTTP/1.1" 200 221798 "" "Prometheus/2.33.4" 2023-06-10T21:53:03.535 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:03 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:03.535 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:03 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:03.651 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:03.702 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:03.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:03 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:04.624 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:04 smithi064 ceph-mon[186706]: pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:04.624 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:04 smithi064 ceph-mon[180297]: pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:04.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:04 smithi152 ceph-mon[156819]: pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:06.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:06 smithi064 ceph-mon[186706]: pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:06.907 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:06 smithi064 ceph-mon[180297]: pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:06.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:06 smithi152 ceph-mon[156819]: pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:07.430 INFO:teuthology.orchestra.run.smithi064.stdout:180388627101 2023-06-10T21:53:07.544 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:07.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:07 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/335543159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:53:07.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:07 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/335543159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:53:07.729 INFO:teuthology.orchestra.run.smithi064.stdout:2177548419094 2023-06-10T21:53:07.807 INFO:teuthology.orchestra.run.smithi064.stdout:515396075931 2023-06-10T21:53:07.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:07 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/335543159' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-10T21:53:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1062450744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:53:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/4107568997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:53:08.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[186706]: pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:08.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1062450744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:53:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/4107568997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:53:08.658 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:08 smithi064 ceph-mon[180297]: pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:08.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1062450744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-10T21:53:08.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:08 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/4107568997' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-10T21:53:08.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:08 smithi152 ceph-mon[156819]: pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:09.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 515396075930 got 515396075931 for osd.2 2023-06-10T21:53:09.187 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:09.494 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:09.521 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627099 got 180388627101 for osd.6 2023-06-10T21:53:09.521 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:09.633 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:10.446 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2177548419093 got 2177548419094 for osd.4 2023-06-10T21:53:10.447 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:10.763 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:10 smithi064 ceph-mon[180297]: pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:10.764 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:10 smithi064 ceph-mon[186706]: pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:10.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:10 smithi152 ceph-mon[156819]: pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:11.348 INFO:teuthology.orchestra.run.smithi064.stdout:1675037245564 2023-06-10T21:53:11.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:11 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2967450135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:53:11.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:11 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2967450135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:53:11.775 INFO:teuthology.orchestra.run.smithi064.stdout:450971566544 2023-06-10T21:53:11.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:11 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2967450135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-10T21:53:12.841 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:12 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1775237994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:53:12.842 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:12 smithi064 ceph-mon[186706]: pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:12.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:12 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1775237994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:53:12.842 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:12 smithi064 ceph-mon[180297]: pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:12.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:12 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1775237994' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-10T21:53:12.944 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:12 smithi152 ceph-mon[156819]: pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:13.022 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1675037245561 got 1675037245564 for osd.5 2023-06-10T21:53:13.023 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:13.110 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:12 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:53:12] "GET /metrics HTTP/1.1" 200 221758 "" "Prometheus/2.33.4" 2023-06-10T21:53:13.127 INFO:tasks.cephadm.ceph_manager.ceph:need seq 450971566541 got 450971566544 for osd.1 2023-06-10T21:53:13.128 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:13.275 INFO:teuthology.orchestra.run.smithi064.stdout:1073741824293 2023-06-10T21:53:13.656 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:13 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/706576135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:53:13.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:13 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/706576135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:53:13.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:13 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/706576135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-10T21:53:14.754 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:14 smithi064 ceph-mon[186706]: pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:14.754 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:14 smithi064 ceph-mon[180297]: pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:14.755 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1073741824290 got 1073741824293 for osd.3 2023-06-10T21:53:14.755 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:14.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:14 smithi152 ceph-mon[156819]: pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:15.429 INFO:teuthology.orchestra.run.smithi064.stdout:206158430874 2023-06-10T21:53:15.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:15 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/1587945034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:53:15.658 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:15 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/1587945034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:53:15.707 INFO:teuthology.orchestra.run.smithi064.stdout:1047972020522 2023-06-10T21:53:15.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:15 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/1587945034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-10T21:53:16.582 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430872 got 206158430874 for osd.7 2023-06-10T21:53:16.582 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:16.656 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:16 smithi064 ceph-mon[186706]: from='client.? 172.21.15.64:0/2671043466' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:53:16.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:16 smithi064 ceph-mon[186706]: pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:16 smithi064 ceph-mon[180297]: from='client.? 172.21.15.64:0/2671043466' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:53:16.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:16 smithi064 ceph-mon[180297]: pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:16.796 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1047972020519 got 1047972020522 for osd.0 2023-06-10T21:53:16.796 DEBUG:teuthology.parallel:result is None 2023-06-10T21:53:16.797 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-10T21:53:16.797 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:53:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:16 smithi152 ceph-mon[156819]: from='client.? 172.21.15.64:0/2671043466' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-10T21:53:16.943 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:16 smithi152 ceph-mon[156819]: pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:17.166 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:18.397 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:18 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:18.397 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:18 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:18.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:18 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-10T21:53:18.806 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:53:18.808 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:53:19.552 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:19 smithi064 ceph-mon[180297]: pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:19.552 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:19 smithi064 ceph-mon[186706]: pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:19.571 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1896,"stamp":"2023-06-10T21:53:18.385643+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":3571814,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5164088,"kb_used_data":43896,"kb_used_omap":56,"kb_used_meta":5120071,"kb_avail":744797128,"statfs":{"total":767960285184,"available":762672259072,"internally_reserved":0,"allocated":44949504,"data_stored":34428544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58123,"internal_metadata":5242952949},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003233"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":574,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.194131+0000","last_change":"2023-06-10T21:52:42.194131+0000","last_active":"2023-06-10T21:52:42.194131+0000","last_peered":"2023-06-10T21:52:42.194131+0000","last_clean":"2023-06-10T21:52:42.194131+0000","last_became_active":"2023-06-10T21:52:42.193697+0000","last_became_peered":"2023-06-10T21:52:42.193697+0000","last_unstale":"2023-06-10T21:52:42.194131+0000","last_undegraded":"2023-06-10T21:52:42.194131+0000","last_fullsized":"2023-06-10T21:52:42.194131+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":216,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.412972+0000","last_change":"2023-06-10T21:52:28.538012+0000","last_active":"2023-06-10T21:52:42.412972+0000","last_peered":"2023-06-10T21:52:42.412972+0000","last_clean":"2023-06-10T21:52:42.412972+0000","last_became_active":"2023-06-10T21:52:28.537553+0000","last_became_peered":"2023-06-10T21:52:28.537553+0000","last_unstale":"2023-06-10T21:52:42.412972+0000","last_undegraded":"2023-06-10T21:52:42.412972+0000","last_fullsized":"2023-06-10T21:52:42.412972+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:47.569689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":508,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193692+0000","last_change":"2023-06-10T21:51:54.605117+0000","last_active":"2023-06-10T21:52:41.193692+0000","last_peered":"2023-06-10T21:52:41.193692+0000","last_clean":"2023-06-10T21:52:41.193692+0000","last_became_active":"2023-06-10T21:51:54.604828+0000","last_became_peered":"2023-06-10T21:51:54.604828+0000","last_unstale":"2023-06-10T21:52:41.193692+0000","last_undegraded":"2023-06-10T21:52:41.193692+0000","last_fullsized":"2023-06-10T21:52:41.193692+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:41:28.154998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":184,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195328+0000","last_change":"2023-06-10T21:52:25.522080+0000","last_active":"2023-06-10T21:52:41.195328+0000","last_peered":"2023-06-10T21:52:41.195328+0000","last_clean":"2023-06-10T21:52:41.195328+0000","last_became_active":"2023-06-10T21:52:25.521740+0000","last_became_peered":"2023-06-10T21:52:25.521740+0000","last_unstale":"2023-06-10T21:52:41.195328+0000","last_undegraded":"2023-06-10T21:52:41.195328+0000","last_fullsized":"2023-06-10T21:52:41.195328+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:43:13.791566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":47,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.190480+0000","last_change":"2023-06-10T21:51:42.454284+0000","last_active":"2023-06-10T21:52:41.190480+0000","last_peered":"2023-06-10T21:52:41.190480+0000","last_clean":"2023-06-10T21:52:41.190480+0000","last_became_active":"2023-06-10T21:51:42.454018+0000","last_became_peered":"2023-06-10T21:51:42.454018+0000","last_unstale":"2023-06-10T21:52:41.190480+0000","last_undegraded":"2023-06-10T21:52:41.190480+0000","last_fullsized":"2023-06-10T21:52:41.190480+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191603+0000","last_change":"2023-06-10T21:51:42.467047+0000","last_active":"2023-06-10T21:52:41.191603+0000","last_peered":"2023-06-10T21:52:41.191603+0000","last_clean":"2023-06-10T21:52:41.191603+0000","last_became_active":"2023-06-10T21:51:42.466703+0000","last_became_peered":"2023-06-10T21:51:42.466703+0000","last_unstale":"2023-06-10T21:52:41.191603+0000","last_undegraded":"2023-06-10T21:52:41.191603+0000","last_fullsized":"2023-06-10T21:52:41.191603+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":131,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195830+0000","last_change":"2023-06-10T21:51:35.620460+0000","last_active":"2023-06-10T21:52:41.195830+0000","last_peered":"2023-06-10T21:52:41.195830+0000","last_clean":"2023-06-10T21:52:41.195830+0000","last_became_active":"2023-06-10T21:51:35.620047+0000","last_became_peered":"2023-06-10T21:51:35.620047+0000","last_unstale":"2023-06-10T21:52:41.195830+0000","last_undegraded":"2023-06-10T21:52:41.195830+0000","last_fullsized":"2023-06-10T21:52:41.195830+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:21:04.932096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186789+0000","last_change":"2023-06-10T21:51:42.449656+0000","last_active":"2023-06-10T21:52:41.186789+0000","last_peered":"2023-06-10T21:52:41.186789+0000","last_clean":"2023-06-10T21:52:41.186789+0000","last_became_active":"2023-06-10T21:51:42.449318+0000","last_became_peered":"2023-06-10T21:51:42.449318+0000","last_unstale":"2023-06-10T21:52:41.186789+0000","last_undegraded":"2023-06-10T21:52:41.186789+0000","last_fullsized":"2023-06-10T21:52:41.186789+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:25:50.362065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":216,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196484+0000","last_change":"2023-06-10T21:52:35.612582+0000","last_active":"2023-06-10T21:52:41.196484+0000","last_peered":"2023-06-10T21:52:41.196484+0000","last_clean":"2023-06-10T21:52:41.196484+0000","last_became_active":"2023-06-10T21:52:35.611930+0000","last_became_peered":"2023-06-10T21:52:35.611930+0000","last_unstale":"2023-06-10T21:52:41.196484+0000","last_undegraded":"2023-06-10T21:52:41.196484+0000","last_fullsized":"2023-06-10T21:52:41.196484+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":301,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187413+0000","last_change":"2023-06-10T21:48:53.403919+0000","last_active":"2023-06-10T21:52:41.187413+0000","last_peered":"2023-06-10T21:52:41.187413+0000","last_clean":"2023-06-10T21:52:41.187413+0000","last_became_active":"2023-06-10T21:48:53.403593+0000","last_became_peered":"2023-06-10T21:48:53.403593+0000","last_unstale":"2023-06-10T21:52:41.187413+0000","last_undegraded":"2023-06-10T21:52:41.187413+0000","last_fullsized":"2023-06-10T21:52:41.187413+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:59.593873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":325,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196546+0000","last_change":"2023-06-10T21:51:38.545495+0000","last_active":"2023-06-10T21:52:41.196546+0000","last_peered":"2023-06-10T21:52:41.196546+0000","last_clean":"2023-06-10T21:52:41.196546+0000","last_became_active":"2023-06-10T21:51:38.438347+0000","last_became_peered":"2023-06-10T21:51:38.438347+0000","last_unstale":"2023-06-10T21:52:41.196546+0000","last_undegraded":"2023-06-10T21:52:41.196546+0000","last_fullsized":"2023-06-10T21:52:41.196546+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194542+0000","last_change":"2023-06-10T21:51:54.772285+0000","last_active":"2023-06-10T21:52:41.194542+0000","last_peered":"2023-06-10T21:52:41.194542+0000","last_clean":"2023-06-10T21:52:41.194542+0000","last_became_active":"2023-06-10T21:51:54.771899+0000","last_became_peered":"2023-06-10T21:51:54.771899+0000","last_unstale":"2023-06-10T21:52:41.194542+0000","last_undegraded":"2023-06-10T21:52:41.194542+0000","last_fullsized":"2023-06-10T21:52:41.194542+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:21:43.352659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":485,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.197541+0000","last_change":"2023-06-10T21:52:42.197541+0000","last_active":"2023-06-10T21:52:42.197541+0000","last_peered":"2023-06-10T21:52:42.197541+0000","last_clean":"2023-06-10T21:52:42.197541+0000","last_became_active":"2023-06-10T21:52:42.197119+0000","last_became_peered":"2023-06-10T21:52:42.197119+0000","last_unstale":"2023-06-10T21:52:42.197541+0000","last_undegraded":"2023-06-10T21:52:42.197541+0000","last_fullsized":"2023-06-10T21:52:42.197541+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23580,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.195127+0000","last_change":"2023-06-10T21:52:42.195127+0000","last_active":"2023-06-10T21:52:42.195127+0000","last_peered":"2023-06-10T21:52:42.195127+0000","last_clean":"2023-06-10T21:52:42.195127+0000","last_became_active":"2023-06-10T21:52:42.194712+0000","last_became_peered":"2023-06-10T21:52:42.194712+0000","last_unstale":"2023-06-10T21:52:42.195127+0000","last_undegraded":"2023-06-10T21:52:42.195127+0000","last_fullsized":"2023-06-10T21:52:42.195127+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":642,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196218+0000","last_change":"2023-06-10T21:51:38.632411+0000","last_active":"2023-06-10T21:52:41.196218+0000","last_peered":"2023-06-10T21:52:41.196218+0000","last_clean":"2023-06-10T21:52:41.196218+0000","last_became_active":"2023-06-10T21:51:38.441963+0000","last_became_peered":"2023-06-10T21:51:38.441963+0000","last_unstale":"2023-06-10T21:52:41.196218+0000","last_undegraded":"2023-06-10T21:52:41.196218+0000","last_fullsized":"2023-06-10T21:52:41.196218+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":466,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187451+0000","last_change":"2023-06-10T21:52:28.539071+0000","last_active":"2023-06-10T21:52:41.187451+0000","last_peered":"2023-06-10T21:52:41.187451+0000","last_clean":"2023-06-10T21:52:41.187451+0000","last_became_active":"2023-06-10T21:52:28.538753+0000","last_became_peered":"2023-06-10T21:52:28.538753+0000","last_unstale":"2023-06-10T21:52:41.187451+0000","last_undegraded":"2023-06-10T21:52:41.187451+0000","last_fullsized":"2023-06-10T21:52:41.187451+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:02:57.302797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195228+0000","last_change":"2023-06-10T21:51:35.623072+0000","last_active":"2023-06-10T21:52:41.195228+0000","last_peered":"2023-06-10T21:52:41.195228+0000","last_clean":"2023-06-10T21:52:41.195228+0000","last_became_active":"2023-06-10T21:51:35.622808+0000","last_became_peered":"2023-06-10T21:51:35.622808+0000","last_unstale":"2023-06-10T21:52:41.195228+0000","last_undegraded":"2023-06-10T21:52:41.195228+0000","last_fullsized":"2023-06-10T21:52:41.195228+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:28.387280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":433,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197250+0000","last_change":"2023-06-10T21:52:35.612763+0000","last_active":"2023-06-10T21:52:41.197250+0000","last_peered":"2023-06-10T21:52:41.197250+0000","last_clean":"2023-06-10T21:52:41.197250+0000","last_became_active":"2023-06-10T21:52:35.612386+0000","last_became_peered":"2023-06-10T21:52:35.612386+0000","last_unstale":"2023-06-10T21:52:41.197250+0000","last_undegraded":"2023-06-10T21:52:41.197250+0000","last_fullsized":"2023-06-10T21:52:41.197250+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":577,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196152+0000","last_change":"2023-06-10T21:51:38.442370+0000","last_active":"2023-06-10T21:52:41.196152+0000","last_peered":"2023-06-10T21:52:41.196152+0000","last_clean":"2023-06-10T21:52:41.196152+0000","last_became_active":"2023-06-10T21:51:38.442034+0000","last_became_peered":"2023-06-10T21:51:38.442034+0000","last_unstale":"2023-06-10T21:52:41.196152+0000","last_undegraded":"2023-06-10T21:52:41.196152+0000","last_fullsized":"2023-06-10T21:52:41.196152+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195115+0000","last_change":"2023-06-10T21:52:11.469475+0000","last_active":"2023-06-10T21:52:41.195115+0000","last_peered":"2023-06-10T21:52:41.195115+0000","last_clean":"2023-06-10T21:52:41.195115+0000","last_became_active":"2023-06-10T21:52:11.469147+0000","last_became_peered":"2023-06-10T21:52:11.469147+0000","last_unstale":"2023-06-10T21:52:41.195115+0000","last_undegraded":"2023-06-10T21:52:41.195115+0000","last_fullsized":"2023-06-10T21:52:41.195115+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:20:03.524750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":122,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.190717+0000","last_change":"2023-06-10T21:52:31.729715+0000","last_active":"2023-06-10T21:52:41.190717+0000","last_peered":"2023-06-10T21:52:41.190717+0000","last_clean":"2023-06-10T21:52:41.190717+0000","last_became_active":"2023-06-10T21:52:31.723271+0000","last_became_peered":"2023-06-10T21:52:31.723271+0000","last_unstale":"2023-06-10T21:52:41.190717+0000","last_undegraded":"2023-06-10T21:52:41.190717+0000","last_fullsized":"2023-06-10T21:52:41.190717+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187188+0000","last_change":"2023-06-10T21:52:25.522116+0000","last_active":"2023-06-10T21:52:41.187188+0000","last_peered":"2023-06-10T21:52:41.187188+0000","last_clean":"2023-06-10T21:52:41.187188+0000","last_became_active":"2023-06-10T21:52:25.521818+0000","last_became_peered":"2023-06-10T21:52:25.521818+0000","last_unstale":"2023-06-10T21:52:41.187188+0000","last_undegraded":"2023-06-10T21:52:41.187188+0000","last_fullsized":"2023-06-10T21:52:41.187188+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:20:20.987643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":382,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196086+0000","last_change":"2023-06-10T21:51:38.457228+0000","last_active":"2023-06-10T21:52:41.196086+0000","last_peered":"2023-06-10T21:52:41.196086+0000","last_clean":"2023-06-10T21:52:41.196086+0000","last_became_active":"2023-06-10T21:51:38.433522+0000","last_became_peered":"2023-06-10T21:51:38.433522+0000","last_unstale":"2023-06-10T21:52:41.196086+0000","last_undegraded":"2023-06-10T21:52:41.196086+0000","last_fullsized":"2023-06-10T21:52:41.196086+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":242,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197063+0000","last_change":"2023-06-10T21:52:35.612340+0000","last_active":"2023-06-10T21:52:41.197063+0000","last_peered":"2023-06-10T21:52:41.197063+0000","last_clean":"2023-06-10T21:52:41.197063+0000","last_became_active":"2023-06-10T21:52:35.611968+0000","last_became_peered":"2023-06-10T21:52:35.611968+0000","last_unstale":"2023-06-10T21:52:41.197063+0000","last_undegraded":"2023-06-10T21:52:41.197063+0000","last_fullsized":"2023-06-10T21:52:41.197063+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":382,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196186+0000","last_change":"2023-06-10T21:51:42.505549+0000","last_active":"2023-06-10T21:52:41.196186+0000","last_peered":"2023-06-10T21:52:41.196186+0000","last_clean":"2023-06-10T21:52:41.196186+0000","last_became_active":"2023-06-10T21:51:42.468939+0000","last_became_peered":"2023-06-10T21:51:42.468939+0000","last_unstale":"2023-06-10T21:52:41.196186+0000","last_undegraded":"2023-06-10T21:52:41.196186+0000","last_fullsized":"2023-06-10T21:52:41.196186+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":114,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415208+0000","last_change":"2023-06-10T21:52:42.415208+0000","last_active":"2023-06-10T21:52:42.415208+0000","last_peered":"2023-06-10T21:52:42.415208+0000","last_clean":"2023-06-10T21:52:42.415208+0000","last_became_active":"2023-06-10T21:52:42.414810+0000","last_became_peered":"2023-06-10T21:52:42.414810+0000","last_unstale":"2023-06-10T21:52:42.415208+0000","last_undegraded":"2023-06-10T21:52:42.415208+0000","last_fullsized":"2023-06-10T21:52:42.415208+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":656,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196617+0000","last_change":"2023-06-10T21:52:31.773663+0000","last_active":"2023-06-10T21:52:41.196617+0000","last_peered":"2023-06-10T21:52:41.196617+0000","last_clean":"2023-06-10T21:52:41.196617+0000","last_became_active":"2023-06-10T21:52:31.723869+0000","last_became_peered":"2023-06-10T21:52:31.723869+0000","last_unstale":"2023-06-10T21:52:41.196617+0000","last_undegraded":"2023-06-10T21:52:41.196617+0000","last_fullsized":"2023-06-10T21:52:41.196617+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192027+0000","last_change":"2023-06-10T21:51:42.453220+0000","last_active":"2023-06-10T21:52:41.192027+0000","last_peered":"2023-06-10T21:52:41.192027+0000","last_clean":"2023-06-10T21:52:41.192027+0000","last_became_active":"2023-06-10T21:51:42.452462+0000","last_became_peered":"2023-06-10T21:51:42.452462+0000","last_unstale":"2023-06-10T21:52:41.192027+0000","last_undegraded":"2023-06-10T21:52:41.192027+0000","last_fullsized":"2023-06-10T21:52:41.192027+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196815+0000","last_change":"2023-06-10T21:52:35.570481+0000","last_active":"2023-06-10T21:52:41.196815+0000","last_peered":"2023-06-10T21:52:41.196815+0000","last_clean":"2023-06-10T21:52:41.196815+0000","last_became_active":"2023-06-10T21:52:35.569623+0000","last_became_peered":"2023-06-10T21:52:35.569623+0000","last_unstale":"2023-06-10T21:52:41.196815+0000","last_undegraded":"2023-06-10T21:52:41.196815+0000","last_fullsized":"2023-06-10T21:52:41.196815+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":148,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196172+0000","last_change":"2023-06-10T21:51:42.453723+0000","last_active":"2023-06-10T21:52:41.196172+0000","last_peered":"2023-06-10T21:52:41.196172+0000","last_clean":"2023-06-10T21:52:41.196172+0000","last_became_active":"2023-06-10T21:51:42.453432+0000","last_became_peered":"2023-06-10T21:51:42.453432+0000","last_unstale":"2023-06-10T21:52:41.196172+0000","last_undegraded":"2023-06-10T21:52:41.196172+0000","last_fullsized":"2023-06-10T21:52:41.196172+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:09:39.344361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":189,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186835+0000","last_change":"2023-06-10T21:52:28.543205+0000","last_active":"2023-06-10T21:52:41.186835+0000","last_peered":"2023-06-10T21:52:41.186835+0000","last_clean":"2023-06-10T21:52:41.186835+0000","last_became_active":"2023-06-10T21:52:28.542871+0000","last_became_peered":"2023-06-10T21:52:28.542871+0000","last_unstale":"2023-06-10T21:52:41.186835+0000","last_undegraded":"2023-06-10T21:52:41.186835+0000","last_fullsized":"2023-06-10T21:52:41.186835+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:35.841640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415194+0000","last_change":"2023-06-10T21:52:42.415194+0000","last_active":"2023-06-10T21:52:42.415194+0000","last_peered":"2023-06-10T21:52:42.415194+0000","last_clean":"2023-06-10T21:52:42.415194+0000","last_became_active":"2023-06-10T21:52:42.414764+0000","last_became_peered":"2023-06-10T21:52:42.414764+0000","last_unstale":"2023-06-10T21:52:42.415194+0000","last_undegraded":"2023-06-10T21:52:42.415194+0000","last_fullsized":"2023-06-10T21:52:42.415194+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191755+0000","last_change":"2023-06-10T21:51:42.467839+0000","last_active":"2023-06-10T21:52:41.191755+0000","last_peered":"2023-06-10T21:52:41.191755+0000","last_clean":"2023-06-10T21:52:41.191755+0000","last_became_active":"2023-06-10T21:51:42.467496+0000","last_became_peered":"2023-06-10T21:51:42.467496+0000","last_unstale":"2023-06-10T21:52:41.191755+0000","last_undegraded":"2023-06-10T21:52:41.191755+0000","last_fullsized":"2023-06-10T21:52:41.191755+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":623,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.201638+0000","last_change":"2023-06-10T21:52:42.201638+0000","last_active":"2023-06-10T21:52:42.201638+0000","last_peered":"2023-06-10T21:52:42.201638+0000","last_clean":"2023-06-10T21:52:42.201638+0000","last_became_active":"2023-06-10T21:52:42.201268+0000","last_became_peered":"2023-06-10T21:52:42.201268+0000","last_unstale":"2023-06-10T21:52:42.201638+0000","last_undegraded":"2023-06-10T21:52:42.201638+0000","last_fullsized":"2023-06-10T21:52:42.201638+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":559,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.197449+0000","last_change":"2023-06-10T21:52:42.197449+0000","last_active":"2023-06-10T21:52:42.197449+0000","last_peered":"2023-06-10T21:52:42.197449+0000","last_clean":"2023-06-10T21:52:42.197449+0000","last_became_active":"2023-06-10T21:52:42.196993+0000","last_became_peered":"2023-06-10T21:52:42.196993+0000","last_unstale":"2023-06-10T21:52:42.197449+0000","last_undegraded":"2023-06-10T21:52:42.197449+0000","last_fullsized":"2023-06-10T21:52:42.197449+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.195128+0000","last_change":"2023-06-10T21:52:42.195128+0000","last_active":"2023-06-10T21:52:42.195128+0000","last_peered":"2023-06-10T21:52:42.195128+0000","last_clean":"2023-06-10T21:52:42.195128+0000","last_became_active":"2023-06-10T21:52:42.194750+0000","last_became_peered":"2023-06-10T21:52:42.194750+0000","last_unstale":"2023-06-10T21:52:42.195128+0000","last_undegraded":"2023-06-10T21:52:42.195128+0000","last_fullsized":"2023-06-10T21:52:42.195128+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186659+0000","last_change":"2023-06-10T21:52:25.522185+0000","last_active":"2023-06-10T21:52:41.186659+0000","last_peered":"2023-06-10T21:52:41.186659+0000","last_clean":"2023-06-10T21:52:41.186659+0000","last_became_active":"2023-06-10T21:52:25.521863+0000","last_became_peered":"2023-06-10T21:52:25.521863+0000","last_unstale":"2023-06-10T21:52:41.186659+0000","last_undegraded":"2023-06-10T21:52:41.186659+0000","last_fullsized":"2023-06-10T21:52:41.186659+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:24.403777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":35,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194822+0000","last_change":"2023-06-10T21:51:42.452748+0000","last_active":"2023-06-10T21:52:41.194822+0000","last_peered":"2023-06-10T21:52:41.194822+0000","last_clean":"2023-06-10T21:52:41.194822+0000","last_became_active":"2023-06-10T21:51:42.452329+0000","last_became_peered":"2023-06-10T21:51:42.452329+0000","last_unstale":"2023-06-10T21:52:41.194822+0000","last_undegraded":"2023-06-10T21:52:41.194822+0000","last_fullsized":"2023-06-10T21:52:41.194822+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:33:37.056969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193887+0000","last_change":"2023-06-10T21:51:54.771722+0000","last_active":"2023-06-10T21:52:41.193887+0000","last_peered":"2023-06-10T21:52:41.193887+0000","last_clean":"2023-06-10T21:52:41.193887+0000","last_became_active":"2023-06-10T21:51:54.771076+0000","last_became_peered":"2023-06-10T21:51:54.771076+0000","last_unstale":"2023-06-10T21:52:41.193887+0000","last_undegraded":"2023-06-10T21:52:41.193887+0000","last_fullsized":"2023-06-10T21:52:41.193887+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:50:38.207479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":166,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197229+0000","last_change":"2023-06-10T21:52:35.570035+0000","last_active":"2023-06-10T21:52:41.197229+0000","last_peered":"2023-06-10T21:52:41.197229+0000","last_clean":"2023-06-10T21:52:41.197229+0000","last_became_active":"2023-06-10T21:52:35.569547+0000","last_became_peered":"2023-06-10T21:52:35.569547+0000","last_unstale":"2023-06-10T21:52:41.197229+0000","last_undegraded":"2023-06-10T21:52:41.197229+0000","last_fullsized":"2023-06-10T21:52:41.197229+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":148,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194768+0000","last_change":"2023-06-10T21:52:31.723307+0000","last_active":"2023-06-10T21:52:41.194768+0000","last_peered":"2023-06-10T21:52:41.194768+0000","last_clean":"2023-06-10T21:52:41.194768+0000","last_became_active":"2023-06-10T21:52:31.722848+0000","last_became_peered":"2023-06-10T21:52:31.722848+0000","last_unstale":"2023-06-10T21:52:41.194768+0000","last_undegraded":"2023-06-10T21:52:41.194768+0000","last_fullsized":"2023-06-10T21:52:41.194768+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:21:32.882245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":104,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196542+0000","last_change":"2023-06-10T21:52:35.570218+0000","last_active":"2023-06-10T21:52:41.196542+0000","last_peered":"2023-06-10T21:52:41.196542+0000","last_clean":"2023-06-10T21:52:41.196542+0000","last_became_active":"2023-06-10T21:52:35.569508+0000","last_became_peered":"2023-06-10T21:52:35.569508+0000","last_unstale":"2023-06-10T21:52:41.196542+0000","last_undegraded":"2023-06-10T21:52:41.196542+0000","last_fullsized":"2023-06-10T21:52:41.196542+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415107+0000","last_change":"2023-06-10T21:52:42.415107+0000","last_active":"2023-06-10T21:52:42.415107+0000","last_peered":"2023-06-10T21:52:42.415107+0000","last_clean":"2023-06-10T21:52:42.415107+0000","last_became_active":"2023-06-10T21:52:42.414682+0000","last_became_peered":"2023-06-10T21:52:42.414682+0000","last_unstale":"2023-06-10T21:52:42.415107+0000","last_undegraded":"2023-06-10T21:52:42.415107+0000","last_fullsized":"2023-06-10T21:52:42.415107+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":423,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193971+0000","last_change":"2023-06-10T21:51:54.605586+0000","last_active":"2023-06-10T21:52:41.193971+0000","last_peered":"2023-06-10T21:52:41.193971+0000","last_clean":"2023-06-10T21:52:41.193971+0000","last_became_active":"2023-06-10T21:51:54.605214+0000","last_became_peered":"2023-06-10T21:51:54.605214+0000","last_unstale":"2023-06-10T21:52:41.193971+0000","last_undegraded":"2023-06-10T21:52:41.193971+0000","last_fullsized":"2023-06-10T21:52:41.193971+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:08:37.070016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":435,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.661137+0000","last_change":"2023-06-10T21:47:18.337074+0000","last_active":"2023-06-10T21:52:41.661137+0000","last_peered":"2023-06-10T21:52:41.661137+0000","last_clean":"2023-06-10T21:52:41.661137+0000","last_became_active":"2023-06-10T21:47:18.336698+0000","last_became_peered":"2023-06-10T21:47:18.336698+0000","last_unstale":"2023-06-10T21:52:41.661137+0000","last_undegraded":"2023-06-10T21:52:41.661137+0000","last_fullsized":"2023-06-10T21:52:41.661137+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:58.371008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":457,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192698+0000","last_change":"2023-06-10T21:51:54.771505+0000","last_active":"2023-06-10T21:52:41.192698+0000","last_peered":"2023-06-10T21:52:41.192698+0000","last_clean":"2023-06-10T21:52:41.192698+0000","last_became_active":"2023-06-10T21:51:54.771170+0000","last_became_peered":"2023-06-10T21:51:54.771170+0000","last_unstale":"2023-06-10T21:52:41.192698+0000","last_undegraded":"2023-06-10T21:52:41.192698+0000","last_fullsized":"2023-06-10T21:52:41.192698+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:31:32.827946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194724+0000","last_change":"2023-06-10T21:51:54.464705+0000","last_active":"2023-06-10T21:52:41.194724+0000","last_peered":"2023-06-10T21:52:41.194724+0000","last_clean":"2023-06-10T21:52:41.194724+0000","last_became_active":"2023-06-10T21:51:54.464393+0000","last_became_peered":"2023-06-10T21:51:54.464393+0000","last_unstale":"2023-06-10T21:52:41.194724+0000","last_undegraded":"2023-06-10T21:52:41.194724+0000","last_fullsized":"2023-06-10T21:52:41.194724+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:14:33.254460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":490,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197134+0000","last_change":"2023-06-10T21:52:35.612393+0000","last_active":"2023-06-10T21:52:41.197134+0000","last_peered":"2023-06-10T21:52:41.197134+0000","last_clean":"2023-06-10T21:52:41.197134+0000","last_became_active":"2023-06-10T21:52:35.611891+0000","last_became_peered":"2023-06-10T21:52:35.611891+0000","last_unstale":"2023-06-10T21:52:41.197134+0000","last_undegraded":"2023-06-10T21:52:41.197134+0000","last_fullsized":"2023-06-10T21:52:41.197134+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":784,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195607+0000","last_change":"2023-06-10T21:48:56.439594+0000","last_active":"2023-06-10T21:52:41.195607+0000","last_peered":"2023-06-10T21:52:41.195607+0000","last_clean":"2023-06-10T21:52:41.195607+0000","last_became_active":"2023-06-10T21:48:56.439122+0000","last_became_peered":"2023-06-10T21:48:56.439122+0000","last_unstale":"2023-06-10T21:52:41.195607+0000","last_undegraded":"2023-06-10T21:52:41.195607+0000","last_fullsized":"2023-06-10T21:52:41.195607+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":206,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.412811+0000","last_change":"2023-06-10T21:52:28.611792+0000","last_active":"2023-06-10T21:52:42.412811+0000","last_peered":"2023-06-10T21:52:42.412811+0000","last_clean":"2023-06-10T21:52:42.412811+0000","last_became_active":"2023-06-10T21:52:28.611535+0000","last_became_peered":"2023-06-10T21:52:28.611535+0000","last_unstale":"2023-06-10T21:52:42.412811+0000","last_undegraded":"2023-06-10T21:52:42.412811+0000","last_fullsized":"2023-06-10T21:52:42.412811+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:50:35.735362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192567+0000","last_change":"2023-06-10T21:51:38.449633+0000","last_active":"2023-06-10T21:52:41.192567+0000","last_peered":"2023-06-10T21:52:41.192567+0000","last_clean":"2023-06-10T21:52:41.192567+0000","last_became_active":"2023-06-10T21:51:38.449039+0000","last_became_peered":"2023-06-10T21:51:38.449039+0000","last_unstale":"2023-06-10T21:52:41.192567+0000","last_undegraded":"2023-06-10T21:52:41.192567+0000","last_fullsized":"2023-06-10T21:52:41.192567+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":608,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195715+0000","last_change":"2023-06-10T21:51:42.467529+0000","last_active":"2023-06-10T21:52:41.195715+0000","last_peered":"2023-06-10T21:52:41.195715+0000","last_clean":"2023-06-10T21:52:41.195715+0000","last_became_active":"2023-06-10T21:51:42.467213+0000","last_became_peered":"2023-06-10T21:51:42.467213+0000","last_unstale":"2023-06-10T21:52:41.195715+0000","last_undegraded":"2023-06-10T21:52:41.195715+0000","last_fullsized":"2023-06-10T21:52:41.195715+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":157,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194903+0000","last_change":"2023-06-10T21:52:25.516479+0000","last_active":"2023-06-10T21:52:41.194903+0000","last_peered":"2023-06-10T21:52:41.194903+0000","last_clean":"2023-06-10T21:52:41.194903+0000","last_became_active":"2023-06-10T21:52:25.516147+0000","last_became_peered":"2023-06-10T21:52:25.516147+0000","last_unstale":"2023-06-10T21:52:41.194903+0000","last_undegraded":"2023-06-10T21:52:41.194903+0000","last_fullsized":"2023-06-10T21:52:41.194903+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:25.809519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195436+0000","last_change":"2023-06-10T21:51:42.455442+0000","last_active":"2023-06-10T21:52:41.195436+0000","last_peered":"2023-06-10T21:52:41.195436+0000","last_clean":"2023-06-10T21:52:41.195436+0000","last_became_active":"2023-06-10T21:51:42.454424+0000","last_became_peered":"2023-06-10T21:51:42.454424+0000","last_unstale":"2023-06-10T21:52:41.195436+0000","last_undegraded":"2023-06-10T21:52:41.195436+0000","last_fullsized":"2023-06-10T21:52:41.195436+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:16:08.780005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":605,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196663+0000","last_change":"2023-06-10T21:51:38.431740+0000","last_active":"2023-06-10T21:52:41.196663+0000","last_peered":"2023-06-10T21:52:41.196663+0000","last_clean":"2023-06-10T21:52:41.196663+0000","last_became_active":"2023-06-10T21:51:38.431383+0000","last_became_peered":"2023-06-10T21:51:38.431383+0000","last_unstale":"2023-06-10T21:52:41.196663+0000","last_undegraded":"2023-06-10T21:52:41.196663+0000","last_fullsized":"2023-06-10T21:52:41.196663+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":456,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194193+0000","last_change":"2023-06-10T21:51:54.457776+0000","last_active":"2023-06-10T21:52:41.194193+0000","last_peered":"2023-06-10T21:52:41.194193+0000","last_clean":"2023-06-10T21:52:41.194193+0000","last_became_active":"2023-06-10T21:51:54.457479+0000","last_became_peered":"2023-06-10T21:51:54.457479+0000","last_unstale":"2023-06-10T21:52:41.194193+0000","last_undegraded":"2023-06-10T21:52:41.194193+0000","last_fullsized":"2023-06-10T21:52:41.194193+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:58.895011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":130,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196339+0000","last_change":"2023-06-10T21:51:34.508129+0000","last_active":"2023-06-10T21:52:41.196339+0000","last_peered":"2023-06-10T21:52:41.196339+0000","last_clean":"2023-06-10T21:52:41.196339+0000","last_became_active":"2023-06-10T21:51:34.507299+0000","last_became_peered":"2023-06-10T21:51:34.507299+0000","last_unstale":"2023-06-10T21:52:41.196339+0000","last_undegraded":"2023-06-10T21:52:41.196339+0000","last_fullsized":"2023-06-10T21:52:41.196339+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:46:03.722972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":542,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.413307+0000","last_change":"2023-06-10T21:52:28.541913+0000","last_active":"2023-06-10T21:52:42.413307+0000","last_peered":"2023-06-10T21:52:42.413307+0000","last_clean":"2023-06-10T21:52:42.413307+0000","last_became_active":"2023-06-10T21:52:28.541543+0000","last_became_peered":"2023-06-10T21:52:28.541543+0000","last_unstale":"2023-06-10T21:52:42.413307+0000","last_undegraded":"2023-06-10T21:52:42.413307+0000","last_fullsized":"2023-06-10T21:52:42.413307+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:00.055916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194543+0000","last_change":"2023-06-10T21:51:54.771711+0000","last_active":"2023-06-10T21:52:41.194543+0000","last_peered":"2023-06-10T21:52:41.194543+0000","last_clean":"2023-06-10T21:52:41.194543+0000","last_became_active":"2023-06-10T21:51:54.771268+0000","last_became_peered":"2023-06-10T21:51:54.771268+0000","last_unstale":"2023-06-10T21:52:41.194543+0000","last_undegraded":"2023-06-10T21:52:41.194543+0000","last_fullsized":"2023-06-10T21:52:41.194543+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:53:11.763424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":435,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186650+0000","last_change":"2023-06-10T21:52:28.542998+0000","last_active":"2023-06-10T21:52:41.186650+0000","last_peered":"2023-06-10T21:52:41.186650+0000","last_clean":"2023-06-10T21:52:41.186650+0000","last_became_active":"2023-06-10T21:52:28.542662+0000","last_became_peered":"2023-06-10T21:52:28.542662+0000","last_unstale":"2023-06-10T21:52:41.186650+0000","last_undegraded":"2023-06-10T21:52:41.186650+0000","last_fullsized":"2023-06-10T21:52:41.186650+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:31:25.639808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191514+0000","last_change":"2023-06-10T21:52:11.470476+0000","last_active":"2023-06-10T21:52:41.191514+0000","last_peered":"2023-06-10T21:52:41.191514+0000","last_clean":"2023-06-10T21:52:41.191514+0000","last_became_active":"2023-06-10T21:52:11.470042+0000","last_became_peered":"2023-06-10T21:52:11.470042+0000","last_unstale":"2023-06-10T21:52:41.191514+0000","last_undegraded":"2023-06-10T21:52:41.191514+0000","last_fullsized":"2023-06-10T21:52:41.191514+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":169,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195232+0000","last_change":"2023-06-10T21:51:42.502505+0000","last_active":"2023-06-10T21:52:41.195232+0000","last_peered":"2023-06-10T21:52:41.195232+0000","last_clean":"2023-06-10T21:52:41.195232+0000","last_became_active":"2023-06-10T21:51:42.454911+0000","last_became_peered":"2023-06-10T21:51:42.454911+0000","last_unstale":"2023-06-10T21:52:41.195232+0000","last_undegraded":"2023-06-10T21:52:41.195232+0000","last_fullsized":"2023-06-10T21:52:41.195232+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:54.520696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.201688+0000","last_change":"2023-06-10T21:52:42.201688+0000","last_active":"2023-06-10T21:52:42.201688+0000","last_peered":"2023-06-10T21:52:42.201688+0000","last_clean":"2023-06-10T21:52:42.201688+0000","last_became_active":"2023-06-10T21:52:42.201294+0000","last_became_peered":"2023-06-10T21:52:42.201294+0000","last_unstale":"2023-06-10T21:52:42.201688+0000","last_undegraded":"2023-06-10T21:52:42.201688+0000","last_fullsized":"2023-06-10T21:52:42.201688+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196294+0000","last_change":"2023-06-10T21:51:38.435920+0000","last_active":"2023-06-10T21:52:41.196294+0000","last_peered":"2023-06-10T21:52:41.196294+0000","last_clean":"2023-06-10T21:52:41.196294+0000","last_became_active":"2023-06-10T21:51:38.435245+0000","last_became_peered":"2023-06-10T21:51:38.435245+0000","last_unstale":"2023-06-10T21:52:41.196294+0000","last_undegraded":"2023-06-10T21:52:41.196294+0000","last_fullsized":"2023-06-10T21:52:41.196294+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:44:11.651949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192457+0000","last_change":"2023-06-10T21:51:58.583757+0000","last_active":"2023-06-10T21:52:41.192457+0000","last_peered":"2023-06-10T21:52:41.192457+0000","last_clean":"2023-06-10T21:52:41.192457+0000","last_became_active":"2023-06-10T21:51:58.583392+0000","last_became_peered":"2023-06-10T21:51:58.583392+0000","last_unstale":"2023-06-10T21:52:41.192457+0000","last_undegraded":"2023-06-10T21:52:41.192457+0000","last_fullsized":"2023-06-10T21:52:41.192457+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:08:33.269257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":112,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196910+0000","last_change":"2023-06-10T21:52:35.570688+0000","last_active":"2023-06-10T21:52:41.196910+0000","last_peered":"2023-06-10T21:52:41.196910+0000","last_clean":"2023-06-10T21:52:41.196910+0000","last_became_active":"2023-06-10T21:52:35.570114+0000","last_became_peered":"2023-06-10T21:52:35.570114+0000","last_unstale":"2023-06-10T21:52:41.196910+0000","last_undegraded":"2023-06-10T21:52:41.196910+0000","last_fullsized":"2023-06-10T21:52:41.196910+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":431,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415636+0000","last_change":"2023-06-10T21:52:42.415636+0000","last_active":"2023-06-10T21:52:42.415636+0000","last_peered":"2023-06-10T21:52:42.415636+0000","last_clean":"2023-06-10T21:52:42.415636+0000","last_became_active":"2023-06-10T21:52:42.415295+0000","last_became_peered":"2023-06-10T21:52:42.415295+0000","last_unstale":"2023-06-10T21:52:42.415636+0000","last_undegraded":"2023-06-10T21:52:42.415636+0000","last_fullsized":"2023-06-10T21:52:42.415636+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197348+0000","last_change":"2023-06-10T21:52:35.569232+0000","last_active":"2023-06-10T21:52:41.197348+0000","last_peered":"2023-06-10T21:52:41.197348+0000","last_clean":"2023-06-10T21:52:41.197348+0000","last_became_active":"2023-06-10T21:52:35.568793+0000","last_became_peered":"2023-06-10T21:52:35.568793+0000","last_unstale":"2023-06-10T21:52:41.197348+0000","last_undegraded":"2023-06-10T21:52:41.197348+0000","last_fullsized":"2023-06-10T21:52:41.197348+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":497,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415252+0000","last_change":"2023-06-10T21:52:42.415252+0000","last_active":"2023-06-10T21:52:42.415252+0000","last_peered":"2023-06-10T21:52:42.415252+0000","last_clean":"2023-06-10T21:52:42.415252+0000","last_became_active":"2023-06-10T21:52:42.414816+0000","last_became_peered":"2023-06-10T21:52:42.414816+0000","last_unstale":"2023-06-10T21:52:42.415252+0000","last_undegraded":"2023-06-10T21:52:42.415252+0000","last_fullsized":"2023-06-10T21:52:42.415252+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":182,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187633+0000","last_change":"2023-06-10T21:52:28.542490+0000","last_active":"2023-06-10T21:52:41.187633+0000","last_peered":"2023-06-10T21:52:41.187633+0000","last_clean":"2023-06-10T21:52:41.187633+0000","last_became_active":"2023-06-10T21:52:28.542147+0000","last_became_peered":"2023-06-10T21:52:28.542147+0000","last_unstale":"2023-06-10T21:52:41.187633+0000","last_undegraded":"2023-06-10T21:52:41.187633+0000","last_fullsized":"2023-06-10T21:52:41.187633+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:57:02.358984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":349,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.703910+0000","last_change":"2023-06-10T21:52:28.543025+0000","last_active":"2023-06-10T21:52:41.703910+0000","last_peered":"2023-06-10T21:52:41.703910+0000","last_clean":"2023-06-10T21:52:41.703910+0000","last_became_active":"2023-06-10T21:52:28.542727+0000","last_became_peered":"2023-06-10T21:52:28.542727+0000","last_unstale":"2023-06-10T21:52:41.703910+0000","last_undegraded":"2023-06-10T21:52:41.703910+0000","last_fullsized":"2023-06-10T21:52:41.703910+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:24:59.071503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":193,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195655+0000","last_change":"2023-06-10T21:51:58.477208+0000","last_active":"2023-06-10T21:52:41.195655+0000","last_peered":"2023-06-10T21:52:41.195655+0000","last_clean":"2023-06-10T21:52:41.195655+0000","last_became_active":"2023-06-10T21:51:58.473617+0000","last_became_peered":"2023-06-10T21:51:58.473617+0000","last_unstale":"2023-06-10T21:52:41.195655+0000","last_undegraded":"2023-06-10T21:52:41.195655+0000","last_fullsized":"2023-06-10T21:52:41.195655+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:57:58.890185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.193806+0000","last_change":"2023-06-10T21:52:42.193806+0000","last_active":"2023-06-10T21:52:42.193806+0000","last_peered":"2023-06-10T21:52:42.193806+0000","last_clean":"2023-06-10T21:52:42.193806+0000","last_became_active":"2023-06-10T21:52:42.193424+0000","last_became_peered":"2023-06-10T21:52:42.193424+0000","last_unstale":"2023-06-10T21:52:42.193806+0000","last_undegraded":"2023-06-10T21:52:42.193806+0000","last_fullsized":"2023-06-10T21:52:42.193806+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":29606,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56940,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430875,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117004,"kb_used_data":5316,"kb_used_omap":10,"kb_used_meta":111669,"kb_avail":93628148,"statfs":{"total":95995035648,"available":95875223552,"internally_reserved":0,"allocated":5443584,"data_stored":4612952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":114349117},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":4.5259999999999998,"15min":2.4980000000000002},"min":{"1min":0.34100000000000003,"5min":0.24399999999999999,"15min":0.23799999999999999},"max":{"1min":0.75,"5min":170.05199999999999,"15min":170.05199999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":5.3440000000000003,"15min":3.044},"min":{"1min":0.39700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.78200000000000003,"5min":170.126,"15min":170.126},"last":0.55700000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.2669999999999999,"15min":1.5920000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.70399999999999996,"5min":13.180999999999999,"15min":38.078000000000003},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.2170000000000001,"15min":1.8360000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28599999999999998},"max":{"1min":0.82699999999999996,"5min":7.3280000000000003,"15min":38.341000000000001},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Jun 10 21:52:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.9409999999999998,"15min":5.5140000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68100000000000005,"5min":80.334000000000003,"15min":218.90600000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":2.9740000000000002,"15min":5.4560000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.76200000000000001,"5min":80.262,"15min":218.851},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Jun 10 21:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.792,"15min":4.6379999999999999},"min":{"1min":0.35499999999999998,"5min":0.34899999999999998,"15min":0.26100000000000001},"max":{"1min":0.80000000000000004,"5min":28.109999999999999,"15min":218.35300000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.60199999999999998,"5min":2.1739999999999999,"15min":4.694},"min":{"1min":0.36499999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.85399999999999998,"5min":30.981000000000002,"15min":218.74600000000001},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.72899999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.79200000000000004,"15min":6.266},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.85599999999999998,"5min":5.5700000000000003,"15min":131.619},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.298,"15min":6.3639999999999999},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.24299999999999999},"max":{"1min":0.92500000000000004,"5min":18.277999999999999,"15min":134.928},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.0609999999999999,"15min":0.94499999999999995},"min":{"1min":0.40999999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.86699999999999999,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.97299999999999998,"15min":1.028},"min":{"1min":0.40000000000000002,"5min":0.34100000000000003,"15min":0.30599999999999999},"max":{"1min":0.97099999999999997,"5min":7.7960000000000003,"15min":8.1020000000000003},"last":0.71399999999999997}]}]},{"osd":6,"up_from":42,"seq":180388627103,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513144,"kb_used_data":5296,"kb_used_omap":0,"kb_used_meta":507840,"kb_avail":93232008,"statfs":{"total":95995035648,"available":95469576192,"internally_reserved":0,"allocated":5423104,"data_stored":4611244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":520028160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.141,"15min":1.397},"min":{"1min":0.307,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63100000000000001,"5min":7.6840000000000002,"15min":107.86},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":2.6259999999999999,"15min":2.0899999999999999},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73399999999999999,"5min":136.85900000000001,"15min":136.85900000000001},"last":0.56200000000000006}]},{"osd":1,"last update":"Sat Jun 10 21:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.90600000000000003,"15min":1.492},"min":{"1min":0.40100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.70999999999999996,"5min":6.3559999999999999,"15min":80.210999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.198,"15min":1.754},"min":{"1min":0.36399999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":0.66700000000000004,"5min":8.0079999999999991,"15min":79.894000000000005},"last":0.67400000000000004}]},{"osd":2,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":6.5369999999999999,"15min":6.6550000000000002},"min":{"1min":0.40799999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.92400000000000004,"5min":265.37099999999998,"15min":265.37099999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.61899999999999999,"5min":6.6399999999999997,"15min":6.7519999999999998},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.93400000000000005,"5min":265.32900000000001,"15min":265.32900000000001},"last":0.78000000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":6.7140000000000004,"15min":6.5739999999999998},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.75800000000000001,"5min":412.55200000000002,"15min":412.55200000000002},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":6.9379999999999997,"15min":6.4950000000000001},"min":{"1min":0.42299999999999999,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":0.77500000000000002,"5min":412.41199999999998,"15min":412.41199999999998},"last":0.86899999999999999}]},{"osd":4,"last update":"Sat Jun 10 21:52:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.82699999999999996}]},{"osd":5,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.93000000000000005,"15min":1.077},"min":{"1min":0.44700000000000001,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":0.90300000000000002,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.95199999999999996,"15min":1.375},"min":{"1min":0.51300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.83699999999999997,"5min":7.3209999999999997,"15min":7.3209999999999997},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":2.093,"15min":2.9079999999999999},"min":{"1min":0.44600000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81899999999999995,"5min":97.108000000000004,"15min":189.042},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.64000000000000001,"5min":2.101,"15min":3.1720000000000002},"min":{"1min":0.46500000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":97.299000000000007,"15min":190.83699999999999},"last":0.88100000000000001}]}]},{"osd":1,"up_from":105,"seq":450971566546,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923892,"kb_used_data":5804,"kb_used_omap":4,"kb_used_meta":918075,"kb_avail":92821260,"statfs":{"total":95995035648,"available":95048970240,"internally_reserved":0,"allocated":5943296,"data_stored":4146871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940109710},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":4.6840000000000002,"15min":3.25},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.187},"max":{"1min":0.97199999999999998,"5min":139.79300000000001,"15min":139.79300000000001},"last":0.311},{"interface":"front","average":{"1min":0.373,"5min":4.641,"15min":2.9900000000000002},"min":{"1min":0.28000000000000003,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":0.52800000000000002,"5min":139.75299999999999,"15min":139.75299999999999},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:53:14 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":6.6799999999999997,"15min":7.1719999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":0.82899999999999996,"5min":162.191,"15min":162.191},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.35499999999999998,"5min":6.4279999999999999,"15min":7.4050000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.51500000000000001,"5min":162.70500000000001,"15min":162.70500000000001},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":3.819,"15min":3.8079999999999998},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.23000000000000001},"max":{"1min":0.58299999999999996,"5min":160.82900000000001,"15min":160.82900000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":4.6440000000000001,"15min":4.6980000000000004},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":0.59699999999999998,"5min":160.93100000000001,"15min":160.93100000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:53:14 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.1140000000000001,"15min":1.9810000000000001},"min":{"1min":0.495,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.94299999999999995,"5min":8.1549999999999994,"15min":22.094000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.1879999999999999,"15min":1.806},"min":{"1min":0.52800000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.98699999999999999,"5min":8.3049999999999997,"15min":19.029},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.3879999999999999,"15min":1.597},"min":{"1min":0.54200000000000004,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.008,"5min":10.611000000000001,"15min":19.762},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.2190000000000001,"15min":1.534},"min":{"1min":0.50700000000000001,"5min":0.41699999999999998,"15min":0.40500000000000003},"max":{"1min":0.90900000000000003,"5min":8.2379999999999995,"15min":18.376999999999999},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.3600000000000001,"15min":2.1139999999999999},"min":{"1min":0.47199999999999998,"5min":0.42999999999999999,"15min":0.36399999999999999},"max":{"1min":0.96899999999999997,"5min":10.891999999999999,"15min":124.527},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.1879999999999999,"15min":2.0960000000000001},"min":{"1min":0.51700000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88,"5min":7.8449999999999998,"15min":124.509},"last":0.627}]}]},{"osd":0,"up_from":244,"seq":1047972020522,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700000,"kb_used_data":6360,"kb_used_omap":10,"kb_used_meta":693621,"kb_avail":93045152,"statfs":{"total":95995035648,"available":95278235648,"internally_reserved":0,"allocated":6512640,"data_stored":4726019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":710267956},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.353,"15min":1.7350000000000001},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.19800000000000001},"max":{"1min":0.51400000000000001,"5min":9.1500000000000004,"15min":27.759},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":1.8400000000000001,"15min":2.3439999999999999},"min":{"1min":0.27000000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.53900000000000003,"5min":14.087,"15min":31.419},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":6.2400000000000002,"15min":6.8819999999999997},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.56699999999999995,"5min":156.31100000000001,"15min":156.31100000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":5.8310000000000004,"15min":6.3410000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":0.73999999999999999,"5min":155.92099999999999,"15min":155.92099999999999},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":4.6879999999999997,"15min":4.4870000000000001},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.503,"5min":150.89699999999999,"15min":150.89699999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.41599999999999998,"5min":4.7709999999999999,"15min":4.2939999999999996},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.25700000000000001},"max":{"1min":0.73199999999999998,"5min":150.97200000000001,"15min":150.97200000000001},"last":0.37}]},{"osd":4,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.425,"15min":3.6219999999999999},"min":{"1min":0.434,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.92600000000000005,"5min":9.1899999999999995,"15min":92.149000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.357,"15min":4.0949999999999998},"min":{"1min":0.497,"5min":0.311,"15min":0.311},"max":{"1min":0.999,"5min":6.9969999999999999,"15min":93.292000000000002},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":3.9430000000000001,"15min":3.0529999999999999},"min":{"1min":0.51200000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":1.0369999999999999,"5min":224.54599999999999,"15min":224.54599999999999},"last":0.625},{"interface":"front","average":{"1min":0.64900000000000002,"5min":3.839,"15min":2.774},"min":{"1min":0.46800000000000003,"5min":0.377,"15min":0.377},"max":{"1min":0.93999999999999995,"5min":224.56100000000001,"15min":224.56100000000001},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":7.8869999999999996,"15min":5.0049999999999999},"min":{"1min":0.46000000000000002,"5min":0.371,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":227.565,"15min":227.565},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":8.1310000000000002,"15min":5.1989999999999998},"min":{"1min":0.50700000000000001,"5min":0.377,"15min":0.36299999999999999},"max":{"1min":0.94199999999999995,"5min":233.90100000000001,"15min":233.90100000000001},"last":0.65400000000000003}]}]},{"osd":2,"up_from":120,"seq":515396075934,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275896,"kb_used_data":5744,"kb_used_omap":7,"kb_used_meta":1270136,"kb_avail":92469256,"statfs":{"total":95995035648,"available":94688518144,"internally_reserved":0,"allocated":5881856,"data_stored":4135992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1300619345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":2.2309999999999999,"15min":3.4900000000000002},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.19700000000000001},"max":{"1min":0.53800000000000003,"5min":78.003,"15min":171.33699999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.9350000000000001,"15min":3.0299999999999998},"min":{"1min":0.29199999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.63800000000000001,"5min":77.950000000000003,"15min":171.56200000000001},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":2.169,"15min":4.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.217},"max":{"1min":0.44900000000000001,"5min":29.315999999999999,"15min":171.286},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.372,"5min":1.478,"15min":2.577},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.52000000000000002,"5min":15.685,"15min":85.501999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":7.2960000000000003,"15min":7.2210000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.22700000000000001},"max":{"1min":0.55500000000000005,"5min":187.751,"15min":187.751},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":6.9489999999999998,"15min":6.827},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56899999999999995,"5min":187.70099999999999,"15min":187.70099999999999},"last":0.432}]},{"osd":4,"last update":"Sat Jun 10 21:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:53:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":4.1200000000000001,"15min":5.6600000000000001},"min":{"1min":0.47299999999999998,"5min":0.376,"15min":0.311},"max":{"1min":0.76700000000000002,"5min":130.864,"15min":157.756},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.6949999999999998,"15min":5.1879999999999997},"min":{"1min":0.47899999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":0.83899999999999997,"5min":131.06299999999999,"15min":157.59899999999999},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.339,"15min":2.1960000000000002},"min":{"1min":0.47699999999999998,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":0.81699999999999995,"5min":19.783999999999999,"15min":171.441},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.68100000000000005,"5min":2.2370000000000001,"15min":3.0499999999999998},"min":{"1min":0.46300000000000002,"5min":0.39100000000000001,"15min":0.34399999999999997},"max":{"1min":0.93700000000000006,"5min":20.393999999999998,"15min":171.24600000000001},"last":0.623}]},{"osd":7,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":3.1560000000000001,"15min":3.589},"min":{"1min":0.442,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":0.90300000000000002,"5min":135.86099999999999,"15min":171.26499999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.67900000000000005,"5min":2.7069999999999999,"15min":3.2269999999999999},"min":{"1min":0.50800000000000001,"5min":0.40999999999999998,"15min":0.32200000000000001},"max":{"1min":0.89500000000000002,"5min":133.45400000000001,"15min":171.52000000000001},"last":0.59499999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824294,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1043964,"kb_used_data":5940,"kb_used_omap":7,"kb_used_meta":1038008,"kb_avail":92701188,"statfs":{"total":95995035648,"available":94926016512,"internally_reserved":0,"allocated":6082560,"data_stored":4156553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":1062920211},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":4.3470000000000004,"15min":2.7120000000000002},"min":{"1min":0.223,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":11.474,"5min":236.97300000000001,"15min":236.97300000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.371,"5min":4.0819999999999999,"15min":2.9710000000000001},"min":{"1min":0.26000000000000001,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":20.341999999999999,"5min":237.375,"15min":237.375},"last":0.249}]},{"osd":1,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.923,"15min":2.0179999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.7050000000000001,"5min":18.116,"15min":21.704999999999998},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":2.3530000000000002,"15min":2.2719999999999998},"min":{"1min":0.313,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.445,"5min":50.264000000000003,"15min":50.264000000000003},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":3.9489999999999998,"15min":4.2400000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.23300000000000001},"max":{"1min":10.065,"5min":117.279,"15min":231.07900000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":4.4420000000000002,"15min":4.7649999999999997},"min":{"1min":0.29799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63400000000000001,"5min":117.398,"15min":231.58600000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:52:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.1200000000000001,"15min":1.4239999999999999},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81499999999999995,"5min":50.399000000000001,"15min":50.399000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.3250000000000002,"15min":2.4329999999999998},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":50.335000000000001,"15min":50.335000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":2.8149999999999999,"15min":2.2509999999999999},"min":{"1min":0.39900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.673,"5min":62.098999999999997,"15min":62.098999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.0649999999999999,"15min":2.028},"min":{"1min":0.48899999999999999,"5min":0.38900000000000001,"15min":0.34699999999999998},"max":{"1min":11.414999999999999,"5min":62.139000000000003,"15min":62.139000000000003},"last":0.70799999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":3.032,"15min":2.6749999999999998},"min":{"1min":0.49199999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.6379999999999999,"5min":69.683000000000007,"15min":69.683000000000007},"last":0.622},{"interface":"front","average":{"1min":1.208,"5min":3.0470000000000002,"15min":2.359},"min":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.316},"max":{"1min":11.384,"5min":69.998999999999995,"15min":69.998999999999995},"last":0.60799999999999998}]}]},{"osd":4,"up_from":507,"seq":2177548419097,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58108,"kb_used_data":4724,"kb_used_omap":3,"kb_used_meta":53372,"kb_avail":93687044,"statfs":{"total":95995035648,"available":95935533056,"internally_reserved":0,"allocated":4837376,"data_stored":4020952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3428,"internal_metadata":54653596},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.63200000000000001}]}]},{"osd":5,"up_from":390,"seq":1675037245565,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532080,"kb_used_data":4712,"kb_used_omap":11,"kb_used_meta":527348,"kb_avail":93213072,"statfs":{"total":95995035648,"available":95450185728,"internally_reserved":0,"allocated":4825088,"data_stored":4017961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":540004854},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":3.7610000000000001,"15min":2.4849999999999999},"min":{"1min":0.39600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.64400000000000002,"5min":140.976,"15min":140.976},"last":0.746},{"interface":"front","average":{"1min":0.51900000000000002,"5min":3.8610000000000002,"15min":2.1389999999999998},"min":{"1min":0.379,"5min":0.251,"15min":0.251},"max":{"1min":0.67700000000000005,"5min":141.06899999999999,"15min":141.06899999999999},"last":0.68300000000000005}]},{"osd":1,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.29,"15min":2},"min":{"1min":0.42099999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66700000000000004,"5min":12.013999999999999,"15min":12.013999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.0840000000000001,"15min":1.72},"min":{"1min":0.46700000000000003,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79200000000000004,"5min":6.4109999999999996,"15min":9.7989999999999995},"last":0.73099999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":4.4329999999999998,"15min":4.1719999999999997},"min":{"1min":0.34599999999999997,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":0.72199999999999998,"5min":89.581999999999994,"15min":219.84700000000001},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":3.9780000000000002,"15min":4.1630000000000003},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.61599999999999999,"5min":89.239999999999995,"15min":216.589},"last":0.71899999999999997}]},{"osd":3,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":5.5960000000000001,"15min":6.3810000000000002},"min":{"1min":0.379,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.78100000000000003,"5min":342.28399999999999,"15min":342.28399999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.59999999999999998,"5min":5.5,"15min":6.2380000000000004},"min":{"1min":0.432,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":0.76300000000000001,"5min":342.46600000000001,"15min":342.46600000000001},"last":0.626}]},{"osd":4,"last update":"Sat Jun 10 21:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":2.1190000000000002,"15min":1.8009999999999999},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.85599999999999998,"5min":108.693,"15min":108.693},"last":0.496},{"interface":"front","average":{"1min":0.54400000000000004,"5min":1.929,"15min":1.4990000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":0.69199999999999995,"5min":108.708,"15min":108.708},"last":0.69299999999999995}]},{"osd":7,"last update":"Sat Jun 10 21:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":2.0539999999999998,"15min":1.7370000000000001},"min":{"1min":0.33600000000000002,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":0.67000000000000004,"5min":111.06999999999999,"15min":111.06999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":2.27,"15min":2.5750000000000002},"min":{"1min":0.34599999999999997,"5min":0.31,"15min":0.31},"max":{"1min":0.85499999999999998,"5min":111.402,"15min":111.402},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:53:19.577 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid b618768e-07d0-11ee-9b1e-001a4aab830c -- ceph pg dump --format=json 2023-06-10T21:53:19.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:19 smithi152 ceph-mon[156819]: pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:19.950 INFO:teuthology.orchestra.run.smithi064.stderr:Inferring config /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/mon.a/config 2023-06-10T21:53:20.470 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:20 smithi064 ceph-mon[180297]: from='client.47777 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:20.470 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:20 smithi064 ceph-mon[186706]: from='client.47777 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:20.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:20 smithi152 ceph-mon[156819]: from='client.47777 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:21.656 INFO:teuthology.orchestra.run.smithi064.stdout: 2023-06-10T21:53:21.657 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:21 smithi064 ceph-mon[186706]: pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:21.657 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:21 smithi064 ceph-mon[180297]: pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:21.658 INFO:teuthology.orchestra.run.smithi064.stderr:dumped all 2023-06-10T21:53:21.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:21 smithi152 ceph-mon[156819]: pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:22.424 INFO:teuthology.orchestra.run.smithi064.stdout:{"pg_ready":true,"pg_map":{"version":1897,"stamp":"2023-06-10T21:53:20.386141+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":3571814,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5164088,"kb_used_data":43896,"kb_used_omap":56,"kb_used_meta":5120071,"kb_avail":744797128,"statfs":{"total":767960285184,"available":762672259072,"internally_reserved":0,"allocated":44949504,"data_stored":34428544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58123,"internal_metadata":5242952949},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003093"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":574,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.194131+0000","last_change":"2023-06-10T21:52:42.194131+0000","last_active":"2023-06-10T21:52:42.194131+0000","last_peered":"2023-06-10T21:52:42.194131+0000","last_clean":"2023-06-10T21:52:42.194131+0000","last_became_active":"2023-06-10T21:52:42.193697+0000","last_became_peered":"2023-06-10T21:52:42.193697+0000","last_unstale":"2023-06-10T21:52:42.194131+0000","last_undegraded":"2023-06-10T21:52:42.194131+0000","last_fullsized":"2023-06-10T21:52:42.194131+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":216,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.412972+0000","last_change":"2023-06-10T21:52:28.538012+0000","last_active":"2023-06-10T21:52:42.412972+0000","last_peered":"2023-06-10T21:52:42.412972+0000","last_clean":"2023-06-10T21:52:42.412972+0000","last_became_active":"2023-06-10T21:52:28.537553+0000","last_became_peered":"2023-06-10T21:52:28.537553+0000","last_unstale":"2023-06-10T21:52:42.412972+0000","last_undegraded":"2023-06-10T21:52:42.412972+0000","last_fullsized":"2023-06-10T21:52:42.412972+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:27:47.569689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":508,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193692+0000","last_change":"2023-06-10T21:51:54.605117+0000","last_active":"2023-06-10T21:52:41.193692+0000","last_peered":"2023-06-10T21:52:41.193692+0000","last_clean":"2023-06-10T21:52:41.193692+0000","last_became_active":"2023-06-10T21:51:54.604828+0000","last_became_peered":"2023-06-10T21:51:54.604828+0000","last_unstale":"2023-06-10T21:52:41.193692+0000","last_undegraded":"2023-06-10T21:52:41.193692+0000","last_fullsized":"2023-06-10T21:52:41.193692+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:41:28.154998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":185,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.412875+0000","last_change":"2023-06-10T21:52:25.522080+0000","last_active":"2023-06-10T21:52:42.412875+0000","last_peered":"2023-06-10T21:52:42.412875+0000","last_clean":"2023-06-10T21:52:42.412875+0000","last_became_active":"2023-06-10T21:52:25.521740+0000","last_became_peered":"2023-06-10T21:52:25.521740+0000","last_unstale":"2023-06-10T21:52:42.412875+0000","last_undegraded":"2023-06-10T21:52:42.412875+0000","last_fullsized":"2023-06-10T21:52:42.412875+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:43:13.791566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":47,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.190480+0000","last_change":"2023-06-10T21:51:42.454284+0000","last_active":"2023-06-10T21:52:41.190480+0000","last_peered":"2023-06-10T21:52:41.190480+0000","last_clean":"2023-06-10T21:52:41.190480+0000","last_became_active":"2023-06-10T21:51:42.454018+0000","last_became_peered":"2023-06-10T21:51:42.454018+0000","last_unstale":"2023-06-10T21:52:41.190480+0000","last_undegraded":"2023-06-10T21:52:41.190480+0000","last_fullsized":"2023-06-10T21:52:41.190480+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191603+0000","last_change":"2023-06-10T21:51:42.467047+0000","last_active":"2023-06-10T21:52:41.191603+0000","last_peered":"2023-06-10T21:52:41.191603+0000","last_clean":"2023-06-10T21:52:41.191603+0000","last_became_active":"2023-06-10T21:51:42.466703+0000","last_became_peered":"2023-06-10T21:51:42.466703+0000","last_unstale":"2023-06-10T21:52:41.191603+0000","last_undegraded":"2023-06-10T21:52:41.191603+0000","last_fullsized":"2023-06-10T21:52:41.191603+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":131,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195830+0000","last_change":"2023-06-10T21:51:35.620460+0000","last_active":"2023-06-10T21:52:41.195830+0000","last_peered":"2023-06-10T21:52:41.195830+0000","last_clean":"2023-06-10T21:52:41.195830+0000","last_became_active":"2023-06-10T21:51:35.620047+0000","last_became_peered":"2023-06-10T21:51:35.620047+0000","last_unstale":"2023-06-10T21:52:41.195830+0000","last_undegraded":"2023-06-10T21:52:41.195830+0000","last_fullsized":"2023-06-10T21:52:41.195830+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:21:04.932096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186789+0000","last_change":"2023-06-10T21:51:42.449656+0000","last_active":"2023-06-10T21:52:41.186789+0000","last_peered":"2023-06-10T21:52:41.186789+0000","last_clean":"2023-06-10T21:52:41.186789+0000","last_became_active":"2023-06-10T21:51:42.449318+0000","last_became_peered":"2023-06-10T21:51:42.449318+0000","last_unstale":"2023-06-10T21:52:41.186789+0000","last_undegraded":"2023-06-10T21:52:41.186789+0000","last_fullsized":"2023-06-10T21:52:41.186789+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:25:50.362065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":216,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196484+0000","last_change":"2023-06-10T21:52:35.612582+0000","last_active":"2023-06-10T21:52:41.196484+0000","last_peered":"2023-06-10T21:52:41.196484+0000","last_clean":"2023-06-10T21:52:41.196484+0000","last_became_active":"2023-06-10T21:52:35.611930+0000","last_became_peered":"2023-06-10T21:52:35.611930+0000","last_unstale":"2023-06-10T21:52:41.196484+0000","last_undegraded":"2023-06-10T21:52:41.196484+0000","last_fullsized":"2023-06-10T21:52:41.196484+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":301,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187413+0000","last_change":"2023-06-10T21:48:53.403919+0000","last_active":"2023-06-10T21:52:41.187413+0000","last_peered":"2023-06-10T21:52:41.187413+0000","last_clean":"2023-06-10T21:52:41.187413+0000","last_became_active":"2023-06-10T21:48:53.403593+0000","last_became_peered":"2023-06-10T21:48:53.403593+0000","last_unstale":"2023-06-10T21:52:41.187413+0000","last_undegraded":"2023-06-10T21:52:41.187413+0000","last_fullsized":"2023-06-10T21:52:41.187413+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:06:59.593873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":325,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196546+0000","last_change":"2023-06-10T21:51:38.545495+0000","last_active":"2023-06-10T21:52:41.196546+0000","last_peered":"2023-06-10T21:52:41.196546+0000","last_clean":"2023-06-10T21:52:41.196546+0000","last_became_active":"2023-06-10T21:51:38.438347+0000","last_became_peered":"2023-06-10T21:51:38.438347+0000","last_unstale":"2023-06-10T21:52:41.196546+0000","last_undegraded":"2023-06-10T21:52:41.196546+0000","last_fullsized":"2023-06-10T21:52:41.196546+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":450,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194542+0000","last_change":"2023-06-10T21:51:54.772285+0000","last_active":"2023-06-10T21:52:41.194542+0000","last_peered":"2023-06-10T21:52:41.194542+0000","last_clean":"2023-06-10T21:52:41.194542+0000","last_became_active":"2023-06-10T21:51:54.771899+0000","last_became_peered":"2023-06-10T21:51:54.771899+0000","last_unstale":"2023-06-10T21:52:41.194542+0000","last_undegraded":"2023-06-10T21:52:41.194542+0000","last_fullsized":"2023-06-10T21:52:41.194542+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:21:43.352659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":485,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.197541+0000","last_change":"2023-06-10T21:52:42.197541+0000","last_active":"2023-06-10T21:52:42.197541+0000","last_peered":"2023-06-10T21:52:42.197541+0000","last_clean":"2023-06-10T21:52:42.197541+0000","last_became_active":"2023-06-10T21:52:42.197119+0000","last_became_peered":"2023-06-10T21:52:42.197119+0000","last_unstale":"2023-06-10T21:52:42.197541+0000","last_undegraded":"2023-06-10T21:52:42.197541+0000","last_fullsized":"2023-06-10T21:52:42.197541+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23580,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":108,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.195127+0000","last_change":"2023-06-10T21:52:42.195127+0000","last_active":"2023-06-10T21:52:42.195127+0000","last_peered":"2023-06-10T21:52:42.195127+0000","last_clean":"2023-06-10T21:52:42.195127+0000","last_became_active":"2023-06-10T21:52:42.194712+0000","last_became_peered":"2023-06-10T21:52:42.194712+0000","last_unstale":"2023-06-10T21:52:42.195127+0000","last_undegraded":"2023-06-10T21:52:42.195127+0000","last_fullsized":"2023-06-10T21:52:42.195127+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":642,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196218+0000","last_change":"2023-06-10T21:51:38.632411+0000","last_active":"2023-06-10T21:52:41.196218+0000","last_peered":"2023-06-10T21:52:41.196218+0000","last_clean":"2023-06-10T21:52:41.196218+0000","last_became_active":"2023-06-10T21:51:38.441963+0000","last_became_peered":"2023-06-10T21:51:38.441963+0000","last_unstale":"2023-06-10T21:52:41.196218+0000","last_undegraded":"2023-06-10T21:52:41.196218+0000","last_fullsized":"2023-06-10T21:52:41.196218+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":466,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187451+0000","last_change":"2023-06-10T21:52:28.539071+0000","last_active":"2023-06-10T21:52:41.187451+0000","last_peered":"2023-06-10T21:52:41.187451+0000","last_clean":"2023-06-10T21:52:41.187451+0000","last_became_active":"2023-06-10T21:52:28.538753+0000","last_became_peered":"2023-06-10T21:52:28.538753+0000","last_unstale":"2023-06-10T21:52:41.187451+0000","last_undegraded":"2023-06-10T21:52:41.187451+0000","last_fullsized":"2023-06-10T21:52:41.187451+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:02:57.302797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195228+0000","last_change":"2023-06-10T21:51:35.623072+0000","last_active":"2023-06-10T21:52:41.195228+0000","last_peered":"2023-06-10T21:52:41.195228+0000","last_clean":"2023-06-10T21:52:41.195228+0000","last_became_active":"2023-06-10T21:51:35.622808+0000","last_became_peered":"2023-06-10T21:51:35.622808+0000","last_unstale":"2023-06-10T21:52:41.195228+0000","last_undegraded":"2023-06-10T21:52:41.195228+0000","last_fullsized":"2023-06-10T21:52:41.195228+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:18:28.387280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":433,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197250+0000","last_change":"2023-06-10T21:52:35.612763+0000","last_active":"2023-06-10T21:52:41.197250+0000","last_peered":"2023-06-10T21:52:41.197250+0000","last_clean":"2023-06-10T21:52:41.197250+0000","last_became_active":"2023-06-10T21:52:35.612386+0000","last_became_peered":"2023-06-10T21:52:35.612386+0000","last_unstale":"2023-06-10T21:52:41.197250+0000","last_undegraded":"2023-06-10T21:52:41.197250+0000","last_fullsized":"2023-06-10T21:52:41.197250+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":577,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196152+0000","last_change":"2023-06-10T21:51:38.442370+0000","last_active":"2023-06-10T21:52:41.196152+0000","last_peered":"2023-06-10T21:52:41.196152+0000","last_clean":"2023-06-10T21:52:41.196152+0000","last_became_active":"2023-06-10T21:51:38.442034+0000","last_became_peered":"2023-06-10T21:51:38.442034+0000","last_unstale":"2023-06-10T21:52:41.196152+0000","last_undegraded":"2023-06-10T21:52:41.196152+0000","last_fullsized":"2023-06-10T21:52:41.196152+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195115+0000","last_change":"2023-06-10T21:52:11.469475+0000","last_active":"2023-06-10T21:52:41.195115+0000","last_peered":"2023-06-10T21:52:41.195115+0000","last_clean":"2023-06-10T21:52:41.195115+0000","last_became_active":"2023-06-10T21:52:11.469147+0000","last_became_peered":"2023-06-10T21:52:11.469147+0000","last_unstale":"2023-06-10T21:52:41.195115+0000","last_undegraded":"2023-06-10T21:52:41.195115+0000","last_fullsized":"2023-06-10T21:52:41.195115+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:20:03.524750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":122,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.190717+0000","last_change":"2023-06-10T21:52:31.729715+0000","last_active":"2023-06-10T21:52:41.190717+0000","last_peered":"2023-06-10T21:52:41.190717+0000","last_clean":"2023-06-10T21:52:41.190717+0000","last_became_active":"2023-06-10T21:52:31.723271+0000","last_became_peered":"2023-06-10T21:52:31.723271+0000","last_unstale":"2023-06-10T21:52:41.190717+0000","last_undegraded":"2023-06-10T21:52:41.190717+0000","last_fullsized":"2023-06-10T21:52:41.190717+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":20,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187188+0000","last_change":"2023-06-10T21:52:25.522116+0000","last_active":"2023-06-10T21:52:41.187188+0000","last_peered":"2023-06-10T21:52:41.187188+0000","last_clean":"2023-06-10T21:52:41.187188+0000","last_became_active":"2023-06-10T21:52:25.521818+0000","last_became_peered":"2023-06-10T21:52:25.521818+0000","last_unstale":"2023-06-10T21:52:41.187188+0000","last_undegraded":"2023-06-10T21:52:41.187188+0000","last_fullsized":"2023-06-10T21:52:41.187188+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:20:20.987643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":382,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196086+0000","last_change":"2023-06-10T21:51:38.457228+0000","last_active":"2023-06-10T21:52:41.196086+0000","last_peered":"2023-06-10T21:52:41.196086+0000","last_clean":"2023-06-10T21:52:41.196086+0000","last_became_active":"2023-06-10T21:51:38.433522+0000","last_became_peered":"2023-06-10T21:51:38.433522+0000","last_unstale":"2023-06-10T21:52:41.196086+0000","last_undegraded":"2023-06-10T21:52:41.196086+0000","last_fullsized":"2023-06-10T21:52:41.196086+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":242,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197063+0000","last_change":"2023-06-10T21:52:35.612340+0000","last_active":"2023-06-10T21:52:41.197063+0000","last_peered":"2023-06-10T21:52:41.197063+0000","last_clean":"2023-06-10T21:52:41.197063+0000","last_became_active":"2023-06-10T21:52:35.611968+0000","last_became_peered":"2023-06-10T21:52:35.611968+0000","last_unstale":"2023-06-10T21:52:41.197063+0000","last_undegraded":"2023-06-10T21:52:41.197063+0000","last_fullsized":"2023-06-10T21:52:41.197063+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":382,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196186+0000","last_change":"2023-06-10T21:51:42.505549+0000","last_active":"2023-06-10T21:52:41.196186+0000","last_peered":"2023-06-10T21:52:41.196186+0000","last_clean":"2023-06-10T21:52:41.196186+0000","last_became_active":"2023-06-10T21:51:42.468939+0000","last_became_peered":"2023-06-10T21:51:42.468939+0000","last_unstale":"2023-06-10T21:52:41.196186+0000","last_undegraded":"2023-06-10T21:52:41.196186+0000","last_fullsized":"2023-06-10T21:52:41.196186+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":114,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415208+0000","last_change":"2023-06-10T21:52:42.415208+0000","last_active":"2023-06-10T21:52:42.415208+0000","last_peered":"2023-06-10T21:52:42.415208+0000","last_clean":"2023-06-10T21:52:42.415208+0000","last_became_active":"2023-06-10T21:52:42.414810+0000","last_became_peered":"2023-06-10T21:52:42.414810+0000","last_unstale":"2023-06-10T21:52:42.415208+0000","last_undegraded":"2023-06-10T21:52:42.415208+0000","last_fullsized":"2023-06-10T21:52:42.415208+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":656,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196617+0000","last_change":"2023-06-10T21:52:31.773663+0000","last_active":"2023-06-10T21:52:41.196617+0000","last_peered":"2023-06-10T21:52:41.196617+0000","last_clean":"2023-06-10T21:52:41.196617+0000","last_became_active":"2023-06-10T21:52:31.723869+0000","last_became_peered":"2023-06-10T21:52:31.723869+0000","last_unstale":"2023-06-10T21:52:41.196617+0000","last_undegraded":"2023-06-10T21:52:41.196617+0000","last_fullsized":"2023-06-10T21:52:41.196617+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192027+0000","last_change":"2023-06-10T21:51:42.453220+0000","last_active":"2023-06-10T21:52:41.192027+0000","last_peered":"2023-06-10T21:52:41.192027+0000","last_clean":"2023-06-10T21:52:41.192027+0000","last_became_active":"2023-06-10T21:51:42.452462+0000","last_became_peered":"2023-06-10T21:51:42.452462+0000","last_unstale":"2023-06-10T21:52:41.192027+0000","last_undegraded":"2023-06-10T21:52:41.192027+0000","last_fullsized":"2023-06-10T21:52:41.192027+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":203,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196815+0000","last_change":"2023-06-10T21:52:35.570481+0000","last_active":"2023-06-10T21:52:41.196815+0000","last_peered":"2023-06-10T21:52:41.196815+0000","last_clean":"2023-06-10T21:52:41.196815+0000","last_became_active":"2023-06-10T21:52:35.569623+0000","last_became_peered":"2023-06-10T21:52:35.569623+0000","last_unstale":"2023-06-10T21:52:41.196815+0000","last_undegraded":"2023-06-10T21:52:41.196815+0000","last_fullsized":"2023-06-10T21:52:41.196815+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":148,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196172+0000","last_change":"2023-06-10T21:51:42.453723+0000","last_active":"2023-06-10T21:52:41.196172+0000","last_peered":"2023-06-10T21:52:41.196172+0000","last_clean":"2023-06-10T21:52:41.196172+0000","last_became_active":"2023-06-10T21:51:42.453432+0000","last_became_peered":"2023-06-10T21:51:42.453432+0000","last_unstale":"2023-06-10T21:52:41.196172+0000","last_undegraded":"2023-06-10T21:52:41.196172+0000","last_fullsized":"2023-06-10T21:52:41.196172+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:09:39.344361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":189,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186835+0000","last_change":"2023-06-10T21:52:28.543205+0000","last_active":"2023-06-10T21:52:41.186835+0000","last_peered":"2023-06-10T21:52:41.186835+0000","last_clean":"2023-06-10T21:52:41.186835+0000","last_became_active":"2023-06-10T21:52:28.542871+0000","last_became_peered":"2023-06-10T21:52:28.542871+0000","last_unstale":"2023-06-10T21:52:41.186835+0000","last_undegraded":"2023-06-10T21:52:41.186835+0000","last_fullsized":"2023-06-10T21:52:41.186835+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:36:35.841640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415194+0000","last_change":"2023-06-10T21:52:42.415194+0000","last_active":"2023-06-10T21:52:42.415194+0000","last_peered":"2023-06-10T21:52:42.415194+0000","last_clean":"2023-06-10T21:52:42.415194+0000","last_became_active":"2023-06-10T21:52:42.414764+0000","last_became_peered":"2023-06-10T21:52:42.414764+0000","last_unstale":"2023-06-10T21:52:42.415194+0000","last_undegraded":"2023-06-10T21:52:42.415194+0000","last_fullsized":"2023-06-10T21:52:42.415194+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191755+0000","last_change":"2023-06-10T21:51:42.467839+0000","last_active":"2023-06-10T21:52:41.191755+0000","last_peered":"2023-06-10T21:52:41.191755+0000","last_clean":"2023-06-10T21:52:41.191755+0000","last_became_active":"2023-06-10T21:51:42.467496+0000","last_became_peered":"2023-06-10T21:51:42.467496+0000","last_unstale":"2023-06-10T21:52:41.191755+0000","last_undegraded":"2023-06-10T21:52:41.191755+0000","last_fullsized":"2023-06-10T21:52:41.191755+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":623,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.201638+0000","last_change":"2023-06-10T21:52:42.201638+0000","last_active":"2023-06-10T21:52:42.201638+0000","last_peered":"2023-06-10T21:52:42.201638+0000","last_clean":"2023-06-10T21:52:42.201638+0000","last_became_active":"2023-06-10T21:52:42.201268+0000","last_became_peered":"2023-06-10T21:52:42.201268+0000","last_unstale":"2023-06-10T21:52:42.201638+0000","last_undegraded":"2023-06-10T21:52:42.201638+0000","last_fullsized":"2023-06-10T21:52:42.201638+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":559,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.197449+0000","last_change":"2023-06-10T21:52:42.197449+0000","last_active":"2023-06-10T21:52:42.197449+0000","last_peered":"2023-06-10T21:52:42.197449+0000","last_clean":"2023-06-10T21:52:42.197449+0000","last_became_active":"2023-06-10T21:52:42.196993+0000","last_became_peered":"2023-06-10T21:52:42.196993+0000","last_unstale":"2023-06-10T21:52:42.197449+0000","last_undegraded":"2023-06-10T21:52:42.197449+0000","last_fullsized":"2023-06-10T21:52:42.197449+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":108,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.195128+0000","last_change":"2023-06-10T21:52:42.195128+0000","last_active":"2023-06-10T21:52:42.195128+0000","last_peered":"2023-06-10T21:52:42.195128+0000","last_clean":"2023-06-10T21:52:42.195128+0000","last_became_active":"2023-06-10T21:52:42.194750+0000","last_became_peered":"2023-06-10T21:52:42.194750+0000","last_unstale":"2023-06-10T21:52:42.195128+0000","last_undegraded":"2023-06-10T21:52:42.195128+0000","last_fullsized":"2023-06-10T21:52:42.195128+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186659+0000","last_change":"2023-06-10T21:52:25.522185+0000","last_active":"2023-06-10T21:52:41.186659+0000","last_peered":"2023-06-10T21:52:41.186659+0000","last_clean":"2023-06-10T21:52:41.186659+0000","last_became_active":"2023-06-10T21:52:25.521863+0000","last_became_peered":"2023-06-10T21:52:25.521863+0000","last_unstale":"2023-06-10T21:52:41.186659+0000","last_undegraded":"2023-06-10T21:52:41.186659+0000","last_fullsized":"2023-06-10T21:52:41.186659+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T23:27:24.403777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":35,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194822+0000","last_change":"2023-06-10T21:51:42.452748+0000","last_active":"2023-06-10T21:52:41.194822+0000","last_peered":"2023-06-10T21:52:41.194822+0000","last_clean":"2023-06-10T21:52:41.194822+0000","last_became_active":"2023-06-10T21:51:42.452329+0000","last_became_peered":"2023-06-10T21:51:42.452329+0000","last_unstale":"2023-06-10T21:52:41.194822+0000","last_undegraded":"2023-06-10T21:52:41.194822+0000","last_fullsized":"2023-06-10T21:52:41.194822+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T02:33:37.056969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193887+0000","last_change":"2023-06-10T21:51:54.771722+0000","last_active":"2023-06-10T21:52:41.193887+0000","last_peered":"2023-06-10T21:52:41.193887+0000","last_clean":"2023-06-10T21:52:41.193887+0000","last_became_active":"2023-06-10T21:51:54.771076+0000","last_became_peered":"2023-06-10T21:51:54.771076+0000","last_unstale":"2023-06-10T21:52:41.193887+0000","last_undegraded":"2023-06-10T21:52:41.193887+0000","last_fullsized":"2023-06-10T21:52:41.193887+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:50:38.207479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":166,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197229+0000","last_change":"2023-06-10T21:52:35.570035+0000","last_active":"2023-06-10T21:52:41.197229+0000","last_peered":"2023-06-10T21:52:41.197229+0000","last_clean":"2023-06-10T21:52:41.197229+0000","last_became_active":"2023-06-10T21:52:35.569547+0000","last_became_peered":"2023-06-10T21:52:35.569547+0000","last_unstale":"2023-06-10T21:52:41.197229+0000","last_undegraded":"2023-06-10T21:52:41.197229+0000","last_fullsized":"2023-06-10T21:52:41.197229+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":148,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194768+0000","last_change":"2023-06-10T21:52:31.723307+0000","last_active":"2023-06-10T21:52:41.194768+0000","last_peered":"2023-06-10T21:52:41.194768+0000","last_clean":"2023-06-10T21:52:41.194768+0000","last_became_active":"2023-06-10T21:52:31.722848+0000","last_became_peered":"2023-06-10T21:52:31.722848+0000","last_unstale":"2023-06-10T21:52:41.194768+0000","last_undegraded":"2023-06-10T21:52:41.194768+0000","last_fullsized":"2023-06-10T21:52:41.194768+0000","mapping_epoch":532,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T05:21:32.882245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":104,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196542+0000","last_change":"2023-06-10T21:52:35.570218+0000","last_active":"2023-06-10T21:52:41.196542+0000","last_peered":"2023-06-10T21:52:41.196542+0000","last_clean":"2023-06-10T21:52:41.196542+0000","last_became_active":"2023-06-10T21:52:35.569508+0000","last_became_peered":"2023-06-10T21:52:35.569508+0000","last_unstale":"2023-06-10T21:52:41.196542+0000","last_undegraded":"2023-06-10T21:52:41.196542+0000","last_fullsized":"2023-06-10T21:52:41.196542+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415107+0000","last_change":"2023-06-10T21:52:42.415107+0000","last_active":"2023-06-10T21:52:42.415107+0000","last_peered":"2023-06-10T21:52:42.415107+0000","last_clean":"2023-06-10T21:52:42.415107+0000","last_became_active":"2023-06-10T21:52:42.414682+0000","last_became_peered":"2023-06-10T21:52:42.414682+0000","last_unstale":"2023-06-10T21:52:42.415107+0000","last_undegraded":"2023-06-10T21:52:42.415107+0000","last_fullsized":"2023-06-10T21:52:42.415107+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":423,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.193971+0000","last_change":"2023-06-10T21:51:54.605586+0000","last_active":"2023-06-10T21:52:41.193971+0000","last_peered":"2023-06-10T21:52:41.193971+0000","last_clean":"2023-06-10T21:52:41.193971+0000","last_became_active":"2023-06-10T21:51:54.605214+0000","last_became_peered":"2023-06-10T21:51:54.605214+0000","last_unstale":"2023-06-10T21:52:41.193971+0000","last_undegraded":"2023-06-10T21:52:41.193971+0000","last_fullsized":"2023-06-10T21:52:41.193971+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:08:37.070016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":435,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.661137+0000","last_change":"2023-06-10T21:47:18.337074+0000","last_active":"2023-06-10T21:52:41.661137+0000","last_peered":"2023-06-10T21:52:41.661137+0000","last_clean":"2023-06-10T21:52:41.661137+0000","last_became_active":"2023-06-10T21:47:18.336698+0000","last_became_peered":"2023-06-10T21:47:18.336698+0000","last_unstale":"2023-06-10T21:52:41.661137+0000","last_undegraded":"2023-06-10T21:52:41.661137+0000","last_fullsized":"2023-06-10T21:52:41.661137+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:29:58.371008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":457,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192698+0000","last_change":"2023-06-10T21:51:54.771505+0000","last_active":"2023-06-10T21:52:41.192698+0000","last_peered":"2023-06-10T21:52:41.192698+0000","last_clean":"2023-06-10T21:52:41.192698+0000","last_became_active":"2023-06-10T21:51:54.771170+0000","last_became_peered":"2023-06-10T21:51:54.771170+0000","last_unstale":"2023-06-10T21:52:41.192698+0000","last_undegraded":"2023-06-10T21:52:41.192698+0000","last_fullsized":"2023-06-10T21:52:41.192698+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T00:31:32.827946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194724+0000","last_change":"2023-06-10T21:51:54.464705+0000","last_active":"2023-06-10T21:52:41.194724+0000","last_peered":"2023-06-10T21:52:41.194724+0000","last_clean":"2023-06-10T21:52:41.194724+0000","last_became_active":"2023-06-10T21:51:54.464393+0000","last_became_peered":"2023-06-10T21:51:54.464393+0000","last_unstale":"2023-06-10T21:52:41.194724+0000","last_undegraded":"2023-06-10T21:52:41.194724+0000","last_fullsized":"2023-06-10T21:52:41.194724+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:14:33.254460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":490,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197134+0000","last_change":"2023-06-10T21:52:35.612393+0000","last_active":"2023-06-10T21:52:41.197134+0000","last_peered":"2023-06-10T21:52:41.197134+0000","last_clean":"2023-06-10T21:52:41.197134+0000","last_became_active":"2023-06-10T21:52:35.611891+0000","last_became_peered":"2023-06-10T21:52:35.611891+0000","last_unstale":"2023-06-10T21:52:41.197134+0000","last_undegraded":"2023-06-10T21:52:41.197134+0000","last_fullsized":"2023-06-10T21:52:41.197134+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"77'139","reported_seq":784,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195607+0000","last_change":"2023-06-10T21:48:56.439594+0000","last_active":"2023-06-10T21:52:41.195607+0000","last_peered":"2023-06-10T21:52:41.195607+0000","last_clean":"2023-06-10T21:52:41.195607+0000","last_became_active":"2023-06-10T21:48:56.439122+0000","last_became_peered":"2023-06-10T21:48:56.439122+0000","last_unstale":"2023-06-10T21:52:41.195607+0000","last_undegraded":"2023-06-10T21:52:41.195607+0000","last_fullsized":"2023-06-10T21:52:41.195607+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:56:10.244684+0000","last_clean_scrub_stamp":"2023-06-10T20:56:10.244684+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":206,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.412811+0000","last_change":"2023-06-10T21:52:28.611792+0000","last_active":"2023-06-10T21:52:42.412811+0000","last_peered":"2023-06-10T21:52:42.412811+0000","last_clean":"2023-06-10T21:52:42.412811+0000","last_became_active":"2023-06-10T21:52:28.611535+0000","last_became_peered":"2023-06-10T21:52:28.611535+0000","last_unstale":"2023-06-10T21:52:42.412811+0000","last_undegraded":"2023-06-10T21:52:42.412811+0000","last_fullsized":"2023-06-10T21:52:42.412811+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:50:35.735362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192567+0000","last_change":"2023-06-10T21:51:38.449633+0000","last_active":"2023-06-10T21:52:41.192567+0000","last_peered":"2023-06-10T21:52:41.192567+0000","last_clean":"2023-06-10T21:52:41.192567+0000","last_became_active":"2023-06-10T21:51:38.449039+0000","last_became_peered":"2023-06-10T21:51:38.449039+0000","last_unstale":"2023-06-10T21:52:41.192567+0000","last_undegraded":"2023-06-10T21:52:41.192567+0000","last_fullsized":"2023-06-10T21:52:41.192567+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":608,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195715+0000","last_change":"2023-06-10T21:51:42.467529+0000","last_active":"2023-06-10T21:52:41.195715+0000","last_peered":"2023-06-10T21:52:41.195715+0000","last_clean":"2023-06-10T21:52:41.195715+0000","last_became_active":"2023-06-10T21:51:42.467213+0000","last_became_peered":"2023-06-10T21:51:42.467213+0000","last_unstale":"2023-06-10T21:52:41.195715+0000","last_undegraded":"2023-06-10T21:52:41.195715+0000","last_fullsized":"2023-06-10T21:52:41.195715+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":157,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194903+0000","last_change":"2023-06-10T21:52:25.516479+0000","last_active":"2023-06-10T21:52:41.194903+0000","last_peered":"2023-06-10T21:52:41.194903+0000","last_clean":"2023-06-10T21:52:41.194903+0000","last_became_active":"2023-06-10T21:52:25.516147+0000","last_became_peered":"2023-06-10T21:52:25.516147+0000","last_unstale":"2023-06-10T21:52:41.194903+0000","last_undegraded":"2023-06-10T21:52:41.194903+0000","last_fullsized":"2023-06-10T21:52:41.194903+0000","mapping_epoch":528,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":529,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T07:13:25.809519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195436+0000","last_change":"2023-06-10T21:51:42.455442+0000","last_active":"2023-06-10T21:52:41.195436+0000","last_peered":"2023-06-10T21:52:41.195436+0000","last_clean":"2023-06-10T21:52:41.195436+0000","last_became_active":"2023-06-10T21:51:42.454424+0000","last_became_peered":"2023-06-10T21:51:42.454424+0000","last_unstale":"2023-06-10T21:52:41.195436+0000","last_undegraded":"2023-06-10T21:52:41.195436+0000","last_fullsized":"2023-06-10T21:52:41.195436+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:16:08.780005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":605,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196663+0000","last_change":"2023-06-10T21:51:38.431740+0000","last_active":"2023-06-10T21:52:41.196663+0000","last_peered":"2023-06-10T21:52:41.196663+0000","last_clean":"2023-06-10T21:52:41.196663+0000","last_became_active":"2023-06-10T21:51:38.431383+0000","last_became_peered":"2023-06-10T21:51:38.431383+0000","last_unstale":"2023-06-10T21:52:41.196663+0000","last_undegraded":"2023-06-10T21:52:41.196663+0000","last_fullsized":"2023-06-10T21:52:41.196663+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":456,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194193+0000","last_change":"2023-06-10T21:51:54.457776+0000","last_active":"2023-06-10T21:52:41.194193+0000","last_peered":"2023-06-10T21:52:41.194193+0000","last_clean":"2023-06-10T21:52:41.194193+0000","last_became_active":"2023-06-10T21:51:54.457479+0000","last_became_peered":"2023-06-10T21:51:54.457479+0000","last_unstale":"2023-06-10T21:52:41.194193+0000","last_undegraded":"2023-06-10T21:52:41.194193+0000","last_fullsized":"2023-06-10T21:52:41.194193+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:00:58.895011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":130,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196339+0000","last_change":"2023-06-10T21:51:34.508129+0000","last_active":"2023-06-10T21:52:41.196339+0000","last_peered":"2023-06-10T21:52:41.196339+0000","last_clean":"2023-06-10T21:52:41.196339+0000","last_became_active":"2023-06-10T21:51:34.507299+0000","last_became_peered":"2023-06-10T21:51:34.507299+0000","last_unstale":"2023-06-10T21:52:41.196339+0000","last_undegraded":"2023-06-10T21:52:41.196339+0000","last_fullsized":"2023-06-10T21:52:41.196339+0000","mapping_epoch":508,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:46:03.722972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":542,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.413307+0000","last_change":"2023-06-10T21:52:28.541913+0000","last_active":"2023-06-10T21:52:42.413307+0000","last_peered":"2023-06-10T21:52:42.413307+0000","last_clean":"2023-06-10T21:52:42.413307+0000","last_became_active":"2023-06-10T21:52:28.541543+0000","last_became_peered":"2023-06-10T21:52:28.541543+0000","last_unstale":"2023-06-10T21:52:42.413307+0000","last_undegraded":"2023-06-10T21:52:42.413307+0000","last_fullsized":"2023-06-10T21:52:42.413307+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:04:00.055916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.194543+0000","last_change":"2023-06-10T21:51:54.771711+0000","last_active":"2023-06-10T21:52:41.194543+0000","last_peered":"2023-06-10T21:52:41.194543+0000","last_clean":"2023-06-10T21:52:41.194543+0000","last_became_active":"2023-06-10T21:51:54.771268+0000","last_became_peered":"2023-06-10T21:51:54.771268+0000","last_unstale":"2023-06-10T21:52:41.194543+0000","last_undegraded":"2023-06-10T21:52:41.194543+0000","last_fullsized":"2023-06-10T21:52:41.194543+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T22:53:11.763424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":435,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.186650+0000","last_change":"2023-06-10T21:52:28.542998+0000","last_active":"2023-06-10T21:52:41.186650+0000","last_peered":"2023-06-10T21:52:41.186650+0000","last_clean":"2023-06-10T21:52:41.186650+0000","last_became_active":"2023-06-10T21:52:28.542662+0000","last_became_peered":"2023-06-10T21:52:28.542662+0000","last_unstale":"2023-06-10T21:52:41.186650+0000","last_undegraded":"2023-06-10T21:52:41.186650+0000","last_fullsized":"2023-06-10T21:52:41.186650+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T08:31:25.639808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.191514+0000","last_change":"2023-06-10T21:52:11.470476+0000","last_active":"2023-06-10T21:52:41.191514+0000","last_peered":"2023-06-10T21:52:41.191514+0000","last_clean":"2023-06-10T21:52:41.191514+0000","last_became_active":"2023-06-10T21:52:11.470042+0000","last_became_peered":"2023-06-10T21:52:11.470042+0000","last_unstale":"2023-06-10T21:52:41.191514+0000","last_undegraded":"2023-06-10T21:52:41.191514+0000","last_fullsized":"2023-06-10T21:52:41.191514+0000","mapping_epoch":523,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":524,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:58:33.354602+0000","last_clean_scrub_stamp":"2023-06-10T20:58:33.354602+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":169,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195232+0000","last_change":"2023-06-10T21:51:42.502505+0000","last_active":"2023-06-10T21:52:41.195232+0000","last_peered":"2023-06-10T21:52:41.195232+0000","last_clean":"2023-06-10T21:52:41.195232+0000","last_became_active":"2023-06-10T21:51:42.454911+0000","last_became_peered":"2023-06-10T21:51:42.454911+0000","last_unstale":"2023-06-10T21:52:41.195232+0000","last_undegraded":"2023-06-10T21:52:41.195232+0000","last_fullsized":"2023-06-10T21:52:41.195232+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T01:31:54.520696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.201688+0000","last_change":"2023-06-10T21:52:42.201688+0000","last_active":"2023-06-10T21:52:42.201688+0000","last_peered":"2023-06-10T21:52:42.201688+0000","last_clean":"2023-06-10T21:52:42.201688+0000","last_became_active":"2023-06-10T21:52:42.201294+0000","last_became_peered":"2023-06-10T21:52:42.201294+0000","last_unstale":"2023-06-10T21:52:42.201688+0000","last_undegraded":"2023-06-10T21:52:42.201688+0000","last_fullsized":"2023-06-10T21:52:42.201688+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":121,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196294+0000","last_change":"2023-06-10T21:51:38.435920+0000","last_active":"2023-06-10T21:52:41.196294+0000","last_peered":"2023-06-10T21:52:41.196294+0000","last_clean":"2023-06-10T21:52:41.196294+0000","last_became_active":"2023-06-10T21:51:38.435245+0000","last_became_peered":"2023-06-10T21:51:38.435245+0000","last_unstale":"2023-06-10T21:52:41.196294+0000","last_undegraded":"2023-06-10T21:52:41.196294+0000","last_fullsized":"2023-06-10T21:52:41.196294+0000","mapping_epoch":511,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":512,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-11T21:44:11.651949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":30,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.192457+0000","last_change":"2023-06-10T21:51:58.583757+0000","last_active":"2023-06-10T21:52:41.192457+0000","last_peered":"2023-06-10T21:52:41.192457+0000","last_clean":"2023-06-10T21:52:41.192457+0000","last_became_active":"2023-06-10T21:51:58.583392+0000","last_became_peered":"2023-06-10T21:51:58.583392+0000","last_unstale":"2023-06-10T21:52:41.192457+0000","last_undegraded":"2023-06-10T21:52:41.192457+0000","last_fullsized":"2023-06-10T21:52:41.192457+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:08:33.269257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":112,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.196910+0000","last_change":"2023-06-10T21:52:35.570688+0000","last_active":"2023-06-10T21:52:41.196910+0000","last_peered":"2023-06-10T21:52:41.196910+0000","last_clean":"2023-06-10T21:52:41.196910+0000","last_became_active":"2023-06-10T21:52:35.570114+0000","last_became_peered":"2023-06-10T21:52:35.570114+0000","last_unstale":"2023-06-10T21:52:41.196910+0000","last_undegraded":"2023-06-10T21:52:41.196910+0000","last_fullsized":"2023-06-10T21:52:41.196910+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":431,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415636+0000","last_change":"2023-06-10T21:52:42.415636+0000","last_active":"2023-06-10T21:52:42.415636+0000","last_peered":"2023-06-10T21:52:42.415636+0000","last_clean":"2023-06-10T21:52:42.415636+0000","last_became_active":"2023-06-10T21:52:42.415295+0000","last_became_peered":"2023-06-10T21:52:42.415295+0000","last_unstale":"2023-06-10T21:52:42.415636+0000","last_undegraded":"2023-06-10T21:52:42.415636+0000","last_fullsized":"2023-06-10T21:52:42.415636+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.197348+0000","last_change":"2023-06-10T21:52:35.569232+0000","last_active":"2023-06-10T21:52:41.197348+0000","last_peered":"2023-06-10T21:52:41.197348+0000","last_clean":"2023-06-10T21:52:41.197348+0000","last_became_active":"2023-06-10T21:52:35.568793+0000","last_became_peered":"2023-06-10T21:52:35.568793+0000","last_unstale":"2023-06-10T21:52:41.197348+0000","last_undegraded":"2023-06-10T21:52:41.197348+0000","last_fullsized":"2023-06-10T21:52:41.197348+0000","mapping_epoch":534,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":535,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":497,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.415252+0000","last_change":"2023-06-10T21:52:42.415252+0000","last_active":"2023-06-10T21:52:42.415252+0000","last_peered":"2023-06-10T21:52:42.415252+0000","last_clean":"2023-06-10T21:52:42.415252+0000","last_became_active":"2023-06-10T21:52:42.414816+0000","last_became_peered":"2023-06-10T21:52:42.414816+0000","last_unstale":"2023-06-10T21:52:42.415252+0000","last_undegraded":"2023-06-10T21:52:42.415252+0000","last_fullsized":"2023-06-10T21:52:42.415252+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":182,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.187633+0000","last_change":"2023-06-10T21:52:28.542490+0000","last_active":"2023-06-10T21:52:41.187633+0000","last_peered":"2023-06-10T21:52:41.187633+0000","last_clean":"2023-06-10T21:52:41.187633+0000","last_became_active":"2023-06-10T21:52:28.542147+0000","last_became_peered":"2023-06-10T21:52:28.542147+0000","last_unstale":"2023-06-10T21:52:41.187633+0000","last_undegraded":"2023-06-10T21:52:41.187633+0000","last_fullsized":"2023-06-10T21:52:41.187633+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T03:57:02.358984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":349,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.703910+0000","last_change":"2023-06-10T21:52:28.543025+0000","last_active":"2023-06-10T21:52:41.703910+0000","last_peered":"2023-06-10T21:52:41.703910+0000","last_clean":"2023-06-10T21:52:41.703910+0000","last_became_active":"2023-06-10T21:52:28.542727+0000","last_became_peered":"2023-06-10T21:52:28.542727+0000","last_unstale":"2023-06-10T21:52:41.703910+0000","last_undegraded":"2023-06-10T21:52:41.703910+0000","last_fullsized":"2023-06-10T21:52:41.703910+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T04:24:59.071503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":193,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-06-10T21:52:41.195655+0000","last_change":"2023-06-10T21:51:58.477208+0000","last_active":"2023-06-10T21:52:41.195655+0000","last_peered":"2023-06-10T21:52:41.195655+0000","last_clean":"2023-06-10T21:52:41.195655+0000","last_became_active":"2023-06-10T21:51:58.473617+0000","last_became_peered":"2023-06-10T21:51:58.473617+0000","last_unstale":"2023-06-10T21:52:41.195655+0000","last_undegraded":"2023-06-10T21:52:41.195655+0000","last_fullsized":"2023-06-10T21:52:41.195655+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:24.384447+0000","last_clean_scrub_stamp":"2023-06-10T20:59:24.384447+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-12T06:57:58.890185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-06-10T21:52:42.193806+0000","last_change":"2023-06-10T21:52:42.193806+0000","last_active":"2023-06-10T21:52:42.193806+0000","last_peered":"2023-06-10T21:52:42.193806+0000","last_clean":"2023-06-10T21:52:42.193806+0000","last_became_active":"2023-06-10T21:52:42.193424+0000","last_became_peered":"2023-06-10T21:52:42.193424+0000","last_unstale":"2023-06-10T21:52:42.193806+0000","last_undegraded":"2023-06-10T21:52:42.193806+0000","last_fullsized":"2023-06-10T21:52:42.193806+0000","mapping_epoch":537,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":538,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-10T20:59:25.389257+0000","last_clean_scrub_stamp":"2023-06-10T20:59:25.389257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":29606,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56940,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430876,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":117004,"kb_used_data":5316,"kb_used_omap":10,"kb_used_meta":111669,"kb_avail":93628148,"statfs":{"total":95995035648,"available":95875223552,"internally_reserved":0,"allocated":5443584,"data_stored":4612952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":114349117},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:16 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":2.831,"15min":2.4790000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.746,"5min":125.93300000000001,"15min":170.05199999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":3.3109999999999999,"15min":3.0179999999999998},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.246},"max":{"1min":0.70499999999999996,"5min":126.14,"15min":170.126},"last":0.499}]},{"osd":1,"last update":"Sat Jun 10 21:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.2669999999999999,"15min":1.5920000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.70399999999999996,"5min":13.180999999999999,"15min":38.078000000000003},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.2170000000000001,"15min":1.8360000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28599999999999998},"max":{"1min":0.82699999999999996,"5min":7.3280000000000003,"15min":38.341000000000001},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Jun 10 21:52:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":2.9409999999999998,"15min":5.5140000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68100000000000005,"5min":80.334000000000003,"15min":218.90600000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":2.9740000000000002,"15min":5.4560000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.76200000000000001,"5min":80.262,"15min":218.851},"last":0.78300000000000003}]},{"osd":3,"last update":"Sat Jun 10 21:52:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.792,"15min":4.6379999999999999},"min":{"1min":0.35499999999999998,"5min":0.34899999999999998,"15min":0.26100000000000001},"max":{"1min":0.80000000000000004,"5min":28.109999999999999,"15min":218.35300000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":2.1739999999999999,"15min":4.694},"min":{"1min":0.36499999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.85399999999999998,"5min":30.981000000000002,"15min":218.74600000000001},"last":0.69899999999999995}]},{"osd":4,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.76500000000000001}]},{"osd":5,"last update":"Sat Jun 10 21:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.79200000000000004,"15min":6.266},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.85599999999999998,"5min":5.5700000000000003,"15min":131.619},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.298,"15min":6.3639999999999999},"min":{"1min":0.38700000000000001,"5min":0.30099999999999999,"15min":0.24299999999999999},"max":{"1min":0.92500000000000004,"5min":18.277999999999999,"15min":134.928},"last":0.752}]},{"osd":6,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.0609999999999999,"15min":0.94499999999999995},"min":{"1min":0.40999999999999998,"5min":0.317,"15min":0.26300000000000001},"max":{"1min":0.86699999999999999,"5min":11.348000000000001,"15min":11.348000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.97299999999999998,"15min":1.028},"min":{"1min":0.40000000000000002,"5min":0.34100000000000003,"15min":0.30599999999999999},"max":{"1min":0.97099999999999997,"5min":7.7960000000000003,"15min":8.1020000000000003},"last":0.86599999999999999}]}]},{"osd":6,"up_from":42,"seq":180388627104,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513144,"kb_used_data":5296,"kb_used_omap":0,"kb_used_meta":507840,"kb_avail":93232008,"statfs":{"total":95995035648,"available":95469576192,"internally_reserved":0,"allocated":5423104,"data_stored":4611244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":520028160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.141,"15min":1.397},"min":{"1min":0.307,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63100000000000001,"5min":7.6840000000000002,"15min":107.86},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":2.6259999999999999,"15min":2.0899999999999999},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73399999999999999,"5min":136.85900000000001,"15min":136.85900000000001},"last":0.52000000000000002}]},{"osd":1,"last update":"Sat Jun 10 21:53:04 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.90600000000000003,"15min":1.492},"min":{"1min":0.40100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.70999999999999996,"5min":6.3559999999999999,"15min":80.210999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.198,"15min":1.754},"min":{"1min":0.36399999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":0.66700000000000004,"5min":8.0079999999999991,"15min":79.894000000000005},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":6.5369999999999999,"15min":6.6550000000000002},"min":{"1min":0.40799999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.92400000000000004,"5min":265.37099999999998,"15min":265.37099999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":6.6399999999999997,"15min":6.7519999999999998},"min":{"1min":0.35799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.93400000000000005,"5min":265.32900000000001,"15min":265.32900000000001},"last":0.374}]},{"osd":3,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":6.7140000000000004,"15min":6.5739999999999998},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.30099999999999999},"max":{"1min":0.75800000000000001,"5min":412.55200000000002,"15min":412.55200000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":6.9379999999999997,"15min":6.4950000000000001},"min":{"1min":0.42299999999999999,"5min":0.39100000000000001,"15min":0.35499999999999998},"max":{"1min":0.77500000000000002,"5min":412.41199999999998,"15min":412.41199999999998},"last":0.67000000000000004}]},{"osd":4,"last update":"Sat Jun 10 21:52:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.497}]},{"osd":5,"last update":"Sat Jun 10 21:53:13 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.93000000000000005,"15min":1.077},"min":{"1min":0.44700000000000001,"5min":0.27700000000000002,"15min":0.27500000000000002},"max":{"1min":0.90300000000000002,"5min":8.0250000000000004,"15min":8.0250000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.95199999999999996,"15min":1.375},"min":{"1min":0.51300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.83699999999999997,"5min":7.3209999999999997,"15min":7.3209999999999997},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":2.093,"15min":2.9079999999999999},"min":{"1min":0.44600000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81899999999999995,"5min":97.108000000000004,"15min":189.042},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":2.101,"15min":3.1720000000000002},"min":{"1min":0.46500000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":97.299000000000007,"15min":190.83699999999999},"last":0.62}]}]},{"osd":1,"up_from":105,"seq":450971566546,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923892,"kb_used_data":5804,"kb_used_omap":4,"kb_used_meta":918075,"kb_avail":92821260,"statfs":{"total":95995035648,"available":95048970240,"internally_reserved":0,"allocated":5943296,"data_stored":4146871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":940109710},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":4.6840000000000002,"15min":3.25},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.187},"max":{"1min":0.97199999999999998,"5min":139.79300000000001,"15min":139.79300000000001},"last":0.311},{"interface":"front","average":{"1min":0.373,"5min":4.641,"15min":2.9900000000000002},"min":{"1min":0.28000000000000003,"5min":0.22700000000000001,"15min":0.215},"max":{"1min":0.52800000000000002,"5min":139.75299999999999,"15min":139.75299999999999},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:53:14 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":6.6799999999999997,"15min":7.1719999999999997},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.222},"max":{"1min":0.82899999999999996,"5min":162.191,"15min":162.191},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.35499999999999998,"5min":6.4279999999999999,"15min":7.4050000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.51500000000000001,"5min":162.70500000000001,"15min":162.70500000000001},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":3.819,"15min":3.8079999999999998},"min":{"1min":0.31900000000000001,"5min":0.26100000000000001,"15min":0.23000000000000001},"max":{"1min":0.58299999999999996,"5min":160.82900000000001,"15min":160.82900000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":4.6440000000000001,"15min":4.6980000000000004},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.25600000000000001},"max":{"1min":0.59699999999999998,"5min":160.93100000000001,"15min":160.93100000000001},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Jun 10 21:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Jun 10 21:53:14 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.1140000000000001,"15min":1.9810000000000001},"min":{"1min":0.495,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.94299999999999995,"5min":8.1549999999999994,"15min":22.094000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":1.1879999999999999,"15min":1.806},"min":{"1min":0.52800000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.98699999999999999,"5min":8.3049999999999997,"15min":19.029},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.3879999999999999,"15min":1.597},"min":{"1min":0.54200000000000004,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.008,"5min":10.611000000000001,"15min":19.762},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.2190000000000001,"15min":1.534},"min":{"1min":0.50700000000000001,"5min":0.41699999999999998,"15min":0.40500000000000003},"max":{"1min":0.90900000000000003,"5min":8.2379999999999995,"15min":18.376999999999999},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:52:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.3600000000000001,"15min":2.1139999999999999},"min":{"1min":0.47199999999999998,"5min":0.42999999999999999,"15min":0.36399999999999999},"max":{"1min":0.96899999999999997,"5min":10.891999999999999,"15min":124.527},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.1879999999999999,"15min":2.0960000000000001},"min":{"1min":0.51700000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.88,"5min":7.8449999999999998,"15min":124.509},"last":0.627}]}]},{"osd":0,"up_from":244,"seq":1047972020523,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700000,"kb_used_data":6360,"kb_used_omap":10,"kb_used_meta":693621,"kb_avail":93045152,"statfs":{"total":95995035648,"available":95278235648,"internally_reserved":0,"allocated":6512640,"data_stored":4726019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":710267956},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.353,"15min":1.7350000000000001},"min":{"1min":0.26500000000000001,"5min":0.215,"15min":0.19800000000000001},"max":{"1min":0.51400000000000001,"5min":9.1500000000000004,"15min":27.759},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.36499999999999999,"5min":1.8400000000000001,"15min":2.3439999999999999},"min":{"1min":0.27000000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.53900000000000003,"5min":14.087,"15min":31.419},"last":0.437}]},{"osd":2,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":6.2400000000000002,"15min":6.8819999999999997},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.56699999999999995,"5min":156.31100000000001,"15min":156.31100000000001},"last":0.317},{"interface":"front","average":{"1min":0.40600000000000003,"5min":5.8310000000000004,"15min":6.3410000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":0.73999999999999999,"5min":155.92099999999999,"15min":155.92099999999999},"last":0.30499999999999999}]},{"osd":3,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.35599999999999998,"5min":4.6879999999999997,"15min":4.4870000000000001},"min":{"1min":0.27400000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.503,"5min":150.89699999999999,"15min":150.89699999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":4.7709999999999999,"15min":4.2939999999999996},"min":{"1min":0.33900000000000002,"5min":0.26500000000000001,"15min":0.25700000000000001},"max":{"1min":0.73199999999999998,"5min":150.97200000000001,"15min":150.97200000000001},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Jun 10 21:52:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.425,"15min":3.6219999999999999},"min":{"1min":0.434,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.92600000000000005,"5min":9.1899999999999995,"15min":92.149000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":1.357,"15min":4.0949999999999998},"min":{"1min":0.497,"5min":0.311,"15min":0.311},"max":{"1min":0.999,"5min":6.9969999999999999,"15min":93.292000000000002},"last":0.60899999999999999}]},{"osd":6,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":3.9430000000000001,"15min":3.0529999999999999},"min":{"1min":0.51200000000000001,"5min":0.35899999999999999,"15min":0.34300000000000003},"max":{"1min":1.0369999999999999,"5min":224.54599999999999,"15min":224.54599999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":3.839,"15min":2.774},"min":{"1min":0.46800000000000003,"5min":0.377,"15min":0.377},"max":{"1min":0.93999999999999995,"5min":224.56100000000001,"15min":224.56100000000001},"last":0.64900000000000002}]},{"osd":7,"last update":"Sat Jun 10 21:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":7.8869999999999996,"15min":5.0049999999999999},"min":{"1min":0.46000000000000002,"5min":0.371,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":227.565,"15min":227.565},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":8.1310000000000002,"15min":5.1989999999999998},"min":{"1min":0.50700000000000001,"5min":0.377,"15min":0.36299999999999999},"max":{"1min":0.94199999999999995,"5min":233.90100000000001,"15min":233.90100000000001},"last":0.54200000000000004}]}]},{"osd":2,"up_from":120,"seq":515396075934,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275896,"kb_used_data":5744,"kb_used_omap":7,"kb_used_meta":1270136,"kb_avail":92469256,"statfs":{"total":95995035648,"available":94688518144,"internally_reserved":0,"allocated":5881856,"data_stored":4135992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":1300619345},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":2.2309999999999999,"15min":3.4900000000000002},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.19700000000000001},"max":{"1min":0.53800000000000003,"5min":78.003,"15min":171.33699999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.9350000000000001,"15min":3.0299999999999998},"min":{"1min":0.29199999999999998,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.63800000000000001,"5min":77.950000000000003,"15min":171.56200000000001},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.33400000000000002,"5min":2.169,"15min":4.0030000000000001},"min":{"1min":0.26200000000000001,"5min":0.23599999999999999,"15min":0.217},"max":{"1min":0.44900000000000001,"5min":29.315999999999999,"15min":171.286},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.372,"5min":1.478,"15min":2.577},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.52000000000000002,"5min":15.685,"15min":85.501999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":7.2960000000000003,"15min":7.2210000000000001},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.22700000000000001},"max":{"1min":0.55500000000000005,"5min":187.751,"15min":187.751},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":6.9489999999999998,"15min":6.827},"min":{"1min":0.25900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56899999999999995,"5min":187.70099999999999,"15min":187.70099999999999},"last":0.432}]},{"osd":4,"last update":"Sat Jun 10 21:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Jun 10 21:53:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":4.1200000000000001,"15min":5.6600000000000001},"min":{"1min":0.47299999999999998,"5min":0.376,"15min":0.311},"max":{"1min":0.76700000000000002,"5min":130.864,"15min":157.756},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":3.6949999999999998,"15min":5.1879999999999997},"min":{"1min":0.47899999999999998,"5min":0.35899999999999999,"15min":0.32900000000000001},"max":{"1min":0.83899999999999997,"5min":131.06299999999999,"15min":157.59899999999999},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":1.339,"15min":2.1960000000000002},"min":{"1min":0.47699999999999998,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":0.81699999999999995,"5min":19.783999999999999,"15min":171.441},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.68100000000000005,"5min":2.2370000000000001,"15min":3.0499999999999998},"min":{"1min":0.46300000000000002,"5min":0.39100000000000001,"15min":0.34399999999999997},"max":{"1min":0.93700000000000006,"5min":20.393999999999998,"15min":171.24600000000001},"last":0.623}]},{"osd":7,"last update":"Sat Jun 10 21:53:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":3.1560000000000001,"15min":3.589},"min":{"1min":0.442,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":0.90300000000000002,"5min":135.86099999999999,"15min":171.26499999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.67900000000000005,"5min":2.7069999999999999,"15min":3.2269999999999999},"min":{"1min":0.50800000000000001,"5min":0.40999999999999998,"15min":0.32200000000000001},"max":{"1min":0.89500000000000002,"5min":133.45400000000001,"15min":171.52000000000001},"last":0.59499999999999997}]}]},{"osd":3,"up_from":250,"seq":1073741824294,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1043964,"kb_used_data":5940,"kb_used_omap":7,"kb_used_meta":1038008,"kb_avail":92701188,"statfs":{"total":95995035648,"available":94926016512,"internally_reserved":0,"allocated":6082560,"data_stored":4156553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":1062920211},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":4.3470000000000004,"15min":2.7120000000000002},"min":{"1min":0.223,"5min":0.21299999999999999,"15min":0.19800000000000001},"max":{"1min":11.474,"5min":236.97300000000001,"15min":236.97300000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.371,"5min":4.0819999999999999,"15min":2.9710000000000001},"min":{"1min":0.26000000000000001,"5min":0.219,"15min":0.20799999999999999},"max":{"1min":20.341999999999999,"5min":237.375,"15min":237.375},"last":0.249}]},{"osd":1,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":1.923,"15min":2.0179999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.7050000000000001,"5min":18.116,"15min":21.704999999999998},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":2.3530000000000002,"15min":2.2719999999999998},"min":{"1min":0.313,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.445,"5min":50.264000000000003,"15min":50.264000000000003},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":3.9489999999999998,"15min":4.2400000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.23300000000000001},"max":{"1min":10.065,"5min":117.279,"15min":231.07900000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":4.4420000000000002,"15min":4.7649999999999997},"min":{"1min":0.29799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63400000000000001,"5min":117.398,"15min":231.58600000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:52:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Jun 10 21:52:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":2.1200000000000001,"15min":1.4239999999999999},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.81499999999999995,"5min":50.399000000000001,"15min":50.399000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":2.3250000000000002,"15min":2.4329999999999998},"min":{"1min":0.41799999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":50.335000000000001,"15min":50.335000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":2.8149999999999999,"15min":2.2509999999999999},"min":{"1min":0.39900000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.673,"5min":62.098999999999997,"15min":62.098999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.2110000000000001,"5min":3.0649999999999999,"15min":2.028},"min":{"1min":0.48899999999999999,"5min":0.38900000000000001,"15min":0.34699999999999998},"max":{"1min":11.414999999999999,"5min":62.139000000000003,"15min":62.139000000000003},"last":0.70799999999999996}]},{"osd":7,"last update":"Sat Jun 10 21:52:47 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":3.032,"15min":2.6749999999999998},"min":{"1min":0.49199999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.6379999999999999,"5min":69.683000000000007,"15min":69.683000000000007},"last":0.622},{"interface":"front","average":{"1min":1.208,"5min":3.0470000000000002,"15min":2.359},"min":{"1min":0.48199999999999998,"5min":0.41699999999999998,"15min":0.316},"max":{"1min":11.384,"5min":69.998999999999995,"15min":69.998999999999995},"last":0.60799999999999998}]}]},{"osd":4,"up_from":507,"seq":2177548419097,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":58108,"kb_used_data":4724,"kb_used_omap":3,"kb_used_meta":53372,"kb_avail":93687044,"statfs":{"total":95995035648,"available":95935533056,"internally_reserved":0,"allocated":4837376,"data_stored":4020952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3428,"internal_metadata":54653596},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Jun 10 21:52:23 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.63200000000000001}]}]},{"osd":5,"up_from":390,"seq":1675037245566,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532080,"kb_used_data":4712,"kb_used_omap":11,"kb_used_meta":527348,"kb_avail":93213072,"statfs":{"total":95995035648,"available":95450185728,"internally_reserved":0,"allocated":4825088,"data_stored":4017961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":540004854},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":3.319,"15min":2.3809999999999998},"min":{"1min":0.32000000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.746,"5min":140.976,"15min":140.976},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":3.2669999999999999,"15min":2.0880000000000001},"min":{"1min":0.35499999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":0.69699999999999995,"5min":141.06899999999999,"15min":141.06899999999999},"last":0.69699999999999995}]},{"osd":1,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.1850000000000001,"15min":1.859},"min":{"1min":0.35399999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71799999999999997,"5min":12.013999999999999,"15min":12.013999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.0049999999999999,"15min":1.607},"min":{"1min":0.376,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77200000000000002,"5min":6.4109999999999996,"15min":9.7989999999999995},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":3.528,"15min":4.0860000000000003},"min":{"1min":0.40799999999999997,"5min":0.25700000000000001,"15min":0.254},"max":{"1min":0.76200000000000001,"5min":89.581999999999994,"15min":219.84700000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":3.109,"15min":4.0449999999999999},"min":{"1min":0.36299999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.71899999999999997,"5min":89.239999999999995,"15min":216.589},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":4.75,"15min":5.8710000000000004},"min":{"1min":0.374,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.77400000000000002,"5min":342.28399999999999,"15min":342.28399999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.64500000000000002,"5min":4.5199999999999996,"15min":5.7549999999999999},"min":{"1min":0.48099999999999998,"5min":0.34999999999999998,"15min":0.34200000000000003},"max":{"1min":0.89900000000000002,"5min":342.46600000000001,"15min":342.46600000000001},"last":0.73099999999999998}]},{"osd":4,"last update":"Sat Jun 10 21:52:28 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":2.0830000000000002,"15min":1.716},"min":{"1min":0.41899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.85299999999999998,"5min":108.693,"15min":108.693},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.8779999999999999,"15min":1.4490000000000001},"min":{"1min":0.36899999999999999,"5min":0.33000000000000002,"15min":0.32200000000000001},"max":{"1min":0.79700000000000004,"5min":108.708,"15min":108.708},"last":0.67100000000000004}]},{"osd":7,"last update":"Sat Jun 10 21:53:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":2.0150000000000001,"15min":1.6699999999999999},"min":{"1min":0.42199999999999999,"5min":0.28999999999999998,"15min":0.25},"max":{"1min":0.878,"5min":111.06999999999999,"15min":111.06999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":2.1930000000000001,"15min":2.4260000000000002},"min":{"1min":0.41499999999999998,"5min":0.31,"15min":0.31},"max":{"1min":0.83199999999999996,"5min":111.402,"15min":111.402},"last":0.747}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11212,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8173,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3428,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11786,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11203,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-10T21:53:22.430 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-10T21:53:22.430 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2023-06-10T21:53:22.987 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=07dcb2527e444659b577878b95475688 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_5937c68aed42cde57f938ad8f8bb264f13176009/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2023-06-10T21:53:22.991 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-06-10T21:53:23.004 INFO:tasks.cephadm:Teardown begin 2023-06-10T21:53:23.005 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-10T21:53:23.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:22 smithi152 ceph-mon[156819]: from='client.37806 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:23.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:22 smithi152 ceph-mon[156819]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:53:23.020 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:22 smithi152 ceph-mon[156819]: pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:23.021 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[180297]: from='client.37806 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:23.021 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[180297]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:53:23.021 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[180297]: pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:23.022 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[186706]: from='client.37806 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-10T21:53:23.022 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[186706]: from='mgr.34101 172.21.15.64:0/288019913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-10T21:53:23.022 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:22 smithi064 ceph-mon[186706]: pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-10T21:53:23.132 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-10T21:53:23.156 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:22 smithi064 conmon[171866]: ::ffff:172.21.15.152 - - [10/Jun/2023:21:53:22] "GET /metrics HTTP/1.1" 200 221738 "" "Prometheus/2.33.4" 2023-06-10T21:53:23.168 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-06-10T21:53:23.168 DEBUG:teuthology.orchestra.run.smithi064:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-06-10T21:53:23.185 DEBUG:teuthology.orchestra.run.smithi152:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-06-10T21:53:23.226 INFO:tasks.cephadm:Stopping all daemons... 2023-06-10T21:53:23.226 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-06-10T21:53:23.227 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a 2023-06-10T21:53:23.595 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:23 smithi064 systemd[1]: Stopping Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:23.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE Bus STOPPING 2023-06-10T21:53:23.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-10T21:53:23.907 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE Bus STOPPED 2023-06-10T21:53:23.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE Bus STARTING 2023-06-10T21:53:23.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE Serving on http://:::9283 2023-06-10T21:53:23.908 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[171866]: [10/Jun/2023:21:53:23] ENGINE Bus STARTED 2023-06-10T21:53:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[180272]: 2023-06-10T21:53:23.591+0000 7f0f6d2bf700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:23.909 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:23 smithi064 conmon[180272]: 2023-06-10T21:53:23.591+0000 7f0f6d2bf700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-06-10T21:53:24.764 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service' 2023-06-10T21:53:24.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:24 smithi064 bash[647604]: 6f8d9991dea01bfb33585ce34ac946675958e6d11fc02020bacb43172552ba19 2023-06-10T21:53:24.791 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:24 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.a.service: Succeeded. 2023-06-10T21:53:24.792 INFO:journalctl@ceph.mon.a.smithi064.stdout:Jun 10 21:53:24 smithi064 systemd[1]: Stopped Ceph mon.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:53:24.818 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:24.819 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-06-10T21:53:24.819 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-06-10T21:53:24.819 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.c 2023-06-10T21:53:25.151 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:24 smithi064 systemd[1]: Stopping Ceph mon.c for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:25.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:25 smithi064 conmon[186683]: 2023-06-10T21:53:25.146+0000 7f5fdd850700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:25.407 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:25 smithi064 conmon[186683]: 2023-06-10T21:53:25.146+0000 7f5fdd850700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-06-10T21:53:25.907 INFO:journalctl@ceph.mon.c.smithi064.stdout:Jun 10 21:53:25 smithi064 bash[648252]: 925081a4753eab527f5566e5d35df0ea73454b8eefac3d3468919ea54a89f994 2023-06-10T21:53:26.264 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.c.service' 2023-06-10T21:53:26.311 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:26.311 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-06-10T21:53:26.312 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-06-10T21:53:26.312 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.b 2023-06-10T21:53:26.693 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:26 smithi152 systemd[1]: Stopping Ceph mon.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:26 smithi152 conmon[156795]: 2023-06-10T21:53:26.584+0000 7f7e4daa4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:26.694 INFO:journalctl@ceph.mon.b.smithi152.stdout:Jun 10 21:53:26 smithi152 conmon[156795]: 2023-06-10T21:53:26.584+0000 7f7e4daa4700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-06-10T21:53:27.662 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mon.b.service' 2023-06-10T21:53:27.703 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:27.703 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-06-10T21:53:27.704 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-06-10T21:53:27.704 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y 2023-06-10T21:53:27.994 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:27 smithi064 systemd[1]: Stopping Ceph mgr.y for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:28.311 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:27 smithi064 conmon[171866]: 2023-06-10T21:53:27.990+0000 7f6d3719d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:28.311 INFO:journalctl@ceph.mgr.y.smithi064.stdout:Jun 10 21:53:28 smithi064 bash[648782]: 2bda388cec1de0737fc16b6e6df8e21be326d4fe8918bfeca94862c45972f86a 2023-06-10T21:53:28.830 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.y.service' 2023-06-10T21:53:28.870 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:28.871 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-06-10T21:53:28.871 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-06-10T21:53:28.872 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x 2023-06-10T21:53:29.193 INFO:journalctl@ceph.mgr.x.smithi152.stdout:Jun 10 21:53:28 smithi152 systemd[1]: Stopping Ceph mgr.x for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:29.930 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@mgr.x.service' 2023-06-10T21:53:29.969 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:29.969 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-06-10T21:53:29.970 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-06-10T21:53:29.970 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0 2023-06-10T21:53:30.258 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:53:30 smithi064 systemd[1]: Stopping Ceph osd.0 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:30.656 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:53:30 smithi064 conmon[348287]: 2023-06-10T21:53:30.254+0000 7f84a529f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:30.657 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:53:30 smithi064 conmon[348287]: 2023-06-10T21:53:30.254+0000 7f84a529f700 -1 osd.0 538 *** Got signal Terminated *** 2023-06-10T21:53:30.657 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:53:30 smithi064 conmon[348287]: 2023-06-10T21:53:30.254+0000 7f84a529f700 -1 osd.0 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:53:36.407 INFO:journalctl@ceph.osd.0.smithi064.stdout:Jun 10 21:53:36 smithi064 bash[649086]: 8f85d9362d65f1adcef68ddeb88d835f35595eb1ddb5c97bc5612409b184281b 2023-06-10T21:53:38.718 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.0.service' 2023-06-10T21:53:38.761 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:38.762 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-06-10T21:53:38.762 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-06-10T21:53:38.762 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.1 2023-06-10T21:53:39.085 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:53:38 smithi064 systemd[1]: Stopping Ceph osd.1 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:39.406 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:53:39 smithi064 conmon[211518]: 2023-06-10T21:53:39.083+0000 7f841b425700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:39.407 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:53:39 smithi064 conmon[211518]: 2023-06-10T21:53:39.083+0000 7f841b425700 -1 osd.1 538 *** Got signal Terminated *** 2023-06-10T21:53:39.407 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:53:39 smithi064 conmon[211518]: 2023-06-10T21:53:39.083+0000 7f841b425700 -1 osd.1 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:53:45.407 INFO:journalctl@ceph.osd.1.smithi064.stdout:Jun 10 21:53:45 smithi064 bash[649675]: b6d15648d91ceceda6d37fea5d47f956729c2fed4fb584bb94e9f000770a37f1 2023-06-10T21:53:46.882 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.1.service' 2023-06-10T21:53:46.924 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:46.925 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-06-10T21:53:46.925 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-06-10T21:53:46.925 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.2 2023-06-10T21:53:47.156 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:53:46 smithi064 systemd[1]: Stopping Ceph osd.2 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:47.657 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:53:47 smithi064 conmon[222603]: 2023-06-10T21:53:47.227+0000 7f425140e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:47.680 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:53:47 smithi064 conmon[222603]: 2023-06-10T21:53:47.227+0000 7f425140e700 -1 osd.2 538 *** Got signal Terminated *** 2023-06-10T21:53:47.680 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:53:47 smithi064 conmon[222603]: 2023-06-10T21:53:47.227+0000 7f425140e700 -1 osd.2 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:53:50.693 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:53:50 smithi152 conmon[146558]: ts=2023-06-10T21:53:50.340Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-06-10T21:53:50.693 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:53:50 smithi152 conmon[146558]: ts=2023-06-10T21:53:50.342Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-06-10T21:53:50.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:53:50 smithi152 conmon[146558]: ts=2023-06-10T21:53:50.344Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-06-10T21:53:50.694 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:53:50 smithi152 conmon[146558]: ts=2023-06-10T21:53:50.344Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.64:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.64:8765: connect: connection refused" 2023-06-10T21:53:53.407 INFO:journalctl@ceph.osd.2.smithi064.stdout:Jun 10 21:53:52 smithi064 bash[650247]: c76c1a3f425a8093aec1d9e92b5a22b4588c9706274e7349fc784248f1715a86 2023-06-10T21:53:55.031 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.2.service' 2023-06-10T21:53:55.073 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:53:55.074 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-06-10T21:53:55.074 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-06-10T21:53:55.074 DEBUG:teuthology.orchestra.run.smithi064:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3 2023-06-10T21:53:55.390 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:53:55 smithi064 systemd[1]: Stopping Ceph osd.3 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:53:55.657 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:53:55 smithi064 conmon[354039]: 2023-06-10T21:53:55.388+0000 7f90b56b3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-06-10T21:53:55.657 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:53:55 smithi064 conmon[354039]: 2023-06-10T21:53:55.388+0000 7f90b56b3700 -1 osd.3 538 *** Got signal Terminated *** 2023-06-10T21:53:55.657 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:53:55 smithi064 conmon[354039]: 2023-06-10T21:53:55.388+0000 7f90b56b3700 -1 osd.3 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:53:58.567 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:53:58 smithi152 conmon[316210]: debug 2023-06-10T21:53:58.199+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:53:58.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:53:58 smithi152 conmon[374343]: debug 2023-06-10T21:53:58.564+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:53:59.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:53:59 smithi152 conmon[316210]: debug 2023-06-10T21:53:59.180+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:53:59.657 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:53:59 smithi064 conmon[354039]: 2023-06-10T21:53:59.351+0000 7f90b1ca9700 -1 osd.3 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:31.236272+0000 front 2023-06-10T21:53:31.236439+0000 (oldest deadline 2023-06-10T21:53:58.837092+0000) 2023-06-10T21:53:59.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:53:59 smithi152 conmon[374343]: debug 2023-06-10T21:53:59.568+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:00.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:00 smithi152 conmon[316210]: debug 2023-06-10T21:54:00.160+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:00.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:00 smithi152 conmon[115220]: debug 2023-06-10T21:54:00.242+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:00.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:00 smithi152 conmon[121150]: debug 2023-06-10T21:54:00.224+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:00.657 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:54:00 smithi064 conmon[354039]: 2023-06-10T21:54:00.334+0000 7f90b1ca9700 -1 osd.3 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:31.236272+0000 front 2023-06-10T21:53:31.236439+0000 (oldest deadline 2023-06-10T21:53:58.837092+0000) 2023-06-10T21:54:00.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:00 smithi152 conmon[374343]: debug 2023-06-10T21:54:00.544+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:01.407 INFO:journalctl@ceph.osd.3.smithi064.stdout:Jun 10 21:54:00 smithi064 bash[650828]: 9f2a2ef8b19107f131c42a2b55c2206ff224cee7b187310f70fd69c01fa7368f 2023-06-10T21:54:01.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:01 smithi152 conmon[316210]: debug 2023-06-10T21:54:01.181+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:01.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:01 smithi152 conmon[115220]: debug 2023-06-10T21:54:01.269+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:01.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:01 smithi152 conmon[121150]: debug 2023-06-10T21:54:01.262+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:01.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:01 smithi152 conmon[374343]: debug 2023-06-10T21:54:01.592+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:02.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:02 smithi152 conmon[316210]: debug 2023-06-10T21:54:02.172+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:02.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:02 smithi152 conmon[115220]: debug 2023-06-10T21:54:02.219+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:02.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:02 smithi152 conmon[121150]: debug 2023-06-10T21:54:02.218+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:02.889 DEBUG:teuthology.orchestra.run.smithi064:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.3.service' 2023-06-10T21:54:02.931 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:02.931 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-06-10T21:54:02.932 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-06-10T21:54:02.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4 2023-06-10T21:54:02.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:02 smithi152 conmon[374343]: debug 2023-06-10T21:54:02.566+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:03.224 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[316210]: debug 2023-06-10T21:54:03.150+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:03.225 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[115220]: debug 2023-06-10T21:54:03.194+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:03.225 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[121150]: debug 2023-06-10T21:54:03.199+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:03.226 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:02 smithi152 systemd[1]: Stopping Ceph osd.4 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:03.551 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[374343]: debug 2023-06-10T21:54:03.221+0000 7f95365dd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:54:03.551 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[374343]: debug 2023-06-10T21:54:03.221+0000 7f95365dd700 -1 osd.4 538 *** Got signal Terminated *** 2023-06-10T21:54:03.552 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[374343]: debug 2023-06-10T21:54:03.221+0000 7f95365dd700 -1 osd.4 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:54:03.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:03 smithi152 conmon[374343]: debug 2023-06-10T21:54:03.549+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:04.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:04 smithi152 conmon[316210]: debug 2023-06-10T21:54:04.124+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:04.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:04 smithi152 conmon[115220]: debug 2023-06-10T21:54:04.188+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:04.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:04 smithi152 conmon[121150]: debug 2023-06-10T21:54:04.241+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:04.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:04 smithi152 conmon[374343]: debug 2023-06-10T21:54:04.524+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:05.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:05 smithi152 conmon[316210]: debug 2023-06-10T21:54:05.086+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:05.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:05 smithi152 conmon[115220]: debug 2023-06-10T21:54:05.189+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:05.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:05 smithi152 conmon[121150]: debug 2023-06-10T21:54:05.233+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:05.445 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:05 smithi152 conmon[121150]: debug 2023-06-10T21:54:05.233+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:05.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:05 smithi152 conmon[374343]: debug 2023-06-10T21:54:05.522+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:06.443 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[316210]: debug 2023-06-10T21:54:06.046+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:06.444 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[316210]: debug 2023-06-10T21:54:06.046+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:06.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[121150]: debug 2023-06-10T21:54:06.256+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:06.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[121150]: debug 2023-06-10T21:54:06.256+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:06.445 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[115220]: debug 2023-06-10T21:54:06.168+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:06.943 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:06 smithi152 conmon[374343]: debug 2023-06-10T21:54:06.512+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:07.277 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[115220]: debug 2023-06-10T21:54:07.180+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:07.278 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[316210]: debug 2023-06-10T21:54:07.021+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:07.279 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[316210]: debug 2023-06-10T21:54:07.021+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:07.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[121150]: debug 2023-06-10T21:54:07.275+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:07.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[121150]: debug 2023-06-10T21:54:07.275+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:07.694 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:07 smithi152 conmon[374343]: debug 2023-06-10T21:54:07.474+0000 7f9531557700 -1 osd.4 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.769026+0000 front 2023-06-10T21:53:34.769039+0000 (oldest deadline 2023-06-10T21:53:57.669095+0000) 2023-06-10T21:54:08.426 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:08 smithi152 conmon[316210]: debug 2023-06-10T21:54:08.060+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:08.426 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:08 smithi152 conmon[316210]: debug 2023-06-10T21:54:08.060+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:08.427 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:08 smithi152 conmon[115220]: debug 2023-06-10T21:54:08.161+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:08.427 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:08 smithi152 conmon[121150]: debug 2023-06-10T21:54:08.273+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:08.427 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:08 smithi152 conmon[121150]: debug 2023-06-10T21:54:08.273+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:08.694 INFO:journalctl@ceph.osd.4.smithi152.stdout:Jun 10 21:54:08 smithi152 bash[382920]: ffc3bc582f6d2b076d17008e2757a4724c7c04f33ef9b4b6caabb9e373a72514 2023-06-10T21:54:09.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[316210]: debug 2023-06-10T21:54:09.019+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:09.351 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[316210]: debug 2023-06-10T21:54:09.019+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:09.352 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[115220]: debug 2023-06-10T21:54:09.174+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:09.352 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[115220]: debug 2023-06-10T21:54:09.174+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:09.353 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[121150]: debug 2023-06-10T21:54:09.234+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:09.353 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:09 smithi152 conmon[121150]: debug 2023-06-10T21:54:09.234+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:10.166 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[316210]: debug 2023-06-10T21:54:10.025+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:10.166 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[316210]: debug 2023-06-10T21:54:10.025+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:10.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[115220]: debug 2023-06-10T21:54:10.163+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:10.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[115220]: debug 2023-06-10T21:54:10.163+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:10.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[121150]: debug 2023-06-10T21:54:10.272+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:10.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[121150]: debug 2023-06-10T21:54:10.272+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:10.455 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.4.service' 2023-06-10T21:54:10.536 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:10.537 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-06-10T21:54:10.537 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-06-10T21:54:10.538 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5 2023-06-10T21:54:10.839 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 systemd[1]: Stopping Ceph osd.5 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:10.840 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[316210]: debug 2023-06-10T21:54:10.836+0000 7ff6a0c40700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:54:10.840 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[316210]: debug 2023-06-10T21:54:10.836+0000 7ff6a0c40700 -1 osd.5 538 *** Got signal Terminated *** 2023-06-10T21:54:11.124 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:10 smithi152 conmon[316210]: debug 2023-06-10T21:54:10.836+0000 7ff6a0c40700 -1 osd.5 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:54:11.125 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[316210]: debug 2023-06-10T21:54:11.043+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:11.125 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[316210]: debug 2023-06-10T21:54:11.043+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:11.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[115220]: debug 2023-06-10T21:54:11.122+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:11.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[115220]: debug 2023-06-10T21:54:11.122+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:11.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[121150]: debug 2023-06-10T21:54:11.273+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:11.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:11 smithi152 conmon[121150]: debug 2023-06-10T21:54:11.273+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:12.320 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[316210]: debug 2023-06-10T21:54:12.006+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:12.320 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[316210]: debug 2023-06-10T21:54:12.006+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:12.321 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[115220]: debug 2023-06-10T21:54:12.123+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:12.321 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[115220]: debug 2023-06-10T21:54:12.123+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:12.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[121150]: debug 2023-06-10T21:54:12.317+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:12.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:12 smithi152 conmon[121150]: debug 2023-06-10T21:54:12.317+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:13.336 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[316210]: debug 2023-06-10T21:54:13.044+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:13.337 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[316210]: debug 2023-06-10T21:54:13.044+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:13.337 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[115220]: debug 2023-06-10T21:54:13.147+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:13.337 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[115220]: debug 2023-06-10T21:54:13.147+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:13.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[121150]: debug 2023-06-10T21:54:13.334+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:13.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:13 smithi152 conmon[121150]: debug 2023-06-10T21:54:13.334+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:14.339 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[316210]: debug 2023-06-10T21:54:14.062+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:14.339 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[316210]: debug 2023-06-10T21:54:14.062+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:14.340 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[115220]: debug 2023-06-10T21:54:14.114+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:14.340 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[115220]: debug 2023-06-10T21:54:14.114+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:14.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[121150]: debug 2023-06-10T21:54:14.337+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:14.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:14 smithi152 conmon[121150]: debug 2023-06-10T21:54:14.337+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:15.364 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[316210]: debug 2023-06-10T21:54:15.109+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:30.119117+0000 front 2023-06-10T21:53:30.119094+0000 (oldest deadline 2023-06-10T21:53:57.719680+0000) 2023-06-10T21:54:15.364 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[316210]: debug 2023-06-10T21:54:15.109+0000 7ff69e3bf700 -1 osd.5 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:41.220889+0000 front 2023-06-10T21:53:41.220989+0000 (oldest deadline 2023-06-10T21:54:05.920753+0000) 2023-06-10T21:54:15.365 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[115220]: debug 2023-06-10T21:54:15.128+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:15.365 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[115220]: debug 2023-06-10T21:54:15.128+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:15.365 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[115220]: debug 2023-06-10T21:54:15.128+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:15.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[121150]: debug 2023-06-10T21:54:15.362+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:15.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[121150]: debug 2023-06-10T21:54:15.362+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:15.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:15 smithi152 conmon[121150]: debug 2023-06-10T21:54:15.362+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:16.377 INFO:journalctl@ceph.osd.5.smithi152.stdout:Jun 10 21:54:16 smithi152 bash[383505]: c1f73cf499b97d49f4b54499cba9bdef02f80f918d9c8d8aa337b172e42cef78 2023-06-10T21:54:16.378 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[115220]: debug 2023-06-10T21:54:16.099+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:16.379 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[115220]: debug 2023-06-10T21:54:16.100+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:16.379 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[115220]: debug 2023-06-10T21:54:16.100+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:16.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[121150]: debug 2023-06-10T21:54:16.374+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:16.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[121150]: debug 2023-06-10T21:54:16.374+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:16.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:16 smithi152 conmon[121150]: debug 2023-06-10T21:54:16.374+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:17.401 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[115220]: debug 2023-06-10T21:54:17.113+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:17.401 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[115220]: debug 2023-06-10T21:54:17.113+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:17.402 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[115220]: debug 2023-06-10T21:54:17.113+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:17.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[121150]: debug 2023-06-10T21:54:17.398+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:17.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[121150]: debug 2023-06-10T21:54:17.398+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:17.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:17 smithi152 conmon[121150]: debug 2023-06-10T21:54:17.398+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:18.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.5.service' 2023-06-10T21:54:18.087 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:18.087 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-06-10T21:54:18.088 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-06-10T21:54:18.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.6 2023-06-10T21:54:18.425 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.146+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:18.426 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.146+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:18.426 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.146+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:18.426 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 systemd[1]: Stopping Ceph osd.6 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:18.426 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.422+0000 7f213cc77700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:54:18.427 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.422+0000 7f213cc77700 -1 osd.6 538 *** Got signal Terminated *** 2023-06-10T21:54:18.693 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[115220]: debug 2023-06-10T21:54:18.422+0000 7f213cc77700 -1 osd.6 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:54:18.693 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[121150]: debug 2023-06-10T21:54:18.423+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:18.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[121150]: debug 2023-06-10T21:54:18.423+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:18.694 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:18 smithi152 conmon[121150]: debug 2023-06-10T21:54:18.423+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:19.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[115220]: debug 2023-06-10T21:54:19.155+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:19.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[115220]: debug 2023-06-10T21:54:19.155+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:19.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[115220]: debug 2023-06-10T21:54:19.155+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:19.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[121150]: debug 2023-06-10T21:54:19.473+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:19.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[121150]: debug 2023-06-10T21:54:19.473+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:19.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:19 smithi152 conmon[121150]: debug 2023-06-10T21:54:19.473+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:20.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[115220]: debug 2023-06-10T21:54:20.151+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:20.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[115220]: debug 2023-06-10T21:54:20.151+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:20.444 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[115220]: debug 2023-06-10T21:54:20.151+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:20.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[121150]: debug 2023-06-10T21:54:20.499+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:20.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[121150]: debug 2023-06-10T21:54:20.499+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:20.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:20 smithi152 conmon[121150]: debug 2023-06-10T21:54:20.499+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:21.487 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[115220]: debug 2023-06-10T21:54:21.195+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:21.488 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[115220]: debug 2023-06-10T21:54:21.195+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:21.488 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[115220]: debug 2023-06-10T21:54:21.195+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:21.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[121150]: debug 2023-06-10T21:54:21.485+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:21.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[121150]: debug 2023-06-10T21:54:21.485+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:21.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:21 smithi152 conmon[121150]: debug 2023-06-10T21:54:21.485+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:22.516 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[115220]: debug 2023-06-10T21:54:22.228+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:22.517 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[115220]: debug 2023-06-10T21:54:22.228+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:22.517 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[115220]: debug 2023-06-10T21:54:22.228+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:22.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[121150]: debug 2023-06-10T21:54:22.514+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:22.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[121150]: debug 2023-06-10T21:54:22.514+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:22.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:22 smithi152 conmon[121150]: debug 2023-06-10T21:54:22.514+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:23.535 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[115220]: debug 2023-06-10T21:54:23.277+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:34.583441+0000 front 2023-06-10T21:53:34.583526+0000 (oldest deadline 2023-06-10T21:53:59.283445+0000) 2023-06-10T21:54:23.535 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[115220]: debug 2023-06-10T21:54:23.277+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:43.185872+0000 front 2023-06-10T21:53:43.185773+0000 (oldest deadline 2023-06-10T21:54:09.085810+0000) 2023-06-10T21:54:23.535 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[115220]: debug 2023-06-10T21:54:23.277+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:51.887401+0000 front 2023-06-10T21:53:51.887334+0000 (oldest deadline 2023-06-10T21:54:14.787280+0000) 2023-06-10T21:54:23.535 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[115220]: debug 2023-06-10T21:54:23.277+0000 7f2139bf5700 -1 osd.6 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.189573+0000 front 2023-06-10T21:53:59.189622+0000 (oldest deadline 2023-06-10T21:54:22.689497+0000) 2023-06-10T21:54:23.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[121150]: debug 2023-06-10T21:54:23.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:23.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[121150]: debug 2023-06-10T21:54:23.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:23.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:23 smithi152 conmon[121150]: debug 2023-06-10T21:54:23.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:24.443 INFO:journalctl@ceph.osd.6.smithi152.stdout:Jun 10 21:54:24 smithi152 bash[384086]: 6f9176fc7b52e483bbd7e7bd12d039c4634d7933be47205fa42eba7a7b80a019 2023-06-10T21:54:24.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:24 smithi152 conmon[121150]: debug 2023-06-10T21:54:24.522+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:24.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:24 smithi152 conmon[121150]: debug 2023-06-10T21:54:24.522+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:24.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:24 smithi152 conmon[121150]: debug 2023-06-10T21:54:24.522+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:24.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:24 smithi152 conmon[121150]: debug 2023-06-10T21:54:24.522+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:25.652 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.6.service' 2023-06-10T21:54:25.692 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:25.692 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-06-10T21:54:25.693 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-06-10T21:54:25.693 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.7 2023-06-10T21:54:25.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:25.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:25.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:25.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.532+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:25.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 systemd[1]: Stopping Ceph osd.7 for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:26.443 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.992+0000 7f947a465700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-10T21:54:26.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.992+0000 7f947a465700 -1 osd.7 538 *** Got signal Terminated *** 2023-06-10T21:54:26.444 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:25 smithi152 conmon[121150]: debug 2023-06-10T21:54:25.992+0000 7f947a465700 -1 osd.7 538 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-10T21:54:26.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:26 smithi152 conmon[121150]: debug 2023-06-10T21:54:26.547+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:26.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:26 smithi152 conmon[121150]: debug 2023-06-10T21:54:26.547+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:26.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:26 smithi152 conmon[121150]: debug 2023-06-10T21:54:26.547+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:26.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:26 smithi152 conmon[121150]: debug 2023-06-10T21:54:26.547+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:27.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:27 smithi152 conmon[121150]: debug 2023-06-10T21:54:27.531+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:27.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:27 smithi152 conmon[121150]: debug 2023-06-10T21:54:27.531+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:27.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:27 smithi152 conmon[121150]: debug 2023-06-10T21:54:27.531+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:27.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:27 smithi152 conmon[121150]: debug 2023-06-10T21:54:27.531+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:28.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:28 smithi152 conmon[121150]: debug 2023-06-10T21:54:28.506+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:28.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:28 smithi152 conmon[121150]: debug 2023-06-10T21:54:28.506+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:28.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:28 smithi152 conmon[121150]: debug 2023-06-10T21:54:28.506+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:28.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:28 smithi152 conmon[121150]: debug 2023-06-10T21:54:28.506+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:29.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:29 smithi152 conmon[121150]: debug 2023-06-10T21:54:29.502+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:29.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:29 smithi152 conmon[121150]: debug 2023-06-10T21:54:29.502+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:29.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:29 smithi152 conmon[121150]: debug 2023-06-10T21:54:29.502+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:29.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:29 smithi152 conmon[121150]: debug 2023-06-10T21:54:29.502+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:30.943 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:30 smithi152 conmon[121150]: debug 2023-06-10T21:54:30.464+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6806 osd.0 since back 2023-06-10T21:53:35.169402+0000 front 2023-06-10T21:53:35.169450+0000 (oldest deadline 2023-06-10T21:53:59.269219+0000) 2023-06-10T21:54:30.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:30 smithi152 conmon[121150]: debug 2023-06-10T21:54:30.464+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6814 osd.1 since back 2023-06-10T21:53:39.269789+0000 front 2023-06-10T21:53:39.269897+0000 (oldest deadline 2023-06-10T21:54:04.569762+0000) 2023-06-10T21:54:30.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:30 smithi152 conmon[121150]: debug 2023-06-10T21:54:30.464+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6822 osd.2 since back 2023-06-10T21:53:49.171656+0000 front 2023-06-10T21:53:49.171750+0000 (oldest deadline 2023-06-10T21:54:15.071586+0000) 2023-06-10T21:54:30.944 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:30 smithi152 conmon[121150]: debug 2023-06-10T21:54:30.464+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.64:6830 osd.3 since back 2023-06-10T21:53:59.772708+0000 front 2023-06-10T21:53:59.772676+0000 (oldest deadline 2023-06-10T21:54:23.872669+0000) 2023-06-10T21:54:30.945 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:30 smithi152 conmon[121150]: debug 2023-06-10T21:54:30.464+0000 7f94753df700 -1 osd.7 538 heartbeat_check: no reply from 172.21.15.152:6804 osd.4 since back 2023-06-10T21:54:04.973688+0000 front 2023-06-10T21:54:04.973631+0000 (oldest deadline 2023-06-10T21:54:30.273619+0000) 2023-06-10T21:54:32.193 INFO:journalctl@ceph.osd.7.smithi152.stdout:Jun 10 21:54:31 smithi152 bash[384660]: 746300ca8a9183d144effbee0fe105f1d88f19abe23bfda625bb604987b926b0 2023-06-10T21:54:33.134 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@osd.7.service' 2023-06-10T21:54:33.174 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:33.175 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-06-10T21:54:33.175 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-06-10T21:54:33.175 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a 2023-06-10T21:54:33.443 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 systemd[1]: Stopping Ceph prometheus.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:33.943 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:54:33.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-10T21:54:33.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-10T21:54:33.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-10T21:54:33.944 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-10T21:54:33.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-10T21:54:33.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-10T21:54:33.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.492Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-10T21:54:33.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.493Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-10T21:54:33.945 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.497Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-10T21:54:33.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.497Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-10T21:54:33.946 INFO:journalctl@ceph.prometheus.a.smithi152.stdout:Jun 10 21:54:33 smithi152 conmon[146558]: ts=2023-06-10T21:54:33.497Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-10T21:54:34.156 DEBUG:teuthology.orchestra.run.smithi152:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@prometheus.a.service' 2023-06-10T21:54:34.193 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-10T21:54:34.193 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-06-10T21:54:34.194 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b618768e-07d0-11ee-9b1e-001a4aab830c --force --keep-logs 2023-06-10T21:54:35.657 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:54:35 smithi064 systemd[1]: Stopping Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:36.530 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:54:36 smithi064 bash[651716]: c7179c864e62f6a284404f0d439e2dd338b8ac2c74f4c122a3725dca11c7dc10 2023-06-10T21:54:36.530 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:54:36 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:54:36.531 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:54:36 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.a.service: Failed with result 'exit-code'. 2023-06-10T21:54:36.531 INFO:journalctl@ceph.node-exporter.a.smithi064.stdout:Jun 10 21:54:36 smithi064 systemd[1]: Stopped Ceph node-exporter.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:54:36.906 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:54:36 smithi064 systemd[1]: Stopping Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:36.907 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:54:36 smithi064 conmon[170068]: level=info ts=2023-06-10T21:54:36.753Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-10T21:54:37.406 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:54:36 smithi064 bash[651999]: 13fd7844ce612dcb5667df3fcfb587e62e7875ea5981408350c3fa7f477c606c 2023-06-10T21:54:37.890 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:54:37 smithi064 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-10T21:54:37.890 INFO:journalctl@ceph.alertmanager.a.smithi064.stdout:Jun 10 21:54:37 smithi064 systemd[1]: Stopped Ceph alertmanager.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:54:45.680 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b618768e-07d0-11ee-9b1e-001a4aab830c --force --keep-logs 2023-06-10T21:54:47.193 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:54:46 smithi152 systemd[1]: Stopping Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:47.916 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:54:47 smithi152 bash[385836]: 91ff512a05b1ce3d4218a5678d8c3eff0b41087521eb9896fbf2000792223583 2023-06-10T21:54:47.917 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:54:47 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-06-10T21:54:47.917 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:54:47 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@node-exporter.b.service: Failed with result 'exit-code'. 2023-06-10T21:54:47.917 INFO:journalctl@ceph.node-exporter.b.smithi152.stdout:Jun 10 21:54:47 smithi152 systemd[1]: Stopped Ceph node-exporter.b for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:54:48.168 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:54:47 smithi152 systemd[1]: Stopping Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c... 2023-06-10T21:54:48.443 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:54:48 smithi152 conmon[142072]: t=2023-06-10T21:54:48+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-06-10T21:54:48.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:54:48 smithi152 bash[386116]: 95a95736b9e95c3172311e8e28d208b26344dff7fc50ed69d3b805c48804e074 2023-06-10T21:54:48.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:54:48 smithi152 systemd[1]: ceph-b618768e-07d0-11ee-9b1e-001a4aab830c@grafana.a.service: Succeeded. 2023-06-10T21:54:48.990 INFO:journalctl@ceph.grafana.a.smithi152.stdout:Jun 10 21:54:48 smithi152 systemd[1]: Stopped Ceph grafana.a for b618768e-07d0-11ee-9b1e-001a4aab830c. 2023-06-10T21:54:53.576 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-10T21:54:53.627 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-10T21:54:53.657 INFO:tasks.cephadm:Archiving crash dumps... 2023-06-10T21:54:53.658 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi064/crash 2023-06-10T21:54:53.659 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash -- . 2023-06-10T21:54:53.698 INFO:teuthology.orchestra.run.smithi064.stderr:tar: /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash: Cannot open: No such file or directory 2023-06-10T21:54:53.698 INFO:teuthology.orchestra.run.smithi064.stderr:tar: Error is not recoverable: exiting now 2023-06-10T21:54:53.700 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi152/crash 2023-06-10T21:54:53.701 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash -- . 2023-06-10T21:54:53.755 INFO:teuthology.orchestra.run.smithi152.stderr:tar: /var/lib/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/crash: Cannot open: No such file or directory 2023-06-10T21:54:53.755 INFO:teuthology.orchestra.run.smithi152.stderr:tar: Error is not recoverable: exiting now 2023-06-10T21:54:53.756 INFO:tasks.cephadm:Checking cluster log for badness... 2023-06-10T21:54:53.756 DEBUG:teuthology.orchestra.run.smithi064:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-06-10T21:54:53.789 INFO:teuthology.orchestra.run.smithi064.stderr:grep: /var/log/ceph/b618768e-07d0-11ee-9b1e-001a4aab830c/ceph.log: No such file or directory 2023-06-10T21:54:53.791 INFO:tasks.cephadm:Compressing logs... 2023-06-10T21:54:53.791 DEBUG:teuthology.orchestra.run.smithi064:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-10T21:54:53.833 DEBUG:teuthology.orchestra.run.smithi152:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-10T21:54:53.870 INFO:teuthology.orchestra.run.smithi064.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-06-10T21:54:53.936 INFO:teuthology.orchestra.run.smithi152.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-06-10T22:10:28.286 INFO:tasks.cephadm:Archiving logs... 2023-06-10T22:10:28.314 DEBUG:teuthology.misc:Transferring archived files from smithi064:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi064/log 2023-06-10T22:10:28.316 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /var/log/ceph -- . 2023-06-10T22:11:53.909 DEBUG:teuthology.misc:Transferring archived files from smithi152:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi152/log 2023-06-10T22:11:53.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /var/log/ceph -- . 2023-06-10T22:12:35.654 INFO:tasks.cephadm:Removing cluster... 2023-06-10T22:12:35.655 DEBUG:teuthology.orchestra.run.smithi064:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b618768e-07d0-11ee-9b1e-001a4aab830c --force 2023-06-10T22:12:37.266 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b618768e-07d0-11ee-9b1e-001a4aab830c --force 2023-06-10T22:12:38.456 INFO:tasks.cephadm:Removing cephadm ... 2023-06-10T22:12:38.456 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf /home/ubuntu/cephtest/cephadm 2023-06-10T22:12:38.479 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/cephadm 2023-06-10T22:12:38.508 INFO:tasks.cephadm:Teardown complete 2023-06-10T22:12:38.509 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-06-10T22:12:38.519 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-06-10T22:12:38.519 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-06-10T22:12:38.522 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-06-10T22:12:38.643 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-06-10T22:12:38.653 INFO:teuthology.task.clock:Checking final clock skew... 2023-06-10T22:12:38.654 DEBUG:teuthology.orchestra.run.smithi064:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-10T22:12:38.656 DEBUG:teuthology.orchestra.run.smithi152:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-10T22:12:38.672 INFO:teuthology.orchestra.run.smithi152.stderr:bash: ntpq: command not found 2023-06-10T22:12:38.673 INFO:teuthology.orchestra.run.smithi064.stderr:bash: ntpq: command not found 2023-06-10T22:12:38.694 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-06-10T22:12:38.694 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2023-06-10T22:12:38.695 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 64 +1744us[+1751us] +/- 82ms 2023-06-10T22:12:38.695 INFO:teuthology.orchestra.run.smithi152.stdout:^* hv02.front.sepia.ceph.com 3 6 377 5 -2602us[-2595us] +/- 78ms 2023-06-10T22:12:38.695 INFO:teuthology.orchestra.run.smithi152.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +1137us[+1137us] +/- 92ms 2023-06-10T22:12:38.695 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T22:12:38.716 INFO:teuthology.orchestra.run.smithi064.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-06-10T22:12:38.716 INFO:teuthology.orchestra.run.smithi064.stdout:=============================================================================== 2023-06-10T22:12:38.716 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 60 +1707us[+1707us] +/- 83ms 2023-06-10T22:12:38.717 INFO:teuthology.orchestra.run.smithi064.stdout:^* hv02.front.sepia.ceph.com 3 6 377 61 -2613us[-2615us] +/- 78ms 2023-06-10T22:12:38.717 INFO:teuthology.orchestra.run.smithi064.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 60 +1090us[+1090us] +/- 91ms 2023-06-10T22:12:38.717 INFO:teuthology.orchestra.run.smithi064.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-06-10T22:12:38.718 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-06-10T22:12:38.728 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-06-10T22:12:38.728 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-06-10T22:12:38.738 DEBUG:teuthology.orchestra.run.smithi064:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-06-10T22:12:38.904 DEBUG:teuthology.orchestra.run.smithi152:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-06-10T22:12:39.058 DEBUG:teuthology.orchestra.run.smithi064:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-06-10T22:12:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T22:12:39.094 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-06-10T22:12:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T22:12:39.124 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-06-10T22:12:39.134 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-06-10T22:12:39.149 INFO:teuthology.task.internal:Duration was 5616.436198 seconds 2023-06-10T22:12:39.150 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-06-10T22:12:39.163 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-06-10T22:12:39.163 DEBUG:teuthology.orchestra.run.smithi064:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-06-10T22:12:39.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-06-10T22:12:39.231 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-06-10T22:12:39.260 INFO:teuthology.orchestra.run.smithi064.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-06-10T22:12:39.445 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-06-10T22:12:39.446 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi064.front.sepia.ceph.com 2023-06-10T22:12:39.446 DEBUG:teuthology.orchestra.run.smithi064:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-06-10T22:12:39.478 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi152.front.sepia.ceph.com 2023-06-10T22:12:39.478 DEBUG:teuthology.orchestra.run.smithi152:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-06-10T22:12:39.511 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-06-10T22:12:39.511 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-10T22:12:39.520 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-10T22:12:39.682 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-06-10T22:12:39.682 DEBUG:teuthology.orchestra.run.smithi064:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-06-10T22:12:39.685 DEBUG:teuthology.orchestra.run.smithi152:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-06-10T22:12:40.089 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-06-10T22:12:40.100 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-06-10T22:12:40.100 DEBUG:teuthology.orchestra.run.smithi064:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-06-10T22:12:40.138 DEBUG:teuthology.orchestra.run.smithi152:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-06-10T22:12:40.241 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-06-10T22:12:40.252 DEBUG:teuthology.orchestra.run.smithi064:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-06-10T22:12:40.254 DEBUG:teuthology.orchestra.run.smithi152:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-06-10T22:12:40.293 INFO:teuthology.orchestra.run.smithi064.stdout:kernel.core_pattern = core 2023-06-10T22:12:40.297 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = core 2023-06-10T22:12:40.316 DEBUG:teuthology.orchestra.run.smithi064:> test -e /home/ubuntu/cephtest/archive/coredump 2023-06-10T22:12:40.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T22:12:40.373 DEBUG:teuthology.orchestra.run.smithi152:> test -e /home/ubuntu/cephtest/archive/coredump 2023-06-10T22:12:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T22:12:40.389 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-06-10T22:12:40.400 INFO:teuthology.task.internal:Transferring archived files... 2023-06-10T22:12:40.402 DEBUG:teuthology.misc:Transferring archived files from smithi064:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi064 2023-06-10T22:12:40.402 DEBUG:teuthology.orchestra.run.smithi064:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-06-10T22:12:40.475 DEBUG:teuthology.misc:Transferring archived files from smithi152:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118/remote/smithi152 2023-06-10T22:12:40.475 DEBUG:teuthology.orchestra.run.smithi152:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-06-10T22:12:40.516 INFO:teuthology.task.internal:Removing archive directory... 2023-06-10T22:12:40.517 DEBUG:teuthology.orchestra.run.smithi064:> rm -rf -- /home/ubuntu/cephtest/archive 2023-06-10T22:12:40.519 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf -- /home/ubuntu/cephtest/archive 2023-06-10T22:12:40.566 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-06-10T22:12:40.579 INFO:teuthology.task.internal:Not uploading archives. 2023-06-10T22:12:40.579 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-06-10T22:12:40.589 INFO:teuthology.task.internal:Tidying up after the test... 2023-06-10T22:12:40.590 DEBUG:teuthology.orchestra.run.smithi064:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-06-10T22:12:40.592 DEBUG:teuthology.orchestra.run.smithi152:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-06-10T22:12:40.606 INFO:teuthology.orchestra.run.smithi064.stdout: 262320 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Jun 10 22:12 /home/ubuntu/cephtest 2023-06-10T22:12:40.607 INFO:teuthology.orchestra.run.smithi064.stdout: 6029337 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Jun 10 21:50 /home/ubuntu/cephtest/mnt.0 2023-06-10T22:12:40.607 INFO:teuthology.orchestra.run.smithi064.stdout: 6029341 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Jun 10 21:50 /home/ubuntu/cephtest/mnt.0/client.0 2023-06-10T22:12:40.607 INFO:teuthology.orchestra.run.smithi064.stdout: 6294322 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Jun 10 21:50 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-06-10T22:12:40.608 INFO:teuthology.orchestra.run.smithi064.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-06-10T22:12:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-10T22:12:40.610 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/run_tasks.py", line 192, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-06-10T22:12:40.615 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-06-10T22:12:40.627 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-06-10T22:12:40.639 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi064 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-06-10T22:12:40.639 INFO:teuthology.nuke:Checking targets against current locks 2023-06-10T22:12:40.645 INFO:teuthology.orchestra.run.smithi152.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 10 22:12 /home/ubuntu/cephtest 2023-06-10T22:12:40.679 DEBUG:teuthology.nuke:shortname: smithi064 2023-06-10T22:12:40.679 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-10T22:12:40.701 DEBUG:teuthology.nuke:shortname: smithi152 2023-06-10T22:12:40.702 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-10T22:12:40.705 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi064.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-10 20:33:39.468106', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac='} 2023-06-10T22:12:40.724 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-10_14:23:08-upgrade:pacific-x-reef-distro-default-smithi/7301118', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-10 20:33:39.466350', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac='} 2023-06-10T22:12:40.757 INFO:teuthology.orchestra.console:Power off smithi064 2023-06-10T22:12:40.758 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-06-10T22:12:40.773 INFO:teuthology.orchestra.console:Power off smithi152 2023-06-10T22:12:40.774 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-06-10T22:12:40.784 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:40.796 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:44.807 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:44.821 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:48.923 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi064.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:48.932 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi152.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-10T22:12:49.135 INFO:teuthology.orchestra.console:Power off for smithi064 completed 2023-06-10T22:12:49.145 INFO:teuthology.orchestra.console:Power off for smithi152 completed 2023-06-10T22:12:49.251 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 5616.436198234558 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi064 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=07dcb2527e444659b577878b95475688 status: fail success: false 2023-06-10T22:12:49.252 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-06-10T22:12:49.342 INFO:teuthology.run:FAIL